From 7c0a8febeb1d35018c53c0820f732825a3f05900 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 11 Aug 2025 16:44:26 +0300 Subject: [PATCH 01/29] Base version of automated test marker --- Lib/test/test_os.py | 193 ++++++++---------------- tools/test_marker/confs/test_os.toml | 103 +++++++++++++ tools/test_marker/main.py | 215 +++++++++++++++++++++++++++ 3 files changed, 379 insertions(+), 132 deletions(-) create mode 100644 tools/test_marker/confs/test_os.toml create mode 100755 tools/test_marker/main.py diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index a025a2d4ff..bfeaad0b89 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,7 +13,6 @@ import locale import os import pickle -import platform import select import selectors import shutil @@ -188,8 +187,7 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) @@ -716,8 +714,7 @@ def check_file_attributes(self, result): self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat return value doesnt have st_file_attributes attribute") @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -739,8 +736,7 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is @@ -763,8 +759,7 @@ def test_access_denied(self): self.assertNotEqual(result.st_size, 0) self.assertTrue(os.path.isfile(fname)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 1] Incorrect function") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -822,8 +817,7 @@ def _test_utime(self, set_time, filename=None): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -835,9 +829,7 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` - # return (ns * 1e-9) + 0.5e-9 - return (ns * 1e-9) + return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter @@ -889,8 +881,7 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -919,24 +910,21 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_utime_nonexistent(self): now = time.time() filename = 'nonexistent' @@ -957,8 +945,7 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -969,7 +956,7 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: NotImplementedError not raised") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1171,9 +1158,8 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: ValueError not raised by putenv") @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): # Empty variable name is invalid. @@ -1748,18 +1734,15 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components def test_compare_to_walk(self): return super().test_compare_to_walk() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components def test_dir_fd(self): return super().test_dir_fd() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components def test_yields_correct_dir_fd(self): return super().test_yields_correct_dir_fd() @@ -1811,8 +1794,7 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1831,8 +1813,7 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -2119,8 +2100,7 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'os'") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -2135,8 +2115,7 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -2224,8 +2203,7 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2285,8 +2263,7 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2308,8 +2285,7 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2390,8 +2366,7 @@ def check_bool(self, f, *args, **kwargs): with self.assertRaises(RuntimeWarning): f(fd, *args, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") self.check_bool(os.fdopen, encoding="utf-8") @@ -2448,8 +2423,7 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( support.is_emscripten or support.is_wasi, @@ -2462,22 +2436,19 @@ def test_fpathconf(self): self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) self.check_bool(os.truncate, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) @@ -2491,8 +2462,7 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2501,8 +2471,7 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) @@ -2514,13 +2483,11 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fchdir(self): return super().test_fchdir() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fsync(self): return super().test_fsync() @@ -2563,7 +2530,6 @@ def test_unicode_name(self): self.file2 = self.file1 + "2" self._test_link(self.file1, self.file2) - @unittest.skipIf(sys.platform == "win32", "Posix specific tests") class PosixUidGidTests(unittest.TestCase): # uid_t and gid_t are 32-bit unsigned integers on Linux @@ -2765,14 +2731,12 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2831,8 +2795,7 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3186,8 +3149,7 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'CreateJunction'") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -3201,8 +3163,7 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'CreateJunction'") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -3262,8 +3223,7 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat PermissionError: [Errno 5] Access is denied") @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading @@ -3473,8 +3433,7 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.spawnv not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3483,8 +3442,7 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust i32") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3608,8 +3566,7 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - # TODO: RUSTPYTHON fix spawnv bytes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -4224,16 +4181,8 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - - # XXX: RUSTPYTHON; AttributeError: module 'platform' has no attribute 'android_ver' - #if sys.platform == "android" and platform.android_ver().is_emulator: - if sys.platform == "android": - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4577,8 +4526,7 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : ) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4671,8 +4619,7 @@ def test_process_cpu_count_affinity(self): # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4717,8 +4664,7 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4734,8 +4680,7 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4749,7 +4694,6 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) - # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) @@ -4759,15 +4703,13 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) - # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.dup not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4907,8 +4849,7 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - # TODO: RUSTPYTHON (AssertionError: TypeError not raised) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4993,13 +4934,11 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by DirEntry def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -5044,8 +4983,7 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by ScandirIter) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by ScandirIter def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -5088,8 +5026,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON; flaky test") def test_attributes(self): link = os_helper.can_hardlink() symlink = os_helper.can_symlink() @@ -5189,8 +5126,7 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_dir() is False") def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -5213,8 +5149,7 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_file() is False") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -5275,8 +5210,7 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -5303,8 +5237,7 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: FileNotFoundError not raised by scandir") @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -5360,8 +5293,7 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -5434,8 +5366,7 @@ class A(os.PathLike): def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pathlike_subclass_slots(self): class A(os.PathLike): __slots__ = () @@ -5443,8 +5374,7 @@ def __fspath__(self): return '' self.assertFalse(hasattr(A(), '__dict__')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fspath_set_to_None(self): class Foo: __fspath__ = None @@ -5547,8 +5477,7 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fork_at_finalization(self): code = """if 1: import atexit diff --git a/tools/test_marker/confs/test_os.toml b/tools/test_marker/confs/test_os.toml new file mode 100644 index 0000000000..8b39f6b1e8 --- /dev/null +++ b/tools/test_marker/confs/test_os.toml @@ -0,0 +1,103 @@ +path = "Lib/test/test_os.py" + +[FileTests] +test_closerange = { skipIf = { cond = "sys.platform == 'win32'", reason = "BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')" } } + +[StatAttributeTests] +test_file_attributes = { expectedFailureIfWindows = { reason = "os.stat return value doesnt have st_file_attributes attribute" } } +test_access_denied = { expectedFailureIfWindows = { reason = "os.stat (PermissionError: [Errno 5] Access is denied" } } +test_stat_block_device = { expectedFailureIfWindows = { reason = "os.stat (PermissionError: [Errno 1] Incorrect function" } } + +[UtimeTests] +test_utime = { expectedFailureIfWindows = { reason = "AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)" } } +test_utime_directory = { expectedFailureIfWindows = { reason = "(AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)" } } +test_utime_current = { expectedFailureIfWindows = { reason = "AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005" } } +test_utime_current_old = { expectedFailureIfWindows = { reason = "AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052" } } +test_utime_nonexistent = { expectedFailure = { reason = "" } } +test_large_time = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } +test_utime_invalid_arguments = { expectedFailureIfWindows = { reason = "AssertionError: NotImplementedError not raised" } } + +[EnvironTests] +test_putenv_unsetenv_error = { expectedFailureIfWindows = { reason = "AssertionError: ValueError not raised by putenv" } } + +[BytesWalkTests] +test_compare_to_walk = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } +test_dir_fd = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } +test_yields_correct_dir_fd = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } + +[MakedirTests] +test_exist_ok_existing_directory = { expectedFailureIfWindows = { reason = "os.umask not implemented yet for all platforms" } } +test_exist_ok_s_isgid_directory = { expectedFailureIfWindows = { reason = "os.umask not implemented yet for all platforms" } } + +[URandomFDTests] +test_urandom_fd_closed = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named 'os'" } } +test_urandom_fd_reopened = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named 'os'" } } + +[ExecTests] +test_execvpe_with_bad_arglist = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } +test_execve_invalid_env = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } +test_execve_with_empty_path = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } + +[TestInvalidFD] +test_fdopen = { expectedFailure = { reason = "" } } +test_fpathconf = { expectedFailure = { reason = "" } } +test_ftruncate = { expectedFailure = { reason = "" } } +test_lseek = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } +test_read = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } +test_write = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } +test_inheritable = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } +test_fchdir = { expectedFailure = { reason = "" } } +test_fsync = { expectedFailure = { reason = "" } } + +[Win32KillTests] +test_kill_sigterm = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } +test_kill_int = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } +test_CTRL_BREAK_EVENT = { expectedFailure = { reason = "" } } + +[Win32JunctionTests] +test_create_junction = { expectedFailureIfWindows = { reason = "AttributeError: module '_winapi' has no attribute 'CreateJunction'" } } +test_unlink_removes_junction = { expectedFailureIfWindows = { reason = "AttributeError: module '_winapi' has no attribute 'CreateJunction'" } } + +[Win32NtTests] +test_stat_unlink_race = { expectedFailureIfWindows = { reason = "os.stat PermissionError: [Errno 5] Access is denied" } } + +[PidTests] +test_waitpid_windows = { expectedFailureIfWindows = { reason = "os.spawnv not implemented yet for all platforms" } } +test_waitstatus_to_exitcode_windows = { expectedFailureIfWindows = { reason = "OverflowError: Python int too large to convert to Rust i32" } } + +[SpawnTests] +test_spawnve_bytes = { expectedFailure = { reason = "fix spawnv bytes" } } + +[OSErrorTests] +test_oserror_filename = { expectedFailure = { reason = "AssertionError: b'@test_22106_tmp\\xe7w\\xf0' is not b'@test_22106_tmp\\xe7w\\xf0' : " } } + +[FDInheritanceTests] +test_get_set_inheritable = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } +test_get_set_inheritable_badf = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } +test_open = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } +test_dup = { skipIf = { cond = "sys.platform == 'win32'", reason = "os.dup on windows" } } +test_dup_standard_stream = { skipIf = { cond = "sys.platform == 'win32'", reason = "os.dup on windows" } } +test_dup_nul = { expectedFailureIfWindows = { reason = "os.dup not implemented yet for all platforms" } } + +[PathTConverterTests] +test_path_t_converter = { expectedFailure = { reason = "AssertionError: TypeError not raised" } } + +[TestDirEntry] +test_uninstantiable = { expectedFailure = { reason = "AssertionError: TypeError not raised by DirEntry" } } +test_unpickable = { expectedFailure = { reason = "pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry" } } + +[TestScandir] +test_uninstantiable = { expectedFailure = { reason = "AssertionError: TypeError not raised by ScandirIter" } } +test_attributes = { skipIf = { cond = "sys.platform == 'linux'", reason = "flaky test" } } +test_removed_dir = { expectedFailureIfWindows = { reason = "entry.is_dir() is False" } } +test_removed_file = { expectedFailureIfWindows = { reason = "entry.is_file() is False" } } +test_fd = { expectedFailure = { reason = "" } } +test_empty_path = { expectedFailureIfWindows = { reason = "AssertionError: FileNotFoundError not raised by scandir" } } +test_resource_warning = { expectedFailure = { reason = "" } } + +[TestPEP519] +test_pathlike_subclass_slots = { expectedFailure = { reason = "" } } +test_fspath_set_to_None = { expectedFailure = { reason = "" } } + +[ForkTests] +test_fork_at_finalization = { expectedFailure = { reason = "" } } diff --git a/tools/test_marker/main.py b/tools/test_marker/main.py new file mode 100755 index 0000000000..14c40b2624 --- /dev/null +++ b/tools/test_marker/main.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python +import ast +import pathlib +import tomllib +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + +COL_OFFSET = 4 +INDENT1 = " " * COL_OFFSET +INDENT2 = " " * COL_OFFSET * 2 +COMMENT = "TODO: RUSTPYTHON" + +ROOT_DIR = pathlib.Path(__file__).parents[2] +CONFS = ROOT_DIR / "tools" / "test_marker" / "confs" + + +type Patch = dict[str, dict[str, str]] +type Conf = dict[str, Patch] + + +def format_patch(patch_conf: Patch) -> str: + """ + Transforms a patch definition to a raw python code. + + Parameters + ---------- + patch_conf : Patch + Conf of the patch. + + Returns + ------- + str + Raw python source code. + + Examples + -------- + >>> patch = {"expectedFailure": {"reason": "lorem ipsum"}} + >>> format_patch(patch) + '@unittest.expectedFailure # TODO: RUSTPYTHON; lorem ipsum' + """ + method, conf = next(iter(patch_conf.items())) + prefix = f"@unittest.{method}" + + reason = conf.get("reason", "") + res = "" + match method: + case "expectedFailure": + res = f"{prefix} # {COMMENT}; {reason}" + case "expectedFailureIfWindows" | "skip": + res = f'{prefix}("{COMMENT}; {reason}")' + case "skipIf": + cond = conf["cond"] + res = f'{prefix}({cond}, "{COMMENT}; {reason}")' + + return res.strip().rstrip(";").strip() + + +def is_patch_present(node: ast.Attribute | ast.Call, patch_conf: Patch) -> bool: + """ + Detect whether an AST node (of a decorator) is matching to our patch. + + We accept both `ast.Attribute` and `ast.Call` because: + * ast.Attribute: `@unittest.expectedFailure` + * ast.Call: `@unittest.expectedFailureIfWindows(...)` / `@unittest.skipIf(...)` + + Parameters + ---------- + node : ast.Attribute | ast.Call + AST node to query. + patch_conf : Patch + Patch(es) to match against. + + Returns + ------- + bool + Whether or not we got a match. + """ + is_attr = isinstance(node, ast.Attribute) + attr_node = node if is_attr else node.func + + if isinstance(attr_node, ast.Name): + return False + + if attr_node.value.id != "unittest": + return False + + if is_attr: + return node.attr in patch_conf + + return "RUSTPYTHON" in ast.unparse(node) + + +def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": + """ + Get needed patches to apply for given ast tree based on the conf. + + Parameters + ---------- + tree : ast.Module + AST tree to iterate on. + conf : Conf + Dict of `{ClassName: {test_name: Patch}}`. + + Yields + ------ + lineno : int + Line number where to insert the patch. + patch : str + Raw python code to be inserted at `lineno`. + """ + # Phase 1: Iterate and mark existing tests + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for i, cls_node in enumerate(nodes): + if not isinstance(cls_node, ast.ClassDef): + continue + + """ + print(dir(cls_node)) + print(cls_node) + print(cls_node.lineno) + print(cls_node.end_lineno) + exit() + """ + + if not (cls_conf := conf.get(cls_node.name)): + continue + + for fn_node in cls_node.body: + if not isinstance(fn_node, ast.FunctionDef): + continue + + if not (patch_conf := cls_conf.pop(fn_node.name, None)): + continue + + if any( + is_patch_present(dec_node, patch_conf) + for dec_node in fn_node.decorator_list + if isinstance(dec_node, (ast.Attribute, ast.Call)) + ): + continue + lineno = min( + (dec_node.lineno for dec_node in fn_node.decorator_list), + default=fn_node.lineno, + ) + + indent = " " * fn_node.col_offset + patch = format_patch(patch_conf) + yield (lineno - 1, f"{indent}{patch}") + + # Phase 2: Iterate and mark inhereted tests + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for i, cls_node in enumerate(nodes): + if not isinstance(cls_node, ast.ClassDef): + continue + + if not (cls_conf := conf.get(cls_node.name)): + continue + + for fn_name, patch_conf in cls_conf.items(): + patch = format_patch(patch_conf) + yield ( + cls_node.end_lineno, + f""" +{INDENT1}{patch} +{INDENT1}def {fn_name}(self): +{INDENT2}return super().{fn_name}() +""".rstrip(), + ) + + +def apply_conf(contents: str, conf: dict) -> str: + """ + Patch a given source code based on the conf. + + Parameters + ---------- + contents : str + Raw python source code. + conf : Conf + Dict of `{ClassName: {test_name: Patch}}`. + + Returns + ------- + str + Patched raw python code. + """ + lines = contents.splitlines() + tree = ast.parse(contents) + + # Going in reverse to not distrupt the lines offset + patches = list(iter_patches(tree, conf)) + for lineno, patch in sorted(patches, reverse=True): + lines.insert(lineno, patch) + + return "\n".join(lines) + + +def main(): + for conf_file in CONFS.rglob("*.toml"): + conf = tomllib.loads(conf_file.read_text()) + path = ROOT_DIR / conf.pop("path") + patched = apply_conf(path.read_text(), conf) + path.write_text(patched + "\n") + + +if __name__ == "__main__": + main() From 69ed76b6ea74959a0ff8f6e057e91b8c1a5134f5 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 11 Aug 2025 18:46:58 +0300 Subject: [PATCH 02/29] Remove debug code --- tools/test_marker/main.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tools/test_marker/main.py b/tools/test_marker/main.py index 14c40b2624..d2334e3075 100755 --- a/tools/test_marker/main.py +++ b/tools/test_marker/main.py @@ -119,14 +119,6 @@ def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": if not isinstance(cls_node, ast.ClassDef): continue - """ - print(dir(cls_node)) - print(cls_node) - print(cls_node.lineno) - print(cls_node.end_lineno) - exit() - """ - if not (cls_conf := conf.get(cls_node.name)): continue From dc3a6de1f7319212aceaf60a561a0aa51d4b1f4d Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 11 Aug 2025 18:48:13 +0300 Subject: [PATCH 03/29] Add newline --- tools/test_marker/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/test_marker/main.py b/tools/test_marker/main.py index d2334e3075..01a9593329 100755 --- a/tools/test_marker/main.py +++ b/tools/test_marker/main.py @@ -135,6 +135,7 @@ def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": if isinstance(dec_node, (ast.Attribute, ast.Call)) ): continue + lineno = min( (dec_node.lineno for dec_node in fn_node.decorator_list), default=fn_node.lineno, From 5c09ef01ed59abc4c4aa30637bfda2639461733f Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 11 Aug 2025 19:07:13 +0300 Subject: [PATCH 04/29] Apply RustPython patch --- Lib/test/test_os.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bfeaad0b89..7816bb3274 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -829,7 +829,9 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - return (ns * 1e-9) + 0.5e-9 + # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` + # return (ns * 1e-9) + 0.5e-9 + return (ns * 1e-9) def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter From ad061b2efe0981fd87bb5cb00e882205053a8ba6 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:23:09 +0300 Subject: [PATCH 05/29] Convert to tool with args --- Lib/argparse.py | 1 + Lib/ast.py | 1 + Lib/decimal.py | 1 + Lib/heapq.py | 1 + Lib/ipaddress.py | 1 + Lib/numbers.py | 12 +- Lib/os.py | 3 +- Lib/pprint.py | 1 + Lib/queue.py | 12 +- Lib/test/test_os.py | 5 +- Lib/this.py | 1 + tools/rpau/README.md | 1 + tools/rpau/confs/Lib/argparse.toml | 1 + tools/rpau/confs/Lib/ast.toml | 1 + tools/rpau/confs/Lib/decimal.toml | 1 + tools/rpau/confs/Lib/heapq.toml | 1 + tools/rpau/confs/Lib/ipaddress.toml | 1 + tools/rpau/confs/Lib/numbers.toml | 1 + tools/rpau/confs/Lib/os.toml | 1 + tools/rpau/confs/Lib/pprint.toml | 1 + tools/rpau/confs/Lib/queue.toml | 1 + .../confs/Lib/test}/test_os.toml | 0 tools/rpau/confs/Lib/this.toml | 1 + tools/{test_marker => rpau}/main.py | 2 +- tools/rpau/pyproject.toml | 17 ++ tools/rpau/src/rpau/__init__.py | 75 +++++++ tools/rpau/src/rpau/cli.py | 134 ++++++++++++ tools/rpau/src/rpau/logger.py | 25 +++ tools/rpau/src/rpau/logic.py | 205 ++++++++++++++++++ tools/rpau/uv.lock | 8 + 30 files changed, 504 insertions(+), 12 deletions(-) create mode 100644 tools/rpau/README.md create mode 100644 tools/rpau/confs/Lib/argparse.toml create mode 100644 tools/rpau/confs/Lib/ast.toml create mode 100644 tools/rpau/confs/Lib/decimal.toml create mode 100644 tools/rpau/confs/Lib/heapq.toml create mode 100644 tools/rpau/confs/Lib/ipaddress.toml create mode 100644 tools/rpau/confs/Lib/numbers.toml create mode 100644 tools/rpau/confs/Lib/os.toml create mode 100644 tools/rpau/confs/Lib/pprint.toml create mode 100644 tools/rpau/confs/Lib/queue.toml rename tools/{test_marker/confs => rpau/confs/Lib/test}/test_os.toml (100%) create mode 100644 tools/rpau/confs/Lib/this.toml rename tools/{test_marker => rpau}/main.py (99%) create mode 100644 tools/rpau/pyproject.toml create mode 100644 tools/rpau/src/rpau/__init__.py create mode 100644 tools/rpau/src/rpau/cli.py create mode 100644 tools/rpau/src/rpau/logger.py create mode 100644 tools/rpau/src/rpau/logic.py create mode 100644 tools/rpau/uv.lock diff --git a/Lib/argparse.py b/Lib/argparse.py index bd088ea0e6..38afbf007e 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Author: Steven J. Bethard . # New maintainer as of 29 August 2019: Raymond Hettinger diff --git a/Lib/ast.py b/Lib/ast.py index 37b20206b8..c785bed409 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """ The `ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with diff --git a/Lib/decimal.py b/Lib/decimal.py index ee3147f5dd..71d612763b 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Decimal fixed-point and floating-point arithmetic. This is an implementation of decimal floating-point arithmetic based on diff --git a/Lib/heapq.py b/Lib/heapq.py index 2fd9d1ff4b..31910cce8a 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Heap queue algorithm (a.k.a. priority queue). Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 67e45450fc..af2fceb508 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Copyright 2007 Google Inc. # Licensed to PSF under a Contributor Agreement. diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cf..4fca6f9229 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. @@ -290,18 +291,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/os.py b/Lib/os.py index b4c9f84c36..d1c895c2be 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 r"""OS routines for NT or Posix depending on what system we're on. This exports: @@ -10,7 +11,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') + - os.linesep is the line separator in text files ('\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) diff --git a/Lib/pprint.py b/Lib/pprint.py index 9314701db3..d9b1ea69b0 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Author: Fred L. Drake, Jr. # fdrake@acm.org # diff --git a/Lib/queue.py b/Lib/queue.py index 25beb46e30..5cb1c4d8bc 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 '''A multi-producer, multi-consumer queue.''' import threading @@ -80,9 +81,6 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -239,9 +237,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 7816bb3274..6d3800fda2 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # As a test suite for the os module, this is woefully inadequate, but this # does add tests for a few functions which have been determined to be more # portable than they had been thought to be. @@ -829,9 +830,7 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` - # return (ns * 1e-9) + 0.5e-9 - return (ns * 1e-9) + return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter diff --git a/Lib/this.py b/Lib/this.py index e68dd3ff39..8e9f9cc378 100644 --- a/Lib/this.py +++ b/Lib/this.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 s = """Gur Mra bs Clguba, ol Gvz Crgref Ornhgvshy vf orggre guna htyl. diff --git a/tools/rpau/README.md b/tools/rpau/README.md new file mode 100644 index 0000000000..f4d6f3ff72 --- /dev/null +++ b/tools/rpau/README.md @@ -0,0 +1 @@ +# **R**ust**P**ython **A**utomatic **U**pdater (rpau) diff --git a/tools/rpau/confs/Lib/argparse.toml b/tools/rpau/confs/Lib/argparse.toml new file mode 100644 index 0000000000..440838d9b4 --- /dev/null +++ b/tools/rpau/confs/Lib/argparse.toml @@ -0,0 +1 @@ +path = "Lib/argparse.py" diff --git a/tools/rpau/confs/Lib/ast.toml b/tools/rpau/confs/Lib/ast.toml new file mode 100644 index 0000000000..220af23775 --- /dev/null +++ b/tools/rpau/confs/Lib/ast.toml @@ -0,0 +1 @@ +path = "Lib/ast.py" diff --git a/tools/rpau/confs/Lib/decimal.toml b/tools/rpau/confs/Lib/decimal.toml new file mode 100644 index 0000000000..8633f1e1d9 --- /dev/null +++ b/tools/rpau/confs/Lib/decimal.toml @@ -0,0 +1 @@ +path = "Lib/decimal.py" diff --git a/tools/rpau/confs/Lib/heapq.toml b/tools/rpau/confs/Lib/heapq.toml new file mode 100644 index 0000000000..5d9ea04401 --- /dev/null +++ b/tools/rpau/confs/Lib/heapq.toml @@ -0,0 +1 @@ +path = "Lib/heapq.py" diff --git a/tools/rpau/confs/Lib/ipaddress.toml b/tools/rpau/confs/Lib/ipaddress.toml new file mode 100644 index 0000000000..520f81df3c --- /dev/null +++ b/tools/rpau/confs/Lib/ipaddress.toml @@ -0,0 +1 @@ +path = "Lib/ipaddress.py" diff --git a/tools/rpau/confs/Lib/numbers.toml b/tools/rpau/confs/Lib/numbers.toml new file mode 100644 index 0000000000..dd7c043a2e --- /dev/null +++ b/tools/rpau/confs/Lib/numbers.toml @@ -0,0 +1 @@ +path = "Lib/numbers.py" diff --git a/tools/rpau/confs/Lib/os.toml b/tools/rpau/confs/Lib/os.toml new file mode 100644 index 0000000000..d65c63a9a3 --- /dev/null +++ b/tools/rpau/confs/Lib/os.toml @@ -0,0 +1 @@ +path = "Lib/os.py" diff --git a/tools/rpau/confs/Lib/pprint.toml b/tools/rpau/confs/Lib/pprint.toml new file mode 100644 index 0000000000..93c1edc92a --- /dev/null +++ b/tools/rpau/confs/Lib/pprint.toml @@ -0,0 +1 @@ +path = "Lib/pprint.py" diff --git a/tools/rpau/confs/Lib/queue.toml b/tools/rpau/confs/Lib/queue.toml new file mode 100644 index 0000000000..164fc36c51 --- /dev/null +++ b/tools/rpau/confs/Lib/queue.toml @@ -0,0 +1 @@ +path = "Lib/queue.py" diff --git a/tools/test_marker/confs/test_os.toml b/tools/rpau/confs/Lib/test/test_os.toml similarity index 100% rename from tools/test_marker/confs/test_os.toml rename to tools/rpau/confs/Lib/test/test_os.toml diff --git a/tools/rpau/confs/Lib/this.toml b/tools/rpau/confs/Lib/this.toml new file mode 100644 index 0000000000..f976d3ccc8 --- /dev/null +++ b/tools/rpau/confs/Lib/this.toml @@ -0,0 +1 @@ +path = "Lib/this.py" diff --git a/tools/test_marker/main.py b/tools/rpau/main.py similarity index 99% rename from tools/test_marker/main.py rename to tools/rpau/main.py index 01a9593329..18c60f174e 100755 --- a/tools/test_marker/main.py +++ b/tools/rpau/main.py @@ -9,7 +9,7 @@ COL_OFFSET = 4 INDENT1 = " " * COL_OFFSET -INDENT2 = " " * COL_OFFSET * 2 +INDENT2 = INDENT1 * 2 COMMENT = "TODO: RUSTPYTHON" ROOT_DIR = pathlib.Path(__file__).parents[2] diff --git a/tools/rpau/pyproject.toml b/tools/rpau/pyproject.toml new file mode 100644 index 0000000000..181e2d8c9a --- /dev/null +++ b/tools/rpau/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "rpau" +version = "0.1.0" +description = "Tool for automaticly update Lib & Tests from CPython" +readme = "README.md" +authors = [ + { name = "RustPython Team", email = "" } +] +requires-python = ">=3.12" +dependencies = [] + +[project.scripts] +rpau = "rpau:main" + +[build-system] +requires = ["uv_build>=0.8.11,<0.9.0"] +build-backend = "uv_build" diff --git a/tools/rpau/src/rpau/__init__.py b/tools/rpau/src/rpau/__init__.py new file mode 100644 index 0000000000..bed3078e33 --- /dev/null +++ b/tools/rpau/src/rpau/__init__.py @@ -0,0 +1,75 @@ +import functools +from concurrent.futures import ProcessPoolExecutor +from typing import TYPE_CHECKING + +import tomllib + +from rpau.cli import build_argparse +from rpau.logger import build_root_logger, get_logger +from rpau.logic import run + +if TYPE_CHECKING: + import pathlib + import re + from collections.abc import Iterator + + +def iter_confs( + conf_dir: "pathlib.Path", *, include: "re.Pattern", exclude: "re.Pattern" +) -> "Iterator[pathlib.Path]": + for conf_file in conf_dir.rglob("**/*.toml"): + if not conf_file.is_file(): + continue + + uri = conf_file.as_uri().removeprefix("file://") + if not include.match(uri): + continue + + if exclude.match(uri): + continue + + yield conf_file + + +def main() -> None: + parser = build_argparse() + args = parser.parse_args() + + logger = build_root_logger(level=args.log_level) + logger.debug(f"{args=}") + + if args.cache: + logger.debug(f"Ensuring {args.cache_dir} exists") + cache_dir = args.cache_dir + cache_dir.mkdir(parents=True, exist_ok=True) + else: + cache_dir = None + + conf_dir = args.conf_dir + with ProcessPoolExecutor(args.workers) as executor: + for conf_file in iter_confs( + conf_dir, include=args.include, exclude=args.exclude + ): + try: + conf = tomllib.loads(conf_file.read_text()) + except tomllib.TOMLDecodeError as err: + logger.warn(f"{conf_file}: {err}") + continue + + try: + path = conf.pop("path") + except KeyError: + logger.warn(f"{conf_file}: has no 'path' key. skipping") + continue + + version = conf.pop("version", args.default_version) + func = functools.partial( + run, + path=path, + conf=conf, + cache_dir=cache_dir, + version=version, + output_dir=args.output_dir, + base_upstream_url=args.base_upstream_url, + ) + executor.submit(func) diff --git a/tools/rpau/src/rpau/cli.py b/tools/rpau/src/rpau/cli.py new file mode 100644 index 0000000000..39da33711f --- /dev/null +++ b/tools/rpau/src/rpau/cli.py @@ -0,0 +1,134 @@ +import argparse +import logging +import os +import pathlib +import re +import sys + + +class CustomArgumentParser(argparse.ArgumentParser): + class _CustomHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): + def _get_help_string(self, action): + help_msg = super()._get_help_string(action) + if action.dest != "help": + env_name = f"{self._prog}_{action.dest}".upper() + env_value = os.environ.get(env_name, "") + help_msg += f" [env: {env_name}={env_value}]" + return help_msg + + def __init__(self, *, formatter_class=_CustomHelpFormatter, **kwargs): + super().__init__(formatter_class=formatter_class, **kwargs) + + def _add_action(self, action): + action.default = os.environ.get( + f"{self.prog}_{action.dest}".upper(), action.default + ) + return super()._add_action(action) + + +def get_cache_dir(prog: str) -> pathlib.Path: + home = pathlib.Path.home() + + if sys.platform.startswith("win"): + local_appdata = pathlib.Path( + os.getenv("LOCALAPPDATA", home / "AppData" / "Local") + ) + path = local_appdata / prog / prog / "Cache" + elif sys.platform == "darwin": + path = home / "Library" / "Caches" / prog + else: + cache_home = pathlib.Path(os.getenv("XDG_CACHE_HOME", home / ".cache")) + path = cache_home / prog + + return path + + +def build_argparse() -> argparse.ArgumentParser: + parser = CustomArgumentParser( + prog="rpau", description="Automatic update code from CPython" + ) + + # Cache + cache_group = parser.add_argument_group("Cache options") + + cache_group.add_argument( + "--cache", + action=argparse.BooleanOptionalAction, + default=True, + help="Whether reading from or writing to the cache is allowed", + ) + cache_group.add_argument( + "--cache-dir", + default=get_cache_dir(parser.prog), + help="Path to the cache directory", + metavar="PATH", + type=pathlib.Path, + ) + + # Output + output_group = parser.add_argument_group("Output option") + + output_group.add_argument( + "-o", + "--output-dir", + default=pathlib.Path(__file__).parents[4], + help="Output dir", + metavar="PATH", + type=pathlib.Path, + ) + + # Filter + filter_group = parser.add_argument_group("Filter options") + + filter_group.add_argument( + "-i", + "--include", + default=".*", + help="RE pattern used to include files and/or directories", + metavar="PATTERN", + type=re.compile, + ) + filter_group.add_argument( + "-e", + "--exclude", + default="^$", + help="RE pattern used to omit files and/or directories", + metavar="PATTERN", + type=re.compile, + ) + + # Global + global_group = parser.add_argument_group("Global options") + + global_group.add_argument( + "--log-level", + choices=logging.getLevelNamesMapping(), + default="WARNING", + help="Log level", + ) + + global_group.add_argument( + "-j", "--workers", default=1, help="Number of processes", type=int + ) + global_group.add_argument( + "-c", + "--conf-dir", + default=pathlib.Path(__file__).parents[2] / "confs", + help="Path to conf dir", + metavar="PATH", + type=pathlib.Path, + ) + global_group.add_argument( + "--base-upstream-url", + default="https://raw.githubusercontent.com/python/cpython/refs/tags", + help="Base upstream url of CPython", + metavar="URL", + ) + global_group.add_argument( + "--default-version", + default="v3.13.7", + help="Fallback version of cpython if none specified in conf", + metavar="VERSION_TAG", + ) + + return parser diff --git a/tools/rpau/src/rpau/logger.py b/tools/rpau/src/rpau/logger.py new file mode 100644 index 0000000000..5af549b2b6 --- /dev/null +++ b/tools/rpau/src/rpau/logger.py @@ -0,0 +1,25 @@ +import logging +import sys + +_NAME = "rpau" + + +def build_root_logger( + name: str = _NAME, level: int = logging.WARNING +) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(level) + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + + sh = logging.StreamHandler(sys.stdout) + sh.setFormatter(formatter) + logger.handlers.clear() + logger.addHandler(sh) + + return logger + + +def get_logger(name: str) -> logging.Logger: + return logging.getLogger(_NAME).getChild(name) diff --git a/tools/rpau/src/rpau/logic.py b/tools/rpau/src/rpau/logic.py new file mode 100644 index 0000000000..f0f09fc495 --- /dev/null +++ b/tools/rpau/src/rpau/logic.py @@ -0,0 +1,205 @@ +import ast +import urllib.request +from typing import TYPE_CHECKING + +from rpau.logger import get_logger + +if TYPE_CHECKING: + import pathlib + +COL_OFFSET = 4 +INDENT1 = " " * COL_OFFSET +INDENT2 = INDENT1 * 2 +COMMENT = "TODO: RUSTPYTHON" + +type Patch = dict[str, dict[str, str]] +type Conf = dict[str, Patch] + +logger = get_logger(__name__) + +def fetch_upstream(*, base_url: str, path: str, version: str) -> str: + upstream_url = "/".join((base_url, version, path)) + logger.debug(f"{upstream_url=}") + + with urllib.request.urlopen(upstream_url) as f: + contents = f.read().decode() + return contents + + +def get_upstream_contents( + *, base_url: str, cache_dir: "pathlib.Path | None", path: str, version: str +) -> str: + fetch = lambda: fetch_upstream(base_url=base_url, path=path, version=version) + + if cache_dir: + cached_file = cache_dir / version / path + try: + contents = cached_file.read_text() + except FileNotFoundError: + cached_file.parent.mkdir(parents=True, exist_ok=True) + contents = fetch() + cached_file.write_text(contents) + + return contents + else: + return fetch() + +def format_patch(patch_conf: Patch) -> str: + """ + Transforms a patch definition to a raw python code. + + Parameters + ---------- + patch_conf : Patch + Conf of the patch. + + Returns + ------- + str + Raw python source code. + + Examples + -------- + >>> patch = {"expectedFailure": {"reason": "lorem ipsum"}} + >>> format_patch(patch) + '@unittest.expectedFailure # TODO: RUSTPYTHON; lorem ipsum' + """ + method, conf = next(iter(patch_conf.items())) + prefix = f"@unittest.{method}" + + reason = conf.get("reason", "") + res = "" + match method: + case "expectedFailure": + res = f"{prefix} # {COMMENT}; {reason}" + case "expectedFailureIfWindows" | "skip": + res = f'{prefix}("{COMMENT}; {reason}")' + case "skipIf": + cond = conf["cond"] + res = f'{prefix}({cond}, "{COMMENT}; {reason}")' + + return res.strip().rstrip(";").strip() + +def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": + """ + Get needed patches to apply for given ast tree based on the conf. + + Parameters + ---------- + tree : ast.Module + AST tree to iterate on. + conf : Conf + Dict of `{ClassName: {test_name: Patch}}`. + + Yields + ------ + lineno : int + Line number where to insert the patch. + patch : str + Raw python code to be inserted at `lineno`. + """ + # Phase 1: Iterate and mark existing tests + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for i, cls_node in enumerate(nodes): + if not isinstance(cls_node, ast.ClassDef): + continue + + if not (cls_conf := conf.get(cls_node.name)): + continue + + for fn_node in cls_node.body: + if not isinstance(fn_node, ast.FunctionDef): + continue + + if not (patch_conf := cls_conf.pop(fn_node.name, None)): + continue + + ''' + if any( + is_patch_present(dec_node, patch_conf) + for dec_node in fn_node.decorator_list + if isinstance(dec_node, (ast.Attribute, ast.Call)) + ): + continue + ''' + + lineno = min( + (dec_node.lineno for dec_node in fn_node.decorator_list), + default=fn_node.lineno, + ) + + indent = " " * fn_node.col_offset + patch = format_patch(patch_conf) + yield (lineno - 1, f"{indent}{patch}") + + # Phase 2: Iterate and mark inhereted tests + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for i, cls_node in enumerate(nodes): + if not isinstance(cls_node, ast.ClassDef): + continue + + if not (cls_conf := conf.get(cls_node.name)): + continue + + for fn_name, patch_conf in cls_conf.items(): + patch = format_patch(patch_conf) + yield ( + cls_node.end_lineno, + f""" +{INDENT1}{patch} +{INDENT1}def {fn_name}(self): +{INDENT2}return super().{fn_name}() +""".rstrip(), + ) + +def apply_conf(contents: str, conf: dict) -> str: + """ + Patch a given source code based on the conf. + + Parameters + ---------- + contents : str + Raw python source code. + conf : Conf + Dict of `{ClassName: {test_name: Patch}}`. + + Returns + ------- + str + Patched raw python code. + """ + lines = contents.splitlines() + tree = ast.parse(contents) + + # Going in reverse to not distrupt the line offset + patches = list(iter_patches(tree, conf)) + for lineno, patch in sorted(patches, reverse=True): + lines.insert(lineno, patch) + + return "\n".join(lines) + +def run( + conf: dict, + path: str, + version: str, + output_dir: "pathlib.Path", + cache_dir: "pathlib.Path | None", + base_upstream_url: str, +): + contents = get_upstream_contents( + path=path, version=version, base_url=base_upstream_url, cache_dir=cache_dir + ) + + patched_contents=apply_conf(contents, conf) + new_contents = f"# upstream_version: {version}\n{patched_contents}" + + output_file = output_dir / path + # TODO: Add logic to preserve file permissions if exists + output_file.parent.mkdir(parents=True, exist_ok=True) + output_file.write_text(f"{new_contents}\n") diff --git a/tools/rpau/uv.lock b/tools/rpau/uv.lock new file mode 100644 index 0000000000..813d8d2ed9 --- /dev/null +++ b/tools/rpau/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "rpau" +version = "0.1.0" +source = { editable = "." } From a6f2324cddbd3fa5c7e4b03fd3842db3b26bad12 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:44:44 +0300 Subject: [PATCH 06/29] Apply patch --- Lib/test/test_os.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 6d3800fda2..4a54971b8a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -830,7 +830,9 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - return (ns * 1e-9) + 0.5e-9 + # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` + # return (ns * 1e-9) + 0.5e-9 + return (ns * 1e-9) def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter From 302b3d1c027acc1099f601d536c37d1a0c7e96c1 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:47:05 +0300 Subject: [PATCH 07/29] Remove old script --- tools/rpau/main.py | 208 --------------------------------------------- 1 file changed, 208 deletions(-) delete mode 100755 tools/rpau/main.py diff --git a/tools/rpau/main.py b/tools/rpau/main.py deleted file mode 100755 index 18c60f174e..0000000000 --- a/tools/rpau/main.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python -import ast -import pathlib -import tomllib -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Iterator - -COL_OFFSET = 4 -INDENT1 = " " * COL_OFFSET -INDENT2 = INDENT1 * 2 -COMMENT = "TODO: RUSTPYTHON" - -ROOT_DIR = pathlib.Path(__file__).parents[2] -CONFS = ROOT_DIR / "tools" / "test_marker" / "confs" - - -type Patch = dict[str, dict[str, str]] -type Conf = dict[str, Patch] - - -def format_patch(patch_conf: Patch) -> str: - """ - Transforms a patch definition to a raw python code. - - Parameters - ---------- - patch_conf : Patch - Conf of the patch. - - Returns - ------- - str - Raw python source code. - - Examples - -------- - >>> patch = {"expectedFailure": {"reason": "lorem ipsum"}} - >>> format_patch(patch) - '@unittest.expectedFailure # TODO: RUSTPYTHON; lorem ipsum' - """ - method, conf = next(iter(patch_conf.items())) - prefix = f"@unittest.{method}" - - reason = conf.get("reason", "") - res = "" - match method: - case "expectedFailure": - res = f"{prefix} # {COMMENT}; {reason}" - case "expectedFailureIfWindows" | "skip": - res = f'{prefix}("{COMMENT}; {reason}")' - case "skipIf": - cond = conf["cond"] - res = f'{prefix}({cond}, "{COMMENT}; {reason}")' - - return res.strip().rstrip(";").strip() - - -def is_patch_present(node: ast.Attribute | ast.Call, patch_conf: Patch) -> bool: - """ - Detect whether an AST node (of a decorator) is matching to our patch. - - We accept both `ast.Attribute` and `ast.Call` because: - * ast.Attribute: `@unittest.expectedFailure` - * ast.Call: `@unittest.expectedFailureIfWindows(...)` / `@unittest.skipIf(...)` - - Parameters - ---------- - node : ast.Attribute | ast.Call - AST node to query. - patch_conf : Patch - Patch(es) to match against. - - Returns - ------- - bool - Whether or not we got a match. - """ - is_attr = isinstance(node, ast.Attribute) - attr_node = node if is_attr else node.func - - if isinstance(attr_node, ast.Name): - return False - - if attr_node.value.id != "unittest": - return False - - if is_attr: - return node.attr in patch_conf - - return "RUSTPYTHON" in ast.unparse(node) - - -def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": - """ - Get needed patches to apply for given ast tree based on the conf. - - Parameters - ---------- - tree : ast.Module - AST tree to iterate on. - conf : Conf - Dict of `{ClassName: {test_name: Patch}}`. - - Yields - ------ - lineno : int - Line number where to insert the patch. - patch : str - Raw python code to be inserted at `lineno`. - """ - # Phase 1: Iterate and mark existing tests - for key, nodes in ast.iter_fields(tree): - if key != "body": - continue - - for i, cls_node in enumerate(nodes): - if not isinstance(cls_node, ast.ClassDef): - continue - - if not (cls_conf := conf.get(cls_node.name)): - continue - - for fn_node in cls_node.body: - if not isinstance(fn_node, ast.FunctionDef): - continue - - if not (patch_conf := cls_conf.pop(fn_node.name, None)): - continue - - if any( - is_patch_present(dec_node, patch_conf) - for dec_node in fn_node.decorator_list - if isinstance(dec_node, (ast.Attribute, ast.Call)) - ): - continue - - lineno = min( - (dec_node.lineno for dec_node in fn_node.decorator_list), - default=fn_node.lineno, - ) - - indent = " " * fn_node.col_offset - patch = format_patch(patch_conf) - yield (lineno - 1, f"{indent}{patch}") - - # Phase 2: Iterate and mark inhereted tests - for key, nodes in ast.iter_fields(tree): - if key != "body": - continue - - for i, cls_node in enumerate(nodes): - if not isinstance(cls_node, ast.ClassDef): - continue - - if not (cls_conf := conf.get(cls_node.name)): - continue - - for fn_name, patch_conf in cls_conf.items(): - patch = format_patch(patch_conf) - yield ( - cls_node.end_lineno, - f""" -{INDENT1}{patch} -{INDENT1}def {fn_name}(self): -{INDENT2}return super().{fn_name}() -""".rstrip(), - ) - - -def apply_conf(contents: str, conf: dict) -> str: - """ - Patch a given source code based on the conf. - - Parameters - ---------- - contents : str - Raw python source code. - conf : Conf - Dict of `{ClassName: {test_name: Patch}}`. - - Returns - ------- - str - Patched raw python code. - """ - lines = contents.splitlines() - tree = ast.parse(contents) - - # Going in reverse to not distrupt the lines offset - patches = list(iter_patches(tree, conf)) - for lineno, patch in sorted(patches, reverse=True): - lines.insert(lineno, patch) - - return "\n".join(lines) - - -def main(): - for conf_file in CONFS.rglob("*.toml"): - conf = tomllib.loads(conf_file.read_text()) - path = ROOT_DIR / conf.pop("path") - patched = apply_conf(path.read_text(), conf) - path.write_text(patched + "\n") - - -if __name__ == "__main__": - main() From db74d2f124288472f6462573177ca28ae0e5453c Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:54:49 +0300 Subject: [PATCH 08/29] Add textwrap.py --- Lib/textwrap.py | 1 + tools/rpau/confs/Lib/textwrap.toml | 1 + 2 files changed, 2 insertions(+) create mode 100644 tools/rpau/confs/Lib/textwrap.toml diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 7ca393d1c3..f85b003efd 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Text wrapping and filling. """ diff --git a/tools/rpau/confs/Lib/textwrap.toml b/tools/rpau/confs/Lib/textwrap.toml new file mode 100644 index 0000000000..0412974786 --- /dev/null +++ b/tools/rpau/confs/Lib/textwrap.toml @@ -0,0 +1 @@ +path = "Lib/textwrap.py" From 11694e3a4f220f1a91bbc4017eb48b5a2752e605 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 22 Aug 2025 14:55:12 +0300 Subject: [PATCH 09/29] ruff fmt --- tools/rpau/src/rpau/logic.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/rpau/src/rpau/logic.py b/tools/rpau/src/rpau/logic.py index f0f09fc495..0b607ff653 100644 --- a/tools/rpau/src/rpau/logic.py +++ b/tools/rpau/src/rpau/logic.py @@ -17,6 +17,7 @@ logger = get_logger(__name__) + def fetch_upstream(*, base_url: str, path: str, version: str) -> str: upstream_url = "/".join((base_url, version, path)) logger.debug(f"{upstream_url=}") @@ -44,6 +45,7 @@ def get_upstream_contents( else: return fetch() + def format_patch(patch_conf: Patch) -> str: """ Transforms a patch definition to a raw python code. @@ -80,6 +82,7 @@ def format_patch(patch_conf: Patch) -> str: return res.strip().rstrip(";").strip() + def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": """ Get needed patches to apply for given ast tree based on the conf. @@ -117,14 +120,14 @@ def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": if not (patch_conf := cls_conf.pop(fn_node.name, None)): continue - ''' + """ if any( is_patch_present(dec_node, patch_conf) for dec_node in fn_node.decorator_list if isinstance(dec_node, (ast.Attribute, ast.Call)) ): continue - ''' + """ lineno = min( (dec_node.lineno for dec_node in fn_node.decorator_list), @@ -158,6 +161,7 @@ def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": """.rstrip(), ) + def apply_conf(contents: str, conf: dict) -> str: """ Patch a given source code based on the conf. @@ -184,6 +188,7 @@ def apply_conf(contents: str, conf: dict) -> str: return "\n".join(lines) + def run( conf: dict, path: str, @@ -196,7 +201,7 @@ def run( path=path, version=version, base_url=base_upstream_url, cache_dir=cache_dir ) - patched_contents=apply_conf(contents, conf) + patched_contents = apply_conf(contents, conf) new_contents = f"# upstream_version: {version}\n{patched_contents}" output_file = output_dir / path From d8b4e26974ab140d981dc07d4cad77c8b97fe79d Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 22 Aug 2025 15:08:32 +0300 Subject: [PATCH 10/29] Add more modules --- Lib/_colorize.py | 1 + Lib/_ios_support.py | 1 + Lib/_osx_support.py | 1 + Lib/_py_abc.py | 1 + Lib/_pydecimal.py | 3 ++- Lib/calendar.py | 1 + Lib/cmd.py | 1 + Lib/colorsys.py | 1 + Lib/csv.py | 1 + Lib/difflib.py | 5 +++-- Lib/fileinput.py | 1 + Lib/keyword.py | 1 + Lib/linecache.py | 1 + Lib/netrc.py | 1 + Lib/ntpath.py | 1 + Lib/operator.py | 1 + Lib/posixpath.py | 1 + tools/rpau/confs/Lib/_colorize.toml | 1 + tools/rpau/confs/Lib/_ios_support.toml | 1 + tools/rpau/confs/Lib/_osx_support.toml | 1 + tools/rpau/confs/Lib/_py_abc.toml | 1 + tools/rpau/confs/Lib/_pydecimal.toml | 1 + tools/rpau/confs/Lib/calendar.toml | 1 + tools/rpau/confs/Lib/cmd.toml | 1 + tools/rpau/confs/Lib/colorsys.toml | 1 + tools/rpau/confs/Lib/csv.toml | 1 + tools/rpau/confs/Lib/difflib.toml | 1 + tools/rpau/confs/Lib/fileinput.toml | 1 + tools/rpau/confs/Lib/keyword.toml | 1 + tools/rpau/confs/Lib/linecache.toml | 1 + tools/rpau/confs/Lib/netrc.toml | 1 + tools/rpau/confs/Lib/ntpath.toml | 1 + tools/rpau/confs/Lib/operator.toml | 1 + tools/rpau/confs/Lib/posixpath.toml | 1 + 34 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 tools/rpau/confs/Lib/_colorize.toml create mode 100644 tools/rpau/confs/Lib/_ios_support.toml create mode 100644 tools/rpau/confs/Lib/_osx_support.toml create mode 100644 tools/rpau/confs/Lib/_py_abc.toml create mode 100644 tools/rpau/confs/Lib/_pydecimal.toml create mode 100644 tools/rpau/confs/Lib/calendar.toml create mode 100644 tools/rpau/confs/Lib/cmd.toml create mode 100644 tools/rpau/confs/Lib/colorsys.toml create mode 100644 tools/rpau/confs/Lib/csv.toml create mode 100644 tools/rpau/confs/Lib/difflib.toml create mode 100644 tools/rpau/confs/Lib/fileinput.toml create mode 100644 tools/rpau/confs/Lib/keyword.toml create mode 100644 tools/rpau/confs/Lib/linecache.toml create mode 100644 tools/rpau/confs/Lib/netrc.toml create mode 100644 tools/rpau/confs/Lib/ntpath.toml create mode 100644 tools/rpau/confs/Lib/operator.toml create mode 100644 tools/rpau/confs/Lib/posixpath.toml diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 9eb6f0933b..b7d31c4ecc 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 from __future__ import annotations import io import os diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py index 20467a7c2b..8b8238cf4c 100644 --- a/Lib/_ios_support.py +++ b/Lib/_ios_support.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 import sys try: from ctypes import cdll, c_void_p, c_char_p, util diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 0cb064fcd7..961318359d 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Shared OS X support functions.""" import os diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c870ae9048..9cff1d05ab 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 from _weakrefset import WeakSet diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ff80180a79..3851ba4438 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Copyright (c) 2004 Python Software Foundation. # All rights reserved. @@ -6083,7 +6084,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P\#)? (?P0)? (?P(?!0)\d+)? -(?P,)? +(?P[,_])? (?:\.(?P0|(?!0)\d+))? (?P[eEfFgGn%])? \Z diff --git a/Lib/calendar.py b/Lib/calendar.py index 8c1c646da4..a3f0d8a22b 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Calendar printing functions Note when comparing these calendars to the ones printed by cal(1): By diff --git a/Lib/cmd.py b/Lib/cmd.py index a37d16cd7b..70893d246e 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """A generic class to build line-oriented command interpreters. Interpreters constructed with this class obey the following conventions: diff --git a/Lib/colorsys.py b/Lib/colorsys.py index e97f91718a..1c592a4726 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Conversion functions between RGB and other color systems. This modules provides two functions for each color system ABC: diff --git a/Lib/csv.py b/Lib/csv.py index cd20265987..db6b6c3b46 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 r""" CSV parsing and writing. diff --git a/Lib/difflib.py b/Lib/difflib.py index 33e7e6c165..38f51f9778 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """ Module difflib -- helpers for computing deltas between objects. @@ -78,8 +79,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 3)) - 0.866 + >>> print(round(s.ratio(), 2)) + 0.87 >>> If you're only interested in where the sequences match, diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 3dba3d2fbf..724eb8b888 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Helper class to quickly write a loop over all standard input files. Typical use is: diff --git a/Lib/keyword.py b/Lib/keyword.py index e22c837835..6f308386b4 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Keywords (from "Grammar/python.gram") This file is automatically generated; please don't muck it up! diff --git a/Lib/linecache.py b/Lib/linecache.py index dc02de19eb..b0b1742582 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Cache lines from Python source files. This is intended to read lines from modules imported -- hence if a filename diff --git a/Lib/netrc.py b/Lib/netrc.py index bd003e80a4..1cca5d1c89 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """An object-oriented interface to .netrc files.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 9cdc16480f..55b51f53cd 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 # Module 'ntpath' -- common operations on WinNT/Win95 pathnames """Common pathname manipulations, WindowsNT/95 version. diff --git a/Lib/operator.py b/Lib/operator.py index 02ccdaa13d..553fe6789d 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """ Operator Interface diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 80561ae7e5..cf8a8c4494 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -1,3 +1,4 @@ +# upstream_version: v3.13.7 """Common operations on Posix pathnames. Instead of importing this module directly, import os and refer to diff --git a/tools/rpau/confs/Lib/_colorize.toml b/tools/rpau/confs/Lib/_colorize.toml new file mode 100644 index 0000000000..bb390f3db9 --- /dev/null +++ b/tools/rpau/confs/Lib/_colorize.toml @@ -0,0 +1 @@ +path = "Lib/_colorize.py" diff --git a/tools/rpau/confs/Lib/_ios_support.toml b/tools/rpau/confs/Lib/_ios_support.toml new file mode 100644 index 0000000000..8107cc2c98 --- /dev/null +++ b/tools/rpau/confs/Lib/_ios_support.toml @@ -0,0 +1 @@ +path = "Lib/_ios_support.py" diff --git a/tools/rpau/confs/Lib/_osx_support.toml b/tools/rpau/confs/Lib/_osx_support.toml new file mode 100644 index 0000000000..4d1606d4f7 --- /dev/null +++ b/tools/rpau/confs/Lib/_osx_support.toml @@ -0,0 +1 @@ +path = "Lib/_osx_support.py" diff --git a/tools/rpau/confs/Lib/_py_abc.toml b/tools/rpau/confs/Lib/_py_abc.toml new file mode 100644 index 0000000000..f2bd748ead --- /dev/null +++ b/tools/rpau/confs/Lib/_py_abc.toml @@ -0,0 +1 @@ +path = "Lib/_py_abc.py" diff --git a/tools/rpau/confs/Lib/_pydecimal.toml b/tools/rpau/confs/Lib/_pydecimal.toml new file mode 100644 index 0000000000..1f7b276c32 --- /dev/null +++ b/tools/rpau/confs/Lib/_pydecimal.toml @@ -0,0 +1 @@ +path = "Lib/_pydecimal.py" diff --git a/tools/rpau/confs/Lib/calendar.toml b/tools/rpau/confs/Lib/calendar.toml new file mode 100644 index 0000000000..91c07b15ff --- /dev/null +++ b/tools/rpau/confs/Lib/calendar.toml @@ -0,0 +1 @@ +path = "Lib/calendar.py" diff --git a/tools/rpau/confs/Lib/cmd.toml b/tools/rpau/confs/Lib/cmd.toml new file mode 100644 index 0000000000..884ec8cf18 --- /dev/null +++ b/tools/rpau/confs/Lib/cmd.toml @@ -0,0 +1 @@ +path = "Lib/cmd.py" diff --git a/tools/rpau/confs/Lib/colorsys.toml b/tools/rpau/confs/Lib/colorsys.toml new file mode 100644 index 0000000000..37ada8492d --- /dev/null +++ b/tools/rpau/confs/Lib/colorsys.toml @@ -0,0 +1 @@ +path = "Lib/colorsys.py" diff --git a/tools/rpau/confs/Lib/csv.toml b/tools/rpau/confs/Lib/csv.toml new file mode 100644 index 0000000000..667d2a6170 --- /dev/null +++ b/tools/rpau/confs/Lib/csv.toml @@ -0,0 +1 @@ +path = "Lib/csv.py" diff --git a/tools/rpau/confs/Lib/difflib.toml b/tools/rpau/confs/Lib/difflib.toml new file mode 100644 index 0000000000..e5d5876b26 --- /dev/null +++ b/tools/rpau/confs/Lib/difflib.toml @@ -0,0 +1 @@ +path = "Lib/difflib.py" diff --git a/tools/rpau/confs/Lib/fileinput.toml b/tools/rpau/confs/Lib/fileinput.toml new file mode 100644 index 0000000000..474eaf7dba --- /dev/null +++ b/tools/rpau/confs/Lib/fileinput.toml @@ -0,0 +1 @@ +path = "Lib/fileinput.py" diff --git a/tools/rpau/confs/Lib/keyword.toml b/tools/rpau/confs/Lib/keyword.toml new file mode 100644 index 0000000000..71f7e66e77 --- /dev/null +++ b/tools/rpau/confs/Lib/keyword.toml @@ -0,0 +1 @@ +path = "Lib/keyword.py" diff --git a/tools/rpau/confs/Lib/linecache.toml b/tools/rpau/confs/Lib/linecache.toml new file mode 100644 index 0000000000..d09be7bb08 --- /dev/null +++ b/tools/rpau/confs/Lib/linecache.toml @@ -0,0 +1 @@ +path = "Lib/linecache.py" diff --git a/tools/rpau/confs/Lib/netrc.toml b/tools/rpau/confs/Lib/netrc.toml new file mode 100644 index 0000000000..5f8fe27089 --- /dev/null +++ b/tools/rpau/confs/Lib/netrc.toml @@ -0,0 +1 @@ +path = "Lib/netrc.py" diff --git a/tools/rpau/confs/Lib/ntpath.toml b/tools/rpau/confs/Lib/ntpath.toml new file mode 100644 index 0000000000..2a92f5c46f --- /dev/null +++ b/tools/rpau/confs/Lib/ntpath.toml @@ -0,0 +1 @@ +path = "Lib/ntpath.py" diff --git a/tools/rpau/confs/Lib/operator.toml b/tools/rpau/confs/Lib/operator.toml new file mode 100644 index 0000000000..7714820f84 --- /dev/null +++ b/tools/rpau/confs/Lib/operator.toml @@ -0,0 +1 @@ +path = "Lib/operator.py" diff --git a/tools/rpau/confs/Lib/posixpath.toml b/tools/rpau/confs/Lib/posixpath.toml new file mode 100644 index 0000000000..f544f51892 --- /dev/null +++ b/tools/rpau/confs/Lib/posixpath.toml @@ -0,0 +1 @@ +path = "Lib/posixpath.py" From 2fb6842f877840a278d85a8d5b77d6e84d656e19 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:42:47 +0300 Subject: [PATCH 11/29] Add `gen` subcommand --- scripts/lib_updater.py | 199 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 scripts/lib_updater.py diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py new file mode 100755 index 0000000000..afdf73a06a --- /dev/null +++ b/scripts/lib_updater.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python +import argparse +import ast +import dataclasses +import enum +import json +import re +import sys +from typing import TYPE_CHECKING, Self + +if TYPE_CHECKING: + from collections.abc import Iterator + +COMMENT = "TODO: RUSTPYTHON" + + +@enum.unique +class ProgName(enum.StrEnum): + Gen = enum.auto() + Patch = enum.auto() + + +@enum.unique +class UtMethod(enum.StrEnum): + """ + UnitTest Method + """ + + ExpectedFailure = "expectedFailure" + ExpectedFailureIf = "expectedFailureIf" + ExpectedFailureIfWindows = "expectedFailureIfWindows" + SkipUnless = "skipUnless" + Skip = enum.auto() + SkipIf = "skipIf" + + +@dataclasses.dataclass(frozen=True, slots=True) +class PatchEntry: + """ + Stores patch metadata. + + Attributes + ---------- + parent_class : str + Parent class of test. + test_name : str + Test name. + ut_method : UtMethod + unittest method. + cond : str, optional + `ut_method` condition. Relevant only for UtMethod.{expectedFailureIf,skipIf}. + reason : str, optional + Reason for why the test is patched in this way. + """ + + parent_class: str + test_name: str + ut_method: UtMethod + cond: str | None = None + reason: str = "" + + @classmethod + def iter_patch_entires(cls, tree: ast.Module, lines: list[str]) -> "Iterator[Self]": + for cls_node, fn_node in iter_tests(tree): + parent_class = cls_node.name + for dec_node in fn_node.decorator_list: + if not isinstance(dec_node, (ast.Attribute, ast.Call)): + continue + + attr_node = ( + dec_node if isinstance(dec_node, ast.Attribute) else dec_node.func + ) + + if isinstance(attr_node, ast.Name) or attr_node.value.id != "unittest": + continue + + cond = None + match attr_node.attr: + case UtMethod.ExpectedFailure: + for line in lines[dec_node.lineno - 2 : dec_node.lineno]: + if COMMENT not in line: + continue + reason = "".join(re.findall(rf"{COMMENT} (.*)", line)) + break + else: + continue + case ( + UtMethod.Skip + | UtMethod.SkipIf + | UtMethod.ExpectedFailureIf + | UtMethod.ExpectedFailureIfWindows + ): + reason = next( + ( + node.value + for node in ast.walk(dec_node) + if isinstance(node, ast.Constant) + and isinstance(node.value, str) + and node.value.startswith(COMMENT) + ), + None, + ) + + # If we didn't find a constant with the COMMENT, then we didn't put this decorator + if not reason: + continue + + if attr_node.attr not in ( + UtMethod.Skip, + UtMethod.ExpectedFailureIfWindows, + ): + cond = ast.unparse(dec_node.args[0]) + case _: + continue + + yield cls( + parent_class, + fn_node.name, + UtMethod(attr_node.attr), + cond, + reason.replace(COMMENT, "").strip().lstrip(";").lstrip(":").strip(), + ) + + +def iter_tests( + tree: ast.Module, +) -> "Iterator[tuple[ast.ClassDef, ast.FunctionDef | ast.AsyncFunctionDef]]": + for key, nodes in ast.iter_fields(tree): + if key != "body": + continue + + for cls_node in nodes: + if not isinstance(cls_node, ast.ClassDef): + continue + + for fn_node in cls_node.body: + if not isinstance(fn_node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + + yield (cls_node, fn_node) + + +def iter_patches(contents: str) -> "Iterator[PatchEntry]": + lines = contents.splitlines() + tree = ast.parse(contents) + yield from PatchEntry.iter_patch_entires(tree, lines) + + +def read_infile(infile: str) -> str: + if infile == "-": + return sys.stdin.read() + + with open(infile, mode="r", encoding="utf-8") as fd: + return fd.read() + + +def build_argparse() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Helper tool for updating files under Lib/" + ) + + subparsers = parser.add_subparsers(dest="pname", required=True) + + # Gen + parser_gen = subparsers.add_parser(ProgName.Gen) + parser_gen.add_argument( + "infile", + default="-", + help="File path to generate patches from, can get from stdin", + nargs="?", + ) + + # Patch + parser_patch = subparsers.add_parser(ProgName.Patch) + parser_patch.add_argument("src", help="File path to apply patches for") + parser_patch.add_argument( + "infile", + default="-", + help="File path containing patches, can get from stdin", + nargs="?", + ) + + return parser + + +if __name__ == "__main__": + parser = build_argparse() + args = parser.parse_args() + + contents = read_infile(args.infile) + match args.pname: + case ProgName.Gen: + patches = list(map(dataclasses.asdict, iter_patches(contents))) + output = json.dumps(patches, indent=4) + case ProgName.Patch: + pass # TODO + + sys.stdout.write(f"{output}\n") + sys.stdout.flush() From db5eb4b308156d0ecba03b540d2283fd09a396f0 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:03:36 +0300 Subject: [PATCH 12/29] Use `_generate_next_value_` --- scripts/lib_updater.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index afdf73a06a..9158aee6bd 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -23,15 +23,18 @@ class ProgName(enum.StrEnum): @enum.unique class UtMethod(enum.StrEnum): """ - UnitTest Method + UnitTest Method. """ - ExpectedFailure = "expectedFailure" - ExpectedFailureIf = "expectedFailureIf" - ExpectedFailureIfWindows = "expectedFailureIfWindows" - SkipUnless = "skipUnless" + def _generate_next_value_(name, start, count, last_values) -> str: + return name[0].lower() + name[1:] + + ExpectedFailure = enum.auto() + ExpectedFailureIf = enum.auto() + ExpectedFailureIfWindows = enum.auto() Skip = enum.auto() - SkipIf = "skipIf" + SkipIf = enum.auto() + SkipUnless = enum.auto() @dataclasses.dataclass(frozen=True, slots=True) From af1c28d09a3de654ef6097d908518d0bdad1227e Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:47:26 +0300 Subject: [PATCH 13/29] Gen & patch --- scripts/lib_updater.py | 240 ++++++++++++++++++++++++++++------------- 1 file changed, 166 insertions(+), 74 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 9158aee6bd..4b753f08fb 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -1,23 +1,23 @@ #!/usr/bin/env python import argparse import ast -import dataclasses +import collections import enum import json +import pathlib import re import sys -from typing import TYPE_CHECKING, Self +import typing -if TYPE_CHECKING: +if typing.TYPE_CHECKING: from collections.abc import Iterator -COMMENT = "TODO: RUSTPYTHON" - +type Patches = dict[str, dict[str, list["PatchSpec"]]] -@enum.unique -class ProgName(enum.StrEnum): - Gen = enum.auto() - Patch = enum.auto() +COL_OFFSET = 4 +INDENT1 = " " * COL_OFFSET +INDENT2 = INDENT1 * 2 +COMMENT = "TODO: RUSTPYTHON" @enum.unique @@ -29,6 +29,9 @@ class UtMethod(enum.StrEnum): def _generate_next_value_(name, start, count, last_values) -> str: return name[0].lower() + name[1:] + def has_cond(self) -> bool: + return self.endswith(("If", "Unless")) + ExpectedFailure = enum.auto() ExpectedFailureIf = enum.auto() ExpectedFailureIfWindows = enum.auto() @@ -37,33 +40,57 @@ def _generate_next_value_(name, start, count, last_values) -> str: SkipUnless = enum.auto() -@dataclasses.dataclass(frozen=True, slots=True) -class PatchEntry: +class PatchSpec(typing.NamedTuple): """ - Stores patch metadata. - Attributes ---------- - parent_class : str - Parent class of test. - test_name : str - Test name. ut_method : UtMethod unittest method. cond : str, optional - `ut_method` condition. Relevant only for UtMethod.{expectedFailureIf,skipIf}. + `ut_method` condition. Relevant only for some of `ut_method` types. reason : str, optional Reason for why the test is patched in this way. """ - parent_class: str - test_name: str ut_method: UtMethod cond: str | None = None reason: str = "" + def fmt(self) -> str: + prefix = prefix = f"@unittest.{self.ut_method}" + match self.ut_method: + case UtMethod.ExpectedFailure: + line = f"{prefix} # {COMMENT}; {self.reason}" + case UtMethod.ExpectedFailureIfWindows | UtMethod.Skip: + line = f'{prefix}("{COMMENT}; {self.reason}")' + case UtMethod.SkipIf | UtMethod.SkipUnless | UtMethod.ExpectedFailureIf: + line = f'{prefix}({self.cond}, "{COMMENT}; {self.reason}")' + + return line.strip().rstrip(";").strip() + + +class PatchEntry(typing.NamedTuple): + """ + Stores patch metadata. + + Attributes + ---------- + parent_class : str + Parent class of test. + test_name : str + Test name. + spec : PatchSpec + Patch spec. + """ + + parent_class: str + test_name: str + spec: PatchSpec + @classmethod - def iter_patch_entires(cls, tree: ast.Module, lines: list[str]) -> "Iterator[Self]": + def iter_patch_entires( + cls, tree: ast.Module, lines: list[str] + ) -> "Iterator[typing.Self]": for cls_node, fn_node in iter_tests(tree): parent_class = cls_node.name for dec_node in fn_node.decorator_list: @@ -78,7 +105,12 @@ def iter_patch_entires(cls, tree: ast.Module, lines: list[str]) -> "Iterator[Sel continue cond = None - match attr_node.attr: + try: + ut_method = UtMethod(attr_node.attr) + except ValueError: + continue + + match ut_method: case UtMethod.ExpectedFailure: for line in lines[dec_node.lineno - 2 : dec_node.lineno]: if COMMENT not in line: @@ -87,42 +119,31 @@ def iter_patch_entires(cls, tree: ast.Module, lines: list[str]) -> "Iterator[Sel break else: continue - case ( - UtMethod.Skip - | UtMethod.SkipIf - | UtMethod.ExpectedFailureIf - | UtMethod.ExpectedFailureIfWindows - ): + case _: reason = next( ( node.value for node in ast.walk(dec_node) if isinstance(node, ast.Constant) and isinstance(node.value, str) - and node.value.startswith(COMMENT) + and COMMENT in node.value ), None, ) - # If we didn't find a constant with the COMMENT, then we didn't put this decorator + # If we didn't find a constant containing , + # then we didn't put this decorator if not reason: continue - if attr_node.attr not in ( - UtMethod.Skip, - UtMethod.ExpectedFailureIfWindows, - ): + if ut_method.has_cond(): cond = ast.unparse(dec_node.args[0]) - case _: - continue - - yield cls( - parent_class, - fn_node.name, - UtMethod(attr_node.attr), - cond, - reason.replace(COMMENT, "").strip().lstrip(";").lstrip(":").strip(), + + reason = ( + reason.replace(COMMENT, "").strip().lstrip(";").lstrip(":").strip() ) + spec = PatchSpec(ut_method, cond, reason) + yield cls(parent_class, fn_node.name, spec) def iter_tests( @@ -149,12 +170,57 @@ def iter_patches(contents: str) -> "Iterator[PatchEntry]": yield from PatchEntry.iter_patch_entires(tree, lines) -def read_infile(infile: str) -> str: - if infile == "-": - return sys.stdin.read() +def build_patch_dict(it: "Iterator[PatchEntry]") -> Patches: + patches = collections.defaultdict(lambda: collections.defaultdict(list)) + for entry in it: + patches[entry.parent_class][entry.test_name].append(entry.spec) - with open(infile, mode="r", encoding="utf-8") as fd: - return fd.read() + return {k: dict(v) for k, v in patches.items()} + + +def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, str]]": + cache = {} # Used in phase 2 + + # Phase 1: Iterate and mark existing tests + for cls_node, fn_node in iter_tests(tree): + cache[cls_node.name] = cls_node.end_lineno + specs = patches.get(cls_node.name, {}).pop(fn_node.name, None) + if not specs: + continue + + lineno = min( + (dec_node.lineno for dec_node in fn_node.decorator_list), + default=fn_node.lineno, + ) + indent = " " * fn_node.col_offset + yield (lineno - 1, "\n".join(f"{indent}{spec.fmt()}" for spec in specs)) + + # Phase 2: Iterate and mark inhereted tests + for cls_name, tests in patches.items(): + lineno = cache[cls_name] + for test_name, specs in tests.items(): + patch_lines = "\n".join(f"{INDENT1}{spec.fmt()}" for spec in specs) + yield ( + lineno, + f""" +{patch_lines} +{INDENT1}def {test_name}(self): +{INDENT2}return super().{test_name}() +""".rstrip(), + ) + + +def apply_patches(contents: str, patches: Patches) -> str: + tree = ast.parse(contents) + lines = contents.splitlines() + + modifications = list(iter_patch_lines(tree, patches)) + # Going in reverse to not distrupt the line offset + for lineno, patch in sorted(modifications, reverse=True): + lines.insert(lineno, patch) + + joined = "\n".join(lines) + return f"{joined}\n" def build_argparse() -> argparse.ArgumentParser: @@ -162,25 +228,31 @@ def build_argparse() -> argparse.ArgumentParser: description="Helper tool for updating files under Lib/" ) - subparsers = parser.add_subparsers(dest="pname", required=True) - - # Gen - parser_gen = subparsers.add_parser(ProgName.Gen) - parser_gen.add_argument( - "infile", - default="-", - help="File path to generate patches from, can get from stdin", - nargs="?", + parser.add_argument( + "orig_file", help="File to gather patches from", type=pathlib.Path ) - # Patch - parser_patch = subparsers.add_parser(ProgName.Patch) - parser_patch.add_argument("src", help="File path to apply patches for") - parser_patch.add_argument( - "infile", - default="-", - help="File path containing patches, can get from stdin", + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "remote_file", nargs="?", + help="File to apply patches to", + type=pathlib.Path, + ) + group.add_argument( + "--show-patches", action="store_true", help="Show the patches and exit" + ) + parser.add_argument( + "-p", + "--patches", + help="File path to file containing patches in a JSON format", + type=pathlib.Path, + ) + parser.add_argument( + "--inplace", + action=argparse.BooleanOptionalAction, + default=False, + help="Whether write the changes", ) return parser @@ -190,13 +262,33 @@ def build_argparse() -> argparse.ArgumentParser: parser = build_argparse() args = parser.parse_args() - contents = read_infile(args.infile) - match args.pname: - case ProgName.Gen: - patches = list(map(dataclasses.asdict, iter_patches(contents))) - output = json.dumps(patches, indent=4) - case ProgName.Patch: - pass # TODO - - sys.stdout.write(f"{output}\n") - sys.stdout.flush() + contents = args.orig_file.read_text() + if args.patches: + patches = { + cls_name: { + test_name: [PatchSpec(**spec) for spec in specs] + for test_name, specs in tests.items() + } + for cls_name, tests in json.loads(args.patches.read_text()).items() + } + else: + patches = build_patch_dict(iter_patches(contents)) + + if args.show_patches: + patches = { + cls_name: { + test_name: [spec._asdict() for spec in specs] + for test_name, specs in tests.items() + } + for cls_name, tests in patches.items() + } + output = json.dumps(patches, indent=4) + sys.stdout.write(f"{output}\n") + sys.exit(0) + + patched = apply_patches(args.remote_file.read_text(), patches) + + if args.inplace: + args.orig_file.write_text(patched) + else: + sys.stdout.write(patched) From 42365d244d8da570eaf85a3feece9d29ae717754 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:48:18 +0300 Subject: [PATCH 14/29] Remove old tool --- tools/rpau/README.md | 1 - tools/rpau/confs/Lib/_colorize.toml | 1 - tools/rpau/confs/Lib/_ios_support.toml | 1 - tools/rpau/confs/Lib/_osx_support.toml | 1 - tools/rpau/confs/Lib/_py_abc.toml | 1 - tools/rpau/confs/Lib/_pydecimal.toml | 1 - tools/rpau/confs/Lib/argparse.toml | 1 - tools/rpau/confs/Lib/ast.toml | 1 - tools/rpau/confs/Lib/calendar.toml | 1 - tools/rpau/confs/Lib/cmd.toml | 1 - tools/rpau/confs/Lib/colorsys.toml | 1 - tools/rpau/confs/Lib/csv.toml | 1 - tools/rpau/confs/Lib/decimal.toml | 1 - tools/rpau/confs/Lib/difflib.toml | 1 - tools/rpau/confs/Lib/fileinput.toml | 1 - tools/rpau/confs/Lib/heapq.toml | 1 - tools/rpau/confs/Lib/ipaddress.toml | 1 - tools/rpau/confs/Lib/keyword.toml | 1 - tools/rpau/confs/Lib/linecache.toml | 1 - tools/rpau/confs/Lib/netrc.toml | 1 - tools/rpau/confs/Lib/ntpath.toml | 1 - tools/rpau/confs/Lib/numbers.toml | 1 - tools/rpau/confs/Lib/operator.toml | 1 - tools/rpau/confs/Lib/os.toml | 1 - tools/rpau/confs/Lib/posixpath.toml | 1 - tools/rpau/confs/Lib/pprint.toml | 1 - tools/rpau/confs/Lib/queue.toml | 1 - tools/rpau/confs/Lib/test/test_os.toml | 103 ------------ tools/rpau/confs/Lib/textwrap.toml | 1 - tools/rpau/confs/Lib/this.toml | 1 - tools/rpau/pyproject.toml | 17 -- tools/rpau/src/rpau/__init__.py | 75 --------- tools/rpau/src/rpau/cli.py | 134 ---------------- tools/rpau/src/rpau/logger.py | 25 --- tools/rpau/src/rpau/logic.py | 210 ------------------------- tools/rpau/uv.lock | 8 - 36 files changed, 601 deletions(-) delete mode 100644 tools/rpau/README.md delete mode 100644 tools/rpau/confs/Lib/_colorize.toml delete mode 100644 tools/rpau/confs/Lib/_ios_support.toml delete mode 100644 tools/rpau/confs/Lib/_osx_support.toml delete mode 100644 tools/rpau/confs/Lib/_py_abc.toml delete mode 100644 tools/rpau/confs/Lib/_pydecimal.toml delete mode 100644 tools/rpau/confs/Lib/argparse.toml delete mode 100644 tools/rpau/confs/Lib/ast.toml delete mode 100644 tools/rpau/confs/Lib/calendar.toml delete mode 100644 tools/rpau/confs/Lib/cmd.toml delete mode 100644 tools/rpau/confs/Lib/colorsys.toml delete mode 100644 tools/rpau/confs/Lib/csv.toml delete mode 100644 tools/rpau/confs/Lib/decimal.toml delete mode 100644 tools/rpau/confs/Lib/difflib.toml delete mode 100644 tools/rpau/confs/Lib/fileinput.toml delete mode 100644 tools/rpau/confs/Lib/heapq.toml delete mode 100644 tools/rpau/confs/Lib/ipaddress.toml delete mode 100644 tools/rpau/confs/Lib/keyword.toml delete mode 100644 tools/rpau/confs/Lib/linecache.toml delete mode 100644 tools/rpau/confs/Lib/netrc.toml delete mode 100644 tools/rpau/confs/Lib/ntpath.toml delete mode 100644 tools/rpau/confs/Lib/numbers.toml delete mode 100644 tools/rpau/confs/Lib/operator.toml delete mode 100644 tools/rpau/confs/Lib/os.toml delete mode 100644 tools/rpau/confs/Lib/posixpath.toml delete mode 100644 tools/rpau/confs/Lib/pprint.toml delete mode 100644 tools/rpau/confs/Lib/queue.toml delete mode 100644 tools/rpau/confs/Lib/test/test_os.toml delete mode 100644 tools/rpau/confs/Lib/textwrap.toml delete mode 100644 tools/rpau/confs/Lib/this.toml delete mode 100644 tools/rpau/pyproject.toml delete mode 100644 tools/rpau/src/rpau/__init__.py delete mode 100644 tools/rpau/src/rpau/cli.py delete mode 100644 tools/rpau/src/rpau/logger.py delete mode 100644 tools/rpau/src/rpau/logic.py delete mode 100644 tools/rpau/uv.lock diff --git a/tools/rpau/README.md b/tools/rpau/README.md deleted file mode 100644 index f4d6f3ff72..0000000000 --- a/tools/rpau/README.md +++ /dev/null @@ -1 +0,0 @@ -# **R**ust**P**ython **A**utomatic **U**pdater (rpau) diff --git a/tools/rpau/confs/Lib/_colorize.toml b/tools/rpau/confs/Lib/_colorize.toml deleted file mode 100644 index bb390f3db9..0000000000 --- a/tools/rpau/confs/Lib/_colorize.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/_colorize.py" diff --git a/tools/rpau/confs/Lib/_ios_support.toml b/tools/rpau/confs/Lib/_ios_support.toml deleted file mode 100644 index 8107cc2c98..0000000000 --- a/tools/rpau/confs/Lib/_ios_support.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/_ios_support.py" diff --git a/tools/rpau/confs/Lib/_osx_support.toml b/tools/rpau/confs/Lib/_osx_support.toml deleted file mode 100644 index 4d1606d4f7..0000000000 --- a/tools/rpau/confs/Lib/_osx_support.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/_osx_support.py" diff --git a/tools/rpau/confs/Lib/_py_abc.toml b/tools/rpau/confs/Lib/_py_abc.toml deleted file mode 100644 index f2bd748ead..0000000000 --- a/tools/rpau/confs/Lib/_py_abc.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/_py_abc.py" diff --git a/tools/rpau/confs/Lib/_pydecimal.toml b/tools/rpau/confs/Lib/_pydecimal.toml deleted file mode 100644 index 1f7b276c32..0000000000 --- a/tools/rpau/confs/Lib/_pydecimal.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/_pydecimal.py" diff --git a/tools/rpau/confs/Lib/argparse.toml b/tools/rpau/confs/Lib/argparse.toml deleted file mode 100644 index 440838d9b4..0000000000 --- a/tools/rpau/confs/Lib/argparse.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/argparse.py" diff --git a/tools/rpau/confs/Lib/ast.toml b/tools/rpau/confs/Lib/ast.toml deleted file mode 100644 index 220af23775..0000000000 --- a/tools/rpau/confs/Lib/ast.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/ast.py" diff --git a/tools/rpau/confs/Lib/calendar.toml b/tools/rpau/confs/Lib/calendar.toml deleted file mode 100644 index 91c07b15ff..0000000000 --- a/tools/rpau/confs/Lib/calendar.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/calendar.py" diff --git a/tools/rpau/confs/Lib/cmd.toml b/tools/rpau/confs/Lib/cmd.toml deleted file mode 100644 index 884ec8cf18..0000000000 --- a/tools/rpau/confs/Lib/cmd.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/cmd.py" diff --git a/tools/rpau/confs/Lib/colorsys.toml b/tools/rpau/confs/Lib/colorsys.toml deleted file mode 100644 index 37ada8492d..0000000000 --- a/tools/rpau/confs/Lib/colorsys.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/colorsys.py" diff --git a/tools/rpau/confs/Lib/csv.toml b/tools/rpau/confs/Lib/csv.toml deleted file mode 100644 index 667d2a6170..0000000000 --- a/tools/rpau/confs/Lib/csv.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/csv.py" diff --git a/tools/rpau/confs/Lib/decimal.toml b/tools/rpau/confs/Lib/decimal.toml deleted file mode 100644 index 8633f1e1d9..0000000000 --- a/tools/rpau/confs/Lib/decimal.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/decimal.py" diff --git a/tools/rpau/confs/Lib/difflib.toml b/tools/rpau/confs/Lib/difflib.toml deleted file mode 100644 index e5d5876b26..0000000000 --- a/tools/rpau/confs/Lib/difflib.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/difflib.py" diff --git a/tools/rpau/confs/Lib/fileinput.toml b/tools/rpau/confs/Lib/fileinput.toml deleted file mode 100644 index 474eaf7dba..0000000000 --- a/tools/rpau/confs/Lib/fileinput.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/fileinput.py" diff --git a/tools/rpau/confs/Lib/heapq.toml b/tools/rpau/confs/Lib/heapq.toml deleted file mode 100644 index 5d9ea04401..0000000000 --- a/tools/rpau/confs/Lib/heapq.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/heapq.py" diff --git a/tools/rpau/confs/Lib/ipaddress.toml b/tools/rpau/confs/Lib/ipaddress.toml deleted file mode 100644 index 520f81df3c..0000000000 --- a/tools/rpau/confs/Lib/ipaddress.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/ipaddress.py" diff --git a/tools/rpau/confs/Lib/keyword.toml b/tools/rpau/confs/Lib/keyword.toml deleted file mode 100644 index 71f7e66e77..0000000000 --- a/tools/rpau/confs/Lib/keyword.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/keyword.py" diff --git a/tools/rpau/confs/Lib/linecache.toml b/tools/rpau/confs/Lib/linecache.toml deleted file mode 100644 index d09be7bb08..0000000000 --- a/tools/rpau/confs/Lib/linecache.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/linecache.py" diff --git a/tools/rpau/confs/Lib/netrc.toml b/tools/rpau/confs/Lib/netrc.toml deleted file mode 100644 index 5f8fe27089..0000000000 --- a/tools/rpau/confs/Lib/netrc.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/netrc.py" diff --git a/tools/rpau/confs/Lib/ntpath.toml b/tools/rpau/confs/Lib/ntpath.toml deleted file mode 100644 index 2a92f5c46f..0000000000 --- a/tools/rpau/confs/Lib/ntpath.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/ntpath.py" diff --git a/tools/rpau/confs/Lib/numbers.toml b/tools/rpau/confs/Lib/numbers.toml deleted file mode 100644 index dd7c043a2e..0000000000 --- a/tools/rpau/confs/Lib/numbers.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/numbers.py" diff --git a/tools/rpau/confs/Lib/operator.toml b/tools/rpau/confs/Lib/operator.toml deleted file mode 100644 index 7714820f84..0000000000 --- a/tools/rpau/confs/Lib/operator.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/operator.py" diff --git a/tools/rpau/confs/Lib/os.toml b/tools/rpau/confs/Lib/os.toml deleted file mode 100644 index d65c63a9a3..0000000000 --- a/tools/rpau/confs/Lib/os.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/os.py" diff --git a/tools/rpau/confs/Lib/posixpath.toml b/tools/rpau/confs/Lib/posixpath.toml deleted file mode 100644 index f544f51892..0000000000 --- a/tools/rpau/confs/Lib/posixpath.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/posixpath.py" diff --git a/tools/rpau/confs/Lib/pprint.toml b/tools/rpau/confs/Lib/pprint.toml deleted file mode 100644 index 93c1edc92a..0000000000 --- a/tools/rpau/confs/Lib/pprint.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/pprint.py" diff --git a/tools/rpau/confs/Lib/queue.toml b/tools/rpau/confs/Lib/queue.toml deleted file mode 100644 index 164fc36c51..0000000000 --- a/tools/rpau/confs/Lib/queue.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/queue.py" diff --git a/tools/rpau/confs/Lib/test/test_os.toml b/tools/rpau/confs/Lib/test/test_os.toml deleted file mode 100644 index 8b39f6b1e8..0000000000 --- a/tools/rpau/confs/Lib/test/test_os.toml +++ /dev/null @@ -1,103 +0,0 @@ -path = "Lib/test/test_os.py" - -[FileTests] -test_closerange = { skipIf = { cond = "sys.platform == 'win32'", reason = "BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')" } } - -[StatAttributeTests] -test_file_attributes = { expectedFailureIfWindows = { reason = "os.stat return value doesnt have st_file_attributes attribute" } } -test_access_denied = { expectedFailureIfWindows = { reason = "os.stat (PermissionError: [Errno 5] Access is denied" } } -test_stat_block_device = { expectedFailureIfWindows = { reason = "os.stat (PermissionError: [Errno 1] Incorrect function" } } - -[UtimeTests] -test_utime = { expectedFailureIfWindows = { reason = "AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)" } } -test_utime_directory = { expectedFailureIfWindows = { reason = "(AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)" } } -test_utime_current = { expectedFailureIfWindows = { reason = "AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005" } } -test_utime_current_old = { expectedFailureIfWindows = { reason = "AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052" } } -test_utime_nonexistent = { expectedFailure = { reason = "" } } -test_large_time = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } -test_utime_invalid_arguments = { expectedFailureIfWindows = { reason = "AssertionError: NotImplementedError not raised" } } - -[EnvironTests] -test_putenv_unsetenv_error = { expectedFailureIfWindows = { reason = "AssertionError: ValueError not raised by putenv" } } - -[BytesWalkTests] -test_compare_to_walk = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } -test_dir_fd = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } -test_yields_correct_dir_fd = { expectedFailure = { reason = "TypeError: Can't mix strings and bytes in path components" } } - -[MakedirTests] -test_exist_ok_existing_directory = { expectedFailureIfWindows = { reason = "os.umask not implemented yet for all platforms" } } -test_exist_ok_s_isgid_directory = { expectedFailureIfWindows = { reason = "os.umask not implemented yet for all platforms" } } - -[URandomFDTests] -test_urandom_fd_closed = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named 'os'" } } -test_urandom_fd_reopened = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named 'os'" } } - -[ExecTests] -test_execvpe_with_bad_arglist = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } -test_execve_invalid_env = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } -test_execve_with_empty_path = { expectedFailureIfWindows = { reason = "os.execve not implemented yet for all platforms" } } - -[TestInvalidFD] -test_fdopen = { expectedFailure = { reason = "" } } -test_fpathconf = { expectedFailure = { reason = "" } } -test_ftruncate = { expectedFailure = { reason = "" } } -test_lseek = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } -test_read = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } -test_write = { expectedFailureIfWindows = { reason = "OSError: [Errno 18] There are no more files" } } -test_inheritable = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } -test_fchdir = { expectedFailure = { reason = "" } } -test_fsync = { expectedFailure = { reason = "" } } - -[Win32KillTests] -test_kill_sigterm = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } -test_kill_int = { expectedFailureIfWindows = { reason = "ModuleNotFoundError: No module named '_ctypes'" } } -test_CTRL_BREAK_EVENT = { expectedFailure = { reason = "" } } - -[Win32JunctionTests] -test_create_junction = { expectedFailureIfWindows = { reason = "AttributeError: module '_winapi' has no attribute 'CreateJunction'" } } -test_unlink_removes_junction = { expectedFailureIfWindows = { reason = "AttributeError: module '_winapi' has no attribute 'CreateJunction'" } } - -[Win32NtTests] -test_stat_unlink_race = { expectedFailureIfWindows = { reason = "os.stat PermissionError: [Errno 5] Access is denied" } } - -[PidTests] -test_waitpid_windows = { expectedFailureIfWindows = { reason = "os.spawnv not implemented yet for all platforms" } } -test_waitstatus_to_exitcode_windows = { expectedFailureIfWindows = { reason = "OverflowError: Python int too large to convert to Rust i32" } } - -[SpawnTests] -test_spawnve_bytes = { expectedFailure = { reason = "fix spawnv bytes" } } - -[OSErrorTests] -test_oserror_filename = { expectedFailure = { reason = "AssertionError: b'@test_22106_tmp\\xe7w\\xf0' is not b'@test_22106_tmp\\xe7w\\xf0' : " } } - -[FDInheritanceTests] -test_get_set_inheritable = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } -test_get_set_inheritable_badf = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } -test_open = { expectedFailureIfWindows = { reason = "os.get_inheritable not implemented yet for all platforms" } } -test_dup = { skipIf = { cond = "sys.platform == 'win32'", reason = "os.dup on windows" } } -test_dup_standard_stream = { skipIf = { cond = "sys.platform == 'win32'", reason = "os.dup on windows" } } -test_dup_nul = { expectedFailureIfWindows = { reason = "os.dup not implemented yet for all platforms" } } - -[PathTConverterTests] -test_path_t_converter = { expectedFailure = { reason = "AssertionError: TypeError not raised" } } - -[TestDirEntry] -test_uninstantiable = { expectedFailure = { reason = "AssertionError: TypeError not raised by DirEntry" } } -test_unpickable = { expectedFailure = { reason = "pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry" } } - -[TestScandir] -test_uninstantiable = { expectedFailure = { reason = "AssertionError: TypeError not raised by ScandirIter" } } -test_attributes = { skipIf = { cond = "sys.platform == 'linux'", reason = "flaky test" } } -test_removed_dir = { expectedFailureIfWindows = { reason = "entry.is_dir() is False" } } -test_removed_file = { expectedFailureIfWindows = { reason = "entry.is_file() is False" } } -test_fd = { expectedFailure = { reason = "" } } -test_empty_path = { expectedFailureIfWindows = { reason = "AssertionError: FileNotFoundError not raised by scandir" } } -test_resource_warning = { expectedFailure = { reason = "" } } - -[TestPEP519] -test_pathlike_subclass_slots = { expectedFailure = { reason = "" } } -test_fspath_set_to_None = { expectedFailure = { reason = "" } } - -[ForkTests] -test_fork_at_finalization = { expectedFailure = { reason = "" } } diff --git a/tools/rpau/confs/Lib/textwrap.toml b/tools/rpau/confs/Lib/textwrap.toml deleted file mode 100644 index 0412974786..0000000000 --- a/tools/rpau/confs/Lib/textwrap.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/textwrap.py" diff --git a/tools/rpau/confs/Lib/this.toml b/tools/rpau/confs/Lib/this.toml deleted file mode 100644 index f976d3ccc8..0000000000 --- a/tools/rpau/confs/Lib/this.toml +++ /dev/null @@ -1 +0,0 @@ -path = "Lib/this.py" diff --git a/tools/rpau/pyproject.toml b/tools/rpau/pyproject.toml deleted file mode 100644 index 181e2d8c9a..0000000000 --- a/tools/rpau/pyproject.toml +++ /dev/null @@ -1,17 +0,0 @@ -[project] -name = "rpau" -version = "0.1.0" -description = "Tool for automaticly update Lib & Tests from CPython" -readme = "README.md" -authors = [ - { name = "RustPython Team", email = "" } -] -requires-python = ">=3.12" -dependencies = [] - -[project.scripts] -rpau = "rpau:main" - -[build-system] -requires = ["uv_build>=0.8.11,<0.9.0"] -build-backend = "uv_build" diff --git a/tools/rpau/src/rpau/__init__.py b/tools/rpau/src/rpau/__init__.py deleted file mode 100644 index bed3078e33..0000000000 --- a/tools/rpau/src/rpau/__init__.py +++ /dev/null @@ -1,75 +0,0 @@ -import functools -from concurrent.futures import ProcessPoolExecutor -from typing import TYPE_CHECKING - -import tomllib - -from rpau.cli import build_argparse -from rpau.logger import build_root_logger, get_logger -from rpau.logic import run - -if TYPE_CHECKING: - import pathlib - import re - from collections.abc import Iterator - - -def iter_confs( - conf_dir: "pathlib.Path", *, include: "re.Pattern", exclude: "re.Pattern" -) -> "Iterator[pathlib.Path]": - for conf_file in conf_dir.rglob("**/*.toml"): - if not conf_file.is_file(): - continue - - uri = conf_file.as_uri().removeprefix("file://") - if not include.match(uri): - continue - - if exclude.match(uri): - continue - - yield conf_file - - -def main() -> None: - parser = build_argparse() - args = parser.parse_args() - - logger = build_root_logger(level=args.log_level) - logger.debug(f"{args=}") - - if args.cache: - logger.debug(f"Ensuring {args.cache_dir} exists") - cache_dir = args.cache_dir - cache_dir.mkdir(parents=True, exist_ok=True) - else: - cache_dir = None - - conf_dir = args.conf_dir - with ProcessPoolExecutor(args.workers) as executor: - for conf_file in iter_confs( - conf_dir, include=args.include, exclude=args.exclude - ): - try: - conf = tomllib.loads(conf_file.read_text()) - except tomllib.TOMLDecodeError as err: - logger.warn(f"{conf_file}: {err}") - continue - - try: - path = conf.pop("path") - except KeyError: - logger.warn(f"{conf_file}: has no 'path' key. skipping") - continue - - version = conf.pop("version", args.default_version) - func = functools.partial( - run, - path=path, - conf=conf, - cache_dir=cache_dir, - version=version, - output_dir=args.output_dir, - base_upstream_url=args.base_upstream_url, - ) - executor.submit(func) diff --git a/tools/rpau/src/rpau/cli.py b/tools/rpau/src/rpau/cli.py deleted file mode 100644 index 39da33711f..0000000000 --- a/tools/rpau/src/rpau/cli.py +++ /dev/null @@ -1,134 +0,0 @@ -import argparse -import logging -import os -import pathlib -import re -import sys - - -class CustomArgumentParser(argparse.ArgumentParser): - class _CustomHelpFormatter(argparse.ArgumentDefaultsHelpFormatter): - def _get_help_string(self, action): - help_msg = super()._get_help_string(action) - if action.dest != "help": - env_name = f"{self._prog}_{action.dest}".upper() - env_value = os.environ.get(env_name, "") - help_msg += f" [env: {env_name}={env_value}]" - return help_msg - - def __init__(self, *, formatter_class=_CustomHelpFormatter, **kwargs): - super().__init__(formatter_class=formatter_class, **kwargs) - - def _add_action(self, action): - action.default = os.environ.get( - f"{self.prog}_{action.dest}".upper(), action.default - ) - return super()._add_action(action) - - -def get_cache_dir(prog: str) -> pathlib.Path: - home = pathlib.Path.home() - - if sys.platform.startswith("win"): - local_appdata = pathlib.Path( - os.getenv("LOCALAPPDATA", home / "AppData" / "Local") - ) - path = local_appdata / prog / prog / "Cache" - elif sys.platform == "darwin": - path = home / "Library" / "Caches" / prog - else: - cache_home = pathlib.Path(os.getenv("XDG_CACHE_HOME", home / ".cache")) - path = cache_home / prog - - return path - - -def build_argparse() -> argparse.ArgumentParser: - parser = CustomArgumentParser( - prog="rpau", description="Automatic update code from CPython" - ) - - # Cache - cache_group = parser.add_argument_group("Cache options") - - cache_group.add_argument( - "--cache", - action=argparse.BooleanOptionalAction, - default=True, - help="Whether reading from or writing to the cache is allowed", - ) - cache_group.add_argument( - "--cache-dir", - default=get_cache_dir(parser.prog), - help="Path to the cache directory", - metavar="PATH", - type=pathlib.Path, - ) - - # Output - output_group = parser.add_argument_group("Output option") - - output_group.add_argument( - "-o", - "--output-dir", - default=pathlib.Path(__file__).parents[4], - help="Output dir", - metavar="PATH", - type=pathlib.Path, - ) - - # Filter - filter_group = parser.add_argument_group("Filter options") - - filter_group.add_argument( - "-i", - "--include", - default=".*", - help="RE pattern used to include files and/or directories", - metavar="PATTERN", - type=re.compile, - ) - filter_group.add_argument( - "-e", - "--exclude", - default="^$", - help="RE pattern used to omit files and/or directories", - metavar="PATTERN", - type=re.compile, - ) - - # Global - global_group = parser.add_argument_group("Global options") - - global_group.add_argument( - "--log-level", - choices=logging.getLevelNamesMapping(), - default="WARNING", - help="Log level", - ) - - global_group.add_argument( - "-j", "--workers", default=1, help="Number of processes", type=int - ) - global_group.add_argument( - "-c", - "--conf-dir", - default=pathlib.Path(__file__).parents[2] / "confs", - help="Path to conf dir", - metavar="PATH", - type=pathlib.Path, - ) - global_group.add_argument( - "--base-upstream-url", - default="https://raw.githubusercontent.com/python/cpython/refs/tags", - help="Base upstream url of CPython", - metavar="URL", - ) - global_group.add_argument( - "--default-version", - default="v3.13.7", - help="Fallback version of cpython if none specified in conf", - metavar="VERSION_TAG", - ) - - return parser diff --git a/tools/rpau/src/rpau/logger.py b/tools/rpau/src/rpau/logger.py deleted file mode 100644 index 5af549b2b6..0000000000 --- a/tools/rpau/src/rpau/logger.py +++ /dev/null @@ -1,25 +0,0 @@ -import logging -import sys - -_NAME = "rpau" - - -def build_root_logger( - name: str = _NAME, level: int = logging.WARNING -) -> logging.Logger: - logger = logging.getLogger(name) - logger.setLevel(level) - formatter = logging.Formatter( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - ) - - sh = logging.StreamHandler(sys.stdout) - sh.setFormatter(formatter) - logger.handlers.clear() - logger.addHandler(sh) - - return logger - - -def get_logger(name: str) -> logging.Logger: - return logging.getLogger(_NAME).getChild(name) diff --git a/tools/rpau/src/rpau/logic.py b/tools/rpau/src/rpau/logic.py deleted file mode 100644 index 0b607ff653..0000000000 --- a/tools/rpau/src/rpau/logic.py +++ /dev/null @@ -1,210 +0,0 @@ -import ast -import urllib.request -from typing import TYPE_CHECKING - -from rpau.logger import get_logger - -if TYPE_CHECKING: - import pathlib - -COL_OFFSET = 4 -INDENT1 = " " * COL_OFFSET -INDENT2 = INDENT1 * 2 -COMMENT = "TODO: RUSTPYTHON" - -type Patch = dict[str, dict[str, str]] -type Conf = dict[str, Patch] - -logger = get_logger(__name__) - - -def fetch_upstream(*, base_url: str, path: str, version: str) -> str: - upstream_url = "/".join((base_url, version, path)) - logger.debug(f"{upstream_url=}") - - with urllib.request.urlopen(upstream_url) as f: - contents = f.read().decode() - return contents - - -def get_upstream_contents( - *, base_url: str, cache_dir: "pathlib.Path | None", path: str, version: str -) -> str: - fetch = lambda: fetch_upstream(base_url=base_url, path=path, version=version) - - if cache_dir: - cached_file = cache_dir / version / path - try: - contents = cached_file.read_text() - except FileNotFoundError: - cached_file.parent.mkdir(parents=True, exist_ok=True) - contents = fetch() - cached_file.write_text(contents) - - return contents - else: - return fetch() - - -def format_patch(patch_conf: Patch) -> str: - """ - Transforms a patch definition to a raw python code. - - Parameters - ---------- - patch_conf : Patch - Conf of the patch. - - Returns - ------- - str - Raw python source code. - - Examples - -------- - >>> patch = {"expectedFailure": {"reason": "lorem ipsum"}} - >>> format_patch(patch) - '@unittest.expectedFailure # TODO: RUSTPYTHON; lorem ipsum' - """ - method, conf = next(iter(patch_conf.items())) - prefix = f"@unittest.{method}" - - reason = conf.get("reason", "") - res = "" - match method: - case "expectedFailure": - res = f"{prefix} # {COMMENT}; {reason}" - case "expectedFailureIfWindows" | "skip": - res = f'{prefix}("{COMMENT}; {reason}")' - case "skipIf": - cond = conf["cond"] - res = f'{prefix}({cond}, "{COMMENT}; {reason}")' - - return res.strip().rstrip(";").strip() - - -def iter_patches(tree: ast.Module, conf: Conf) -> "Iterator[tuple[int, str]]": - """ - Get needed patches to apply for given ast tree based on the conf. - - Parameters - ---------- - tree : ast.Module - AST tree to iterate on. - conf : Conf - Dict of `{ClassName: {test_name: Patch}}`. - - Yields - ------ - lineno : int - Line number where to insert the patch. - patch : str - Raw python code to be inserted at `lineno`. - """ - # Phase 1: Iterate and mark existing tests - for key, nodes in ast.iter_fields(tree): - if key != "body": - continue - - for i, cls_node in enumerate(nodes): - if not isinstance(cls_node, ast.ClassDef): - continue - - if not (cls_conf := conf.get(cls_node.name)): - continue - - for fn_node in cls_node.body: - if not isinstance(fn_node, ast.FunctionDef): - continue - - if not (patch_conf := cls_conf.pop(fn_node.name, None)): - continue - - """ - if any( - is_patch_present(dec_node, patch_conf) - for dec_node in fn_node.decorator_list - if isinstance(dec_node, (ast.Attribute, ast.Call)) - ): - continue - """ - - lineno = min( - (dec_node.lineno for dec_node in fn_node.decorator_list), - default=fn_node.lineno, - ) - - indent = " " * fn_node.col_offset - patch = format_patch(patch_conf) - yield (lineno - 1, f"{indent}{patch}") - - # Phase 2: Iterate and mark inhereted tests - for key, nodes in ast.iter_fields(tree): - if key != "body": - continue - - for i, cls_node in enumerate(nodes): - if not isinstance(cls_node, ast.ClassDef): - continue - - if not (cls_conf := conf.get(cls_node.name)): - continue - - for fn_name, patch_conf in cls_conf.items(): - patch = format_patch(patch_conf) - yield ( - cls_node.end_lineno, - f""" -{INDENT1}{patch} -{INDENT1}def {fn_name}(self): -{INDENT2}return super().{fn_name}() -""".rstrip(), - ) - - -def apply_conf(contents: str, conf: dict) -> str: - """ - Patch a given source code based on the conf. - - Parameters - ---------- - contents : str - Raw python source code. - conf : Conf - Dict of `{ClassName: {test_name: Patch}}`. - - Returns - ------- - str - Patched raw python code. - """ - lines = contents.splitlines() - tree = ast.parse(contents) - - # Going in reverse to not distrupt the line offset - patches = list(iter_patches(tree, conf)) - for lineno, patch in sorted(patches, reverse=True): - lines.insert(lineno, patch) - - return "\n".join(lines) - - -def run( - conf: dict, - path: str, - version: str, - output_dir: "pathlib.Path", - cache_dir: "pathlib.Path | None", - base_upstream_url: str, -): - contents = get_upstream_contents( - path=path, version=version, base_url=base_upstream_url, cache_dir=cache_dir - ) - - patched_contents = apply_conf(contents, conf) - new_contents = f"# upstream_version: {version}\n{patched_contents}" - - output_file = output_dir / path - # TODO: Add logic to preserve file permissions if exists - output_file.parent.mkdir(parents=True, exist_ok=True) - output_file.write_text(f"{new_contents}\n") diff --git a/tools/rpau/uv.lock b/tools/rpau/uv.lock deleted file mode 100644 index 813d8d2ed9..0000000000 --- a/tools/rpau/uv.lock +++ /dev/null @@ -1,8 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.12" - -[[package]] -name = "rpau" -version = "0.1.0" -source = { editable = "." } From 32baa80d7f9483c7b8e5450a3632332386940ce0 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:52:42 +0300 Subject: [PATCH 15/29] Revert changes under `Lib/` --- Lib/_colorize.py | 1 - Lib/_ios_support.py | 1 - Lib/_osx_support.py | 1 - Lib/_py_abc.py | 1 - Lib/_pydecimal.py | 3 +- Lib/argparse.py | 1 - Lib/ast.py | 1 - Lib/calendar.py | 1 - Lib/cmd.py | 1 - Lib/colorsys.py | 1 - Lib/csv.py | 1 - Lib/decimal.py | 1 - Lib/difflib.py | 5 +- Lib/fileinput.py | 1 - Lib/heapq.py | 1 - Lib/ipaddress.py | 1 - Lib/keyword.py | 1 - Lib/linecache.py | 1 - Lib/netrc.py | 1 - Lib/ntpath.py | 1 - Lib/numbers.py | 12 +-- Lib/operator.py | 1 - Lib/os.py | 3 +- Lib/posixpath.py | 1 - Lib/pprint.py | 1 - Lib/queue.py | 12 +-- Lib/test/test_os.py | 190 ++++++++++++++++++++++++++++++-------------- Lib/textwrap.py | 1 - Lib/this.py | 1 - 29 files changed, 140 insertions(+), 108 deletions(-) diff --git a/Lib/_colorize.py b/Lib/_colorize.py index b7d31c4ecc..9eb6f0933b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 from __future__ import annotations import io import os diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py index 8b8238cf4c..20467a7c2b 100644 --- a/Lib/_ios_support.py +++ b/Lib/_ios_support.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 import sys try: from ctypes import cdll, c_void_p, c_char_p, util diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 961318359d..0cb064fcd7 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Shared OS X support functions.""" import os diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index 9cff1d05ab..c870ae9048 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 from _weakrefset import WeakSet diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 3851ba4438..ff80180a79 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Copyright (c) 2004 Python Software Foundation. # All rights reserved. @@ -6084,7 +6083,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P\#)? (?P0)? (?P(?!0)\d+)? -(?P[,_])? +(?P,)? (?:\.(?P0|(?!0)\d+))? (?P[eEfFgGn%])? \Z diff --git a/Lib/argparse.py b/Lib/argparse.py index 38afbf007e..bd088ea0e6 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Author: Steven J. Bethard . # New maintainer as of 29 August 2019: Raymond Hettinger diff --git a/Lib/ast.py b/Lib/ast.py index c785bed409..37b20206b8 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """ The `ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with diff --git a/Lib/calendar.py b/Lib/calendar.py index a3f0d8a22b..8c1c646da4 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Calendar printing functions Note when comparing these calendars to the ones printed by cal(1): By diff --git a/Lib/cmd.py b/Lib/cmd.py index 70893d246e..a37d16cd7b 100644 --- a/Lib/cmd.py +++ b/Lib/cmd.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """A generic class to build line-oriented command interpreters. Interpreters constructed with this class obey the following conventions: diff --git a/Lib/colorsys.py b/Lib/colorsys.py index 1c592a4726..e97f91718a 100644 --- a/Lib/colorsys.py +++ b/Lib/colorsys.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Conversion functions between RGB and other color systems. This modules provides two functions for each color system ABC: diff --git a/Lib/csv.py b/Lib/csv.py index db6b6c3b46..cd20265987 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 r""" CSV parsing and writing. diff --git a/Lib/decimal.py b/Lib/decimal.py index 71d612763b..ee3147f5dd 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Decimal fixed-point and floating-point arithmetic. This is an implementation of decimal floating-point arithmetic based on diff --git a/Lib/difflib.py b/Lib/difflib.py index 38f51f9778..33e7e6c165 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """ Module difflib -- helpers for computing deltas between objects. @@ -79,8 +78,8 @@ class SequenceMatcher: sequences. As a rule of thumb, a .ratio() value over 0.6 means the sequences are close matches: - >>> print(round(s.ratio(), 2)) - 0.87 + >>> print(round(s.ratio(), 3)) + 0.866 >>> If you're only interested in where the sequences match, diff --git a/Lib/fileinput.py b/Lib/fileinput.py index 724eb8b888..3dba3d2fbf 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Helper class to quickly write a loop over all standard input files. Typical use is: diff --git a/Lib/heapq.py b/Lib/heapq.py index 31910cce8a..2fd9d1ff4b 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Heap queue algorithm (a.k.a. priority queue). Heaps are arrays for which a[k] <= a[2*k+1] and a[k] <= a[2*k+2] for diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index af2fceb508..67e45450fc 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Copyright 2007 Google Inc. # Licensed to PSF under a Contributor Agreement. diff --git a/Lib/keyword.py b/Lib/keyword.py index 6f308386b4..e22c837835 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Keywords (from "Grammar/python.gram") This file is automatically generated; please don't muck it up! diff --git a/Lib/linecache.py b/Lib/linecache.py index b0b1742582..dc02de19eb 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Cache lines from Python source files. This is intended to read lines from modules imported -- hence if a filename diff --git a/Lib/netrc.py b/Lib/netrc.py index 1cca5d1c89..bd003e80a4 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """An object-oriented interface to .netrc files.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 55b51f53cd..9cdc16480f 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Module 'ntpath' -- common operations on WinNT/Win95 pathnames """Common pathname manipulations, WindowsNT/95 version. diff --git a/Lib/numbers.py b/Lib/numbers.py index 4fca6f9229..a2913e32cf 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Copyright 2007 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. @@ -291,27 +290,18 @@ def conjugate(self): class Rational(Real): - """To Real, Rational adds numerator and denominator properties. - - The numerator and denominator values should be in lowest terms, - with a positive denominator. - """ + """.numerator and .denominator should be in lowest terms.""" __slots__ = () @property @abstractmethod def numerator(self): - """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): - """The denominator of a rational number in lowest terms. - - This denominator should be positive. - """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/operator.py b/Lib/operator.py index 553fe6789d..02ccdaa13d 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """ Operator Interface diff --git a/Lib/os.py b/Lib/os.py index d1c895c2be..b4c9f84c36 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 r"""OS routines for NT or Posix depending on what system we're on. This exports: @@ -11,7 +10,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\n' or '\r\n') + - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index cf8a8c4494..80561ae7e5 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Common operations on Posix pathnames. Instead of importing this module directly, import os and refer to diff --git a/Lib/pprint.py b/Lib/pprint.py index d9b1ea69b0..9314701db3 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # Author: Fred L. Drake, Jr. # fdrake@acm.org # diff --git a/Lib/queue.py b/Lib/queue.py index 5cb1c4d8bc..25beb46e30 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 '''A multi-producer, multi-consumer queue.''' import threading @@ -81,6 +80,9 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). + shutdown(immediate=True) calls task_done() for each remaining item in + the queue. + Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -237,11 +239,9 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. - - If 'immediate', the queue is drained and unfinished tasks - is reduced by the number of drained tasks. If unfinished tasks - is reduced to zero, callers of Queue.join are unblocked. + All blocked callers of put() and get() will be unblocked. If + 'immediate', a task is marked as done for each item remaining in + the queue, which may unblock callers of join(). ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 4a54971b8a..a025a2d4ff 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 # As a test suite for the os module, this is woefully inadequate, but this # does add tests for a few functions which have been determined to be more # portable than they had been thought to be. @@ -14,6 +13,7 @@ import locale import os import pickle +import platform import select import selectors import shutil @@ -188,7 +188,8 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) @@ -715,7 +716,8 @@ def check_file_attributes(self, result): self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat return value doesnt have st_file_attributes attribute") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -737,7 +739,8 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is @@ -760,7 +763,8 @@ def test_access_denied(self): self.assertNotEqual(result.st_size, 0) self.assertTrue(os.path.isfile(fname)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 1] Incorrect function") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -818,7 +822,8 @@ def _test_utime(self, set_time, filename=None): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -884,7 +889,8 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference)") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -913,21 +919,24 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_utime_nonexistent(self): now = time.time() filename = 'nonexistent' @@ -948,7 +957,8 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -959,7 +969,7 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: NotImplementedError not raised") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1161,8 +1171,9 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: ValueError not raised by putenv") @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): # Empty variable name is invalid. @@ -1737,15 +1748,18 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components + # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) + @unittest.expectedFailure def test_compare_to_walk(self): return super().test_compare_to_walk() - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components + # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) + @unittest.expectedFailure def test_dir_fd(self): return super().test_dir_fd() - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Can't mix strings and bytes in path components + # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) + @unittest.expectedFailure def test_yields_correct_dir_fd(self): return super().test_yields_correct_dir_fd() @@ -1797,7 +1811,8 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1816,7 +1831,8 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -2103,7 +2119,8 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'os'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -2118,7 +2135,8 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named 'os'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -2206,7 +2224,8 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2266,7 +2285,8 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2288,7 +2308,8 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2369,7 +2390,8 @@ def check_bool(self, f, *args, **kwargs): with self.assertRaises(RuntimeWarning): f(fd, *args, **kwargs) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") self.check_bool(os.fdopen, encoding="utf-8") @@ -2426,7 +2448,8 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( support.is_emscripten or support.is_wasi, @@ -2439,19 +2462,22 @@ def test_fpathconf(self): self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) self.check_bool(os.truncate, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) @@ -2465,7 +2491,8 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OSError: [Errno 18] There are no more files") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2474,7 +2501,8 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) @@ -2486,11 +2514,13 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fchdir(self): return super().test_fchdir() - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fsync(self): return super().test_fsync() @@ -2533,6 +2563,7 @@ def test_unicode_name(self): self.file2 = self.file1 + "2" self._test_link(self.file1, self.file2) + @unittest.skipIf(sys.platform == "win32", "Posix specific tests") class PosixUidGidTests(unittest.TestCase): # uid_t and gid_t are 32-bit unsigned integers on Linux @@ -2734,12 +2765,14 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ModuleNotFoundError: No module named '_ctypes'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2798,7 +2831,8 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3152,7 +3186,8 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'CreateJunction'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -3166,7 +3201,8 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AttributeError: module '_winapi' has no attribute 'CreateJunction'") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -3226,7 +3262,8 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat PermissionError: [Errno 5] Access is denied") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading @@ -3436,7 +3473,8 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.spawnv not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3445,7 +3483,8 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; OverflowError: Python int too large to convert to Rust i32") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3569,7 +3608,8 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes + # TODO: RUSTPYTHON fix spawnv bytes + @unittest.expectedFailure @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -4184,8 +4224,16 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # gh-126112: Use 10 ms to tolerate slow buildbots - CLOCK_RES_PLACES = 2 # 10 ms + # 1 ms accuracy is reliably achievable on every platform except Android + # emulators, where we allow 10 ms (gh-108277). + + # XXX: RUSTPYTHON; AttributeError: module 'platform' has no attribute 'android_ver' + #if sys.platform == "android" and platform.android_ver().is_emulator: + if sys.platform == "android": + CLOCK_RES_PLACES = 2 + else: + CLOCK_RES_PLACES = 3 + CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4529,7 +4577,8 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : + # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : ) + @unittest.expectedFailure def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4622,7 +4671,8 @@ def test_process_cpu_count_affinity(self): # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4667,7 +4717,8 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4683,7 +4734,8 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4697,6 +4749,7 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) + # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) @@ -4706,13 +4759,15 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) + # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.dup not implemented yet for all platforms") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4852,7 +4907,8 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised + # TODO: RUSTPYTHON (AssertionError: TypeError not raised) + @unittest.expectedFailure def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4937,11 +4993,13 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by DirEntry + # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) + @unittest.expectedFailure def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - @unittest.expectedFailure # TODO: RUSTPYTHON; pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry + # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry) + @unittest.expectedFailure def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -4986,7 +5044,8 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by ScandirIter + # TODO: RUSTPYTHON (AssertionError: TypeError not raised by ScandirIter) + @unittest.expectedFailure def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -5029,7 +5088,8 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON; flaky test") + # TODO: RUSTPYTHON + @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") def test_attributes(self): link = os_helper.can_hardlink() symlink = os_helper.can_symlink() @@ -5129,7 +5189,8 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_dir() is False") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -5152,7 +5213,8 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_file() is False") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -5213,7 +5275,8 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -5240,7 +5303,8 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: FileNotFoundError not raised by scandir") + # TODO: RUSTPYTHON + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -5296,7 +5360,8 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -5369,7 +5434,8 @@ class A(os.PathLike): def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pathlike_subclass_slots(self): class A(os.PathLike): __slots__ = () @@ -5377,7 +5443,8 @@ def __fspath__(self): return '' self.assertFalse(hasattr(A(), '__dict__')) - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fspath_set_to_None(self): class Foo: __fspath__ = None @@ -5480,7 +5547,8 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - @unittest.expectedFailure # TODO: RUSTPYTHON + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_fork_at_finalization(self): code = """if 1: import atexit diff --git a/Lib/textwrap.py b/Lib/textwrap.py index f85b003efd..7ca393d1c3 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 """Text wrapping and filling. """ diff --git a/Lib/this.py b/Lib/this.py index 8e9f9cc378..e68dd3ff39 100644 --- a/Lib/this.py +++ b/Lib/this.py @@ -1,4 +1,3 @@ -# upstream_version: v3.13.7 s = """Gur Mra bs Clguba, ol Gvz Crgref Ornhgvshy vf orggre guna htyl. From fb7324d37fb813e1281723aa849911b690bdabe4 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 16:56:33 +0300 Subject: [PATCH 16/29] Don't crash if cls renamed/moved --- scripts/lib_updater.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 4b753f08fb..cf102cd0eb 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -197,7 +197,10 @@ def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, # Phase 2: Iterate and mark inhereted tests for cls_name, tests in patches.items(): - lineno = cache[cls_name] + lineno = cache.get(cls_name) + if not lineno: + print(f"WARNING: {cls_name} does not exist in remote file") + continue for test_name, specs in tests.items(): patch_lines = "\n".join(f"{INDENT1}{spec.fmt()}" for spec in specs) yield ( From f4056ac0cfa0b0fb1b3d9edb3cc8472e15a02ada Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 17:02:13 +0300 Subject: [PATCH 17/29] Update `Lib/test/test_os.py` with tool --- Lib/test/test_os.py | 193 ++++++++++++++------------------------------ 1 file changed, 61 insertions(+), 132 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index a025a2d4ff..03996a6036 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,7 +13,6 @@ import locale import os import pickle -import platform import select import selectors import shutil @@ -188,8 +187,7 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; , BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) @@ -716,8 +714,7 @@ def check_file_attributes(self, result): self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat return value doesnt have st_file_attributes attribute") @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -739,8 +736,7 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is @@ -763,8 +759,7 @@ def test_access_denied(self): self.assertNotEqual(result.st_size, 0) self.assertTrue(os.path.isfile(fname)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 1] Incorrect function.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -822,8 +817,7 @@ def _test_utime(self, set_time, filename=None): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -835,9 +829,7 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` - # return (ns * 1e-9) + 0.5e-9 - return (ns * 1e-9) + return (ns * 1e-9) + 0.5e-9 def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter @@ -889,8 +881,7 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -919,24 +910,21 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_utime_nonexistent(self): now = time.time() filename = 'nonexistent' @@ -957,8 +945,7 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -969,7 +956,7 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: NotImplementedError not raised)") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1171,9 +1158,8 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: ValueError not raised by putenv)") @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): # Empty variable name is invalid. @@ -1748,18 +1734,15 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_compare_to_walk(self): return super().test_compare_to_walk() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_dir_fd(self): return super().test_dir_fd() - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (TypeError: Can't mix strings and bytes in path components) def test_yields_correct_dir_fd(self): return super().test_yields_correct_dir_fd() @@ -1811,8 +1794,7 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1831,8 +1813,7 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -2119,8 +2100,7 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; on Windows (ModuleNotFoundError: No module named 'os')") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -2135,8 +2115,7 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -2224,8 +2203,7 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2285,8 +2263,7 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2308,8 +2285,7 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.execve not implemented yet for all platforms") @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2390,8 +2366,7 @@ def check_bool(self, f, *args, **kwargs): with self.assertRaises(RuntimeWarning): f(fd, *args, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") self.check_bool(os.fdopen, encoding="utf-8") @@ -2448,8 +2423,7 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( support.is_emscripten or support.is_wasi, @@ -2462,22 +2436,19 @@ def test_fpathconf(self): self.check_bool(os.pathconf, "PC_NAME_MAX") self.check_bool(os.fpathconf, "PC_NAME_MAX") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) self.check_bool(os.truncate, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) @@ -2491,8 +2462,7 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2501,8 +2471,7 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) @@ -2514,13 +2483,11 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fchdir(self): return super().test_fchdir() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fsync(self): return super().test_fsync() @@ -2563,7 +2530,6 @@ def test_unicode_name(self): self.file2 = self.file1 + "2" self._test_link(self.file1, self.file2) - @unittest.skipIf(sys.platform == "win32", "Posix specific tests") class PosixUidGidTests(unittest.TestCase): # uid_t and gid_t are 32-bit unsigned integers on Linux @@ -2765,14 +2731,12 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2831,8 +2795,7 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3186,8 +3149,7 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -3201,8 +3163,7 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -3262,8 +3223,7 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.stat (PermissionError: [Errno 5] Access is denied.)") @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading @@ -3473,8 +3433,7 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.spawnv not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3483,8 +3442,7 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (OverflowError: Python int too large to convert to Rust i32)") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3608,8 +3566,7 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - # TODO: RUSTPYTHON fix spawnv bytes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -4224,16 +4181,8 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - - # XXX: RUSTPYTHON; AttributeError: module 'platform' has no attribute 'android_ver' - #if sys.platform == "android" and platform.android_ver().is_emulator: - if sys.platform == "android": - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4577,8 +4526,7 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : ) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : ) def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4671,8 +4619,7 @@ def test_process_cpu_count_affinity(self): # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4717,8 +4664,7 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4734,8 +4680,7 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.get_inheritable not implemented yet for all platforms") def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4749,7 +4694,6 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) - # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) @@ -4759,15 +4703,13 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) - # TODO: RUSTPYTHON @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; os.dup not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4907,8 +4849,7 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - # TODO: RUSTPYTHON (AssertionError: TypeError not raised) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised) def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4993,13 +4934,11 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised by DirEntry) def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (pickle.PicklingError: Can't pickle : it's not found as _os.DirEntry) def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -5044,8 +4983,7 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by ScandirIter) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; (AssertionError: TypeError not raised by ScandirIter) def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -5088,8 +5026,7 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - # TODO: RUSTPYTHON - @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") + @unittest.skipIf(sys.platform == 'linux', "TODO: RUSTPYTHON; , flaky test") def test_attributes(self): link = os_helper.can_hardlink() symlink = os_helper.can_symlink() @@ -5189,8 +5126,7 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_dir() is False") def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -5213,8 +5149,7 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; entry.is_file() is False") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -5275,8 +5210,7 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -5303,8 +5237,7 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - # TODO: RUSTPYTHON - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") + @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; (AssertionError: FileNotFoundError not raised by scandir)") @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -5360,8 +5293,7 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -5434,8 +5366,7 @@ class A(os.PathLike): def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pathlike_subclass_slots(self): class A(os.PathLike): __slots__ = () @@ -5443,8 +5374,7 @@ def __fspath__(self): return '' self.assertFalse(hasattr(A(), '__dict__')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fspath_set_to_None(self): class Foo: __fspath__ = None @@ -5547,8 +5477,7 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fork_at_finalization(self): code = """if 1: import atexit From 4296b59f6e20824293980f6632785a5a0a575749 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 17:06:38 +0300 Subject: [PATCH 18/29] apply patch --- Lib/test/test_os.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 03996a6036..442336fce8 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -829,7 +829,9 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - return (ns * 1e-9) + 0.5e-9 + # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` + # return (ns * 1e-9) + 0.5e-9 + return (ns * 1e-9) def test_utime_by_indexed(self): # pass times as floating-point seconds as the second indexed parameter From e2aa220feebbccd292d60ddf54b90c78d35bdc19 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 17:40:14 +0300 Subject: [PATCH 19/29] Fix double assignment --- scripts/lib_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index cf102cd0eb..611a5cdaf6 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -57,7 +57,7 @@ class PatchSpec(typing.NamedTuple): reason: str = "" def fmt(self) -> str: - prefix = prefix = f"@unittest.{self.ut_method}" + prefix = f"@unittest.{self.ut_method}" match self.ut_method: case UtMethod.ExpectedFailure: line = f"{prefix} # {COMMENT}; {self.reason}" From 74b47a0e679d5c7c56f35c21b17fbfb395c52389 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:42:44 +0300 Subject: [PATCH 20/29] Better args --- scripts/lib_updater.py | 50 +++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 611a5cdaf6..c17165bad9 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -112,7 +112,7 @@ def iter_patch_entires( match ut_method: case UtMethod.ExpectedFailure: - for line in lines[dec_node.lineno - 2 : dec_node.lineno]: + for line in lines[dec_node.lineno - 2 : dec_node.lineno + 1]: if COMMENT not in line: continue reason = "".join(re.findall(rf"{COMMENT} (.*)", line)) @@ -231,32 +231,31 @@ def build_argparse() -> argparse.ArgumentParser: description="Helper tool for updating files under Lib/" ) - parser.add_argument( - "orig_file", help="File to gather patches from", type=pathlib.Path + patches_group = parser.add_mutually_exclusive_group(required=True) + patches_group.add_argument( + "-p", + "--patches", + help="File path to file containing patches in a JSON format", + type=pathlib.Path, + ) + patches_group.add_argument( + "--from", + help="File to gather patches from", + dest="gather_from", + type=pathlib.Path, ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( - "remote_file", - nargs="?", + "--to", help="File to apply patches to", type=pathlib.Path, ) group.add_argument( "--show-patches", action="store_true", help="Show the patches and exit" ) - parser.add_argument( - "-p", - "--patches", - help="File path to file containing patches in a JSON format", - type=pathlib.Path, - ) - parser.add_argument( - "--inplace", - action=argparse.BooleanOptionalAction, - default=False, - help="Whether write the changes", - ) + + parser.add_argument("-o", "--output", help="Output file", type=pathlib.Path) return parser @@ -265,7 +264,6 @@ def build_argparse() -> argparse.ArgumentParser: parser = build_argparse() args = parser.parse_args() - contents = args.orig_file.read_text() if args.patches: patches = { cls_name: { @@ -275,7 +273,7 @@ def build_argparse() -> argparse.ArgumentParser: for cls_name, tests in json.loads(args.patches.read_text()).items() } else: - patches = build_patch_dict(iter_patches(contents)) + patches = build_patch_dict(iter_patches(args.gather_from.read_text())) if args.show_patches: patches = { @@ -285,13 +283,15 @@ def build_argparse() -> argparse.ArgumentParser: } for cls_name, tests in patches.items() } - output = json.dumps(patches, indent=4) - sys.stdout.write(f"{output}\n") + output = json.dumps(patches, indent=4) + "\n" + if args.output: + args.output.write_text(output) + else: + sys.stdout.write(f"{output}\n") sys.exit(0) - patched = apply_patches(args.remote_file.read_text(), patches) - - if args.inplace: - args.orig_file.write_text(patched) + patched = apply_patches(args.to.read_text(), patches) + if args.output: + args.output.write_text(patched) else: sys.stdout.write(patched) From 51c6ad9b56e0207972b38bffd235a3442e857b43 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:43:12 +0300 Subject: [PATCH 21/29] Update `test_list.py` as well --- Lib/test/test_list.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 42d8dcbbe1..ed061384f1 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -48,8 +48,7 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): list(sequence=[]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): class subclass(list): pass @@ -105,8 +104,7 @@ def test_empty_slice(self): x[:] = x self.assertEqual(x, []) - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON crash") + @unittest.skip("TODO: RUSTPYTHON; crash") def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -120,8 +118,7 @@ def test_list_resize_overflow(self): with self.assertRaises((MemoryError, OverflowError)): lst *= size - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON hangs") + @unittest.skip("TODO: RUSTPYTHON; hangs") def test_repr_mutate(self): class Obj: @staticmethod @@ -230,7 +227,6 @@ class L(list): pass with self.assertRaises(TypeError): (3,) + L([1,2]) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_equal_operator_modifying_operand(self): # test fix for seg fault reported in bpo-38588 part 2. @@ -257,7 +253,6 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_lt_operator_modifying_operand(self): # See gh-120298 From 01a90efd5fed82e77045ce451993cbd9e322948a Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 20:53:55 +0300 Subject: [PATCH 22/29] Less complex print --- scripts/lib_updater.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index c17165bad9..886715a3ce 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -199,7 +199,7 @@ def iter_patch_lines(tree: ast.Module, patches: Patches) -> "Iterator[tuple[int, for cls_name, tests in patches.items(): lineno = cache.get(cls_name) if not lineno: - print(f"WARNING: {cls_name} does not exist in remote file") + print(f"WARNING: {cls_name} does not exist in remote file", file=sys.stderr) continue for test_name, specs in tests.items(): patch_lines = "\n".join(f"{INDENT1}{spec.fmt()}" for spec in specs) @@ -287,11 +287,11 @@ def build_argparse() -> argparse.ArgumentParser: if args.output: args.output.write_text(output) else: - sys.stdout.write(f"{output}\n") + print(output, end="") sys.exit(0) patched = apply_patches(args.to.read_text(), patches) if args.output: args.output.write_text(patched) else: - sys.stdout.write(patched) + print(patched, end="") From a0e56ae47b5ca0afb646bb5cef0303d5d5131c83 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 21:12:45 +0300 Subject: [PATCH 23/29] Improve exoectedFailure match --- scripts/lib_updater.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 886715a3ce..aea62c3d49 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -112,7 +112,10 @@ def iter_patch_entires( match ut_method: case UtMethod.ExpectedFailure: - for line in lines[dec_node.lineno - 2 : dec_node.lineno + 1]: + # Search first on decorator line, then in the line before + for line in lines[ + dec_node.lineno - 3 : dec_node.lineno - 1 : -1 + ]: if COMMENT not in line: continue reason = "".join(re.findall(rf"{COMMENT} (.*)", line)) From 43a63a8808f2de36a7f3689dd32fd2600275ea07 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 30 Aug 2025 21:41:46 +0300 Subject: [PATCH 24/29] fix list slice --- scripts/lib_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index aea62c3d49..bd7063a6a8 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -114,7 +114,7 @@ def iter_patch_entires( case UtMethod.ExpectedFailure: # Search first on decorator line, then in the line before for line in lines[ - dec_node.lineno - 3 : dec_node.lineno - 1 : -1 + dec_node.lineno - 1 : dec_node.lineno - 3 : -1 ]: if COMMENT not in line: continue From 3e2c1f1f3575c696add0a50141a9a1d8155bc84c Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Tue, 2 Sep 2025 19:25:56 +0300 Subject: [PATCH 25/29] Add __doc__ and to help --- scripts/lib_updater.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index bd7063a6a8..11cdfc940a 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -1,4 +1,29 @@ #!/usr/bin/env python +__doc__ = """ +This tool helps with updating test files from CPython. + +Examples +-------- +To move the patches found in `Lib/test/foo.py` to ` ~/cpython/Lib/test/foo.py` then write the contents back to `Lib/test/foo.py` + +>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py -o Lib/test/foo.py + +You can run the same command without `-o` to print the output to stdout: + +>>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py + +To get a baseline of patches, you can alter the patches file with your favorite tool/script/etc and then reapply it with: + +>>> ./{fname} --from Lib/test/foo.py --show-patches -o my_patches.json + +(omit the `-o` flag to print to stdout instead). + +When you want to apply your own patches: + +>>> ./{fname} -p my_patches.json --to Lib/test/foo.py +""".format(fname=__import__("os").path.basename(__file__)) + + import argparse import ast import collections @@ -231,7 +256,7 @@ def apply_patches(contents: str, patches: Patches) -> str: def build_argparse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( - description="Helper tool for updating files under Lib/" + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) patches_group = parser.add_mutually_exclusive_group(required=True) From 3e9872ca4f0e3e8a30803a7678388d1b8fade14e Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Thu, 4 Sep 2025 11:53:09 +0300 Subject: [PATCH 26/29] Update scripts/lib_updater.py Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com> --- scripts/lib_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 11cdfc940a..463eae5e70 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -8,7 +8,7 @@ >>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py -o Lib/test/foo.py -You can run the same command without `-o` to print the output to stdout: +You can run the same command without `-o` to override the `--from` path: >>> ./{fname} --from Lib/test/foo.py --to ~/cpython/Lib/test/foo.py From d0763e45f777d16257b219ccc457edf37c19f6ba Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:15:34 +0300 Subject: [PATCH 27/29] Clearer output arg --- scripts/lib_updater.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 11cdfc940a..77fc0fa201 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -16,7 +16,7 @@ >>> ./{fname} --from Lib/test/foo.py --show-patches -o my_patches.json -(omit the `-o` flag to print to stdout instead). +(By default the output is set to print to stdout). When you want to apply your own patches: @@ -254,6 +254,15 @@ def apply_patches(contents: str, patches: Patches) -> str: return f"{joined}\n" +def write_output(data: str, dest: str) -> None: + if dest == "-": + print(data, end="") + return + + with open(dest, "w") as fd: + fd.write(data) + + def build_argparse() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -283,7 +292,9 @@ def build_argparse() -> argparse.ArgumentParser: "--show-patches", action="store_true", help="Show the patches and exit" ) - parser.add_argument("-o", "--output", help="Output file", type=pathlib.Path) + parser.add_argument( + "-o", "--output", default="-", help="Output file. Set to '-' for stdout" + ) return parser @@ -312,14 +323,8 @@ def build_argparse() -> argparse.ArgumentParser: for cls_name, tests in patches.items() } output = json.dumps(patches, indent=4) + "\n" - if args.output: - args.output.write_text(output) - else: - print(output, end="") + write_output(output, args.output) sys.exit(0) patched = apply_patches(args.to.read_text(), patches) - if args.output: - args.output.write_text(patched) - else: - print(patched, end="") + write_output(patched, args.output) From e00bb28c1008ff5d859a23e8fd8d3da9e99010aa Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Fri, 5 Sep 2025 11:20:06 +0300 Subject: [PATCH 28/29] Don't crash on missing id --- scripts/lib_updater.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 77fc0fa201..9fc4c4373a 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -126,7 +126,10 @@ def iter_patch_entires( dec_node if isinstance(dec_node, ast.Attribute) else dec_node.func ) - if isinstance(attr_node, ast.Name) or attr_node.value.id != "unittest": + if ( + isinstance(attr_node, ast.Name) + or getattr(attr_node.value, "id", None) != "unittest" + ): continue cond = None From c0e90cc58f5940c5e0b5e33ad89d1f9b52e85d56 Mon Sep 17 00:00:00 2001 From: ShaharNaveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Sat, 6 Sep 2025 22:28:00 +0300 Subject: [PATCH 29/29] Fix comment regex --- scripts/lib_updater.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib_updater.py b/scripts/lib_updater.py index 1db2cc5132..b019521982 100755 --- a/scripts/lib_updater.py +++ b/scripts/lib_updater.py @@ -146,7 +146,7 @@ def iter_patch_entires( ]: if COMMENT not in line: continue - reason = "".join(re.findall(rf"{COMMENT} (.*)", line)) + reason = "".join(re.findall(rf"{COMMENT}.?(.*)", line)) break else: continue