diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af904a567cfb7e2..bdf134254121e58 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -156,7 +156,7 @@ Misc/libabigail.abignore @encukou # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -164,8 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team @@ -176,8 +175,8 @@ Lib/test/test__osx_support.py @python/macos-team Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) Platforms/WASI @brettcannon @emmatyping @savannahostrowski @@ -574,9 +573,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner @@ -587,10 +586,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -623,9 +622,6 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1af3a0607f9ad2a..12bf160178e3c72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,53 @@ jobs: if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check-abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-22.04 # 24.04 causes spurious errors + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq --no-install-recommends abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + id: check + run: | + if ! make check-abidump; then + echo "Generated ABI file is not up to date." + echo "Please add the release manager of this branch as a reviewer of this PR." + echo "" + echo "The up to date ABI file should be attached to this build as an artifact." + echo "" + echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump" + echo "" + exit 1 + fi + - name: Generate updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + run: | + make regen-abidump + - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + name: Publish updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + with: + name: abi-data + path: ./Doc/data/*.abi + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 490c32ecfc9a629..d748b6ff63e68a1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -69,12 +69,11 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "3.15" - allow-prereleases: true - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index f10503055b2259a..93b419159fa8177 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -38,9 +38,8 @@ jobs: run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.5 xz gdbm tcl-tk@9 make - # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@9 + brew bundle --file=Misc/Brewfile + brew install make - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4c8d0c8a2f984fc..c6e8128884e90c2 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -22,8 +22,6 @@ permissions: env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: diff --git a/.gitignore b/.gitignore index 118eb5ee76e8051..78b6d4efb0e1097 100644 --- a/.gitignore +++ b/.gitignore @@ -177,7 +177,3 @@ Python/frozen_modules/MANIFEST # People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. /.claude/ CLAUDE.local.md - -#### main branch only stuff below this line, things to backport go above. #### -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 254a22f2622bd8e..a6ea0a72e76f9db 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -12,6 +12,9 @@ It can be sometimes faster to fix bugs yourself and contribute patches to Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. + +.. _reporting-documentation-bugs: + Documentation bugs ================== diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 629312bd771beb2..10f96c7cb75e882 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -130,7 +130,7 @@ rather than dereferencing them through pointers. Please note, that these functions are :term:`soft deprecated` since Python 3.15. Avoid using this API in a new code to do complex arithmetic: either use -the `Number Protocol `_ API or use native complex types, like +the :ref:`Number Protocol ` API or use native complex types, like :c:expr:`double complex`. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index a2a0d0d80657ebf..556113a97bf772f 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -151,7 +151,7 @@ Dictionary objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. The first argument can be a :class:`dict` or a :class:`frozendict`. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index fd9ea6272df7d84..f3f408c400bed08 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -412,7 +412,7 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. @@ -1392,7 +1392,7 @@ Tracebacks This function will return ``NULL`` on success, or an error message on error. - This function is meant to debug debug situations such as segfaults, fatal + This function is meant to debug situations such as segfaults, fatal errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each thread. It only writes the tracebacks of the first *max_threads* threads, further output is truncated with the line ``...``. If *max_threads* is 0, the diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index e2d363b911a87c6..ec9462931d56c2c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -393,11 +393,6 @@ Importing Modules Make all imports lazy by default. - .. c:enumerator:: PyImport_LAZY_NONE - - Disable lazy imports entirely. Even explicit ``lazy`` statements become - eager imports. - .. versionadded:: 3.15 .. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 60e3ae4a064e729..874e422d4701dd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -71,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -81,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index bd05e628faaaa10..a962c4ee09ad77d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -49,3 +49,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst index 89e0a28bf3b835b..b1b7329a5d42c59 100644 --- a/Doc/c-api/sentinel.rst +++ b/Doc/c-api/sentinel.rst @@ -14,17 +14,29 @@ Sentinel objects .. c:function:: int PySentinel_Check(PyObject *o) - Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type - does not allow subclasses, so this check is exact. + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. .. versionadded:: 3.15 -.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name) +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name, const char *repr) Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to *name* and :attr:`~sentinel.__module__` set to *module_name*. *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` - is set to ``None``. + is set to ``None``. If *repr* is ``NULL``, ``repr()`` returns :attr:`~sentinel.__name__`. Return ``NULL`` with an exception set on failure. For pickling to work, *module_name* must be the name of an importable diff --git a/Doc/c-api/slots.rst b/Doc/c-api/slots.rst index b61c2f2e17bbc3a..84a125cb60bae7e 100644 --- a/Doc/c-api/slots.rst +++ b/Doc/c-api/slots.rst @@ -7,10 +7,10 @@ Definition slots To define :ref:`module objects ` and :ref:`classes ` using the C API, you may use -an array of *slots* -- essentally, key-value pairs that describe features +an array of *slots* -- essentially, key-value pairs that describe features of the object to create. This decouples the data from the structures used at runtime, allowing CPython --- and other Python C API implementations -- to update the stuctures without +-- and other Python C API implementations -- to update the structures without breaking backwards compatibility. This section documents slots in general. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 0ff066680b8c733..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -114,7 +114,7 @@ versions of Python. All functions in Stable ABI are present as functions in Python's shared library, not solely as macros. -This makes them usable are usable from languages that don't use the C +This makes them usable in languages that don't use the C preprocessor, including Python's :py:mod:`ctypes`. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst index 7e9894f4d692d6b..6f18c047a24a92f 100644 --- a/Doc/c-api/synchronization.rst +++ b/Doc/c-api/synchronization.rst @@ -238,7 +238,7 @@ are not available. .. c:function:: void PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); void PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); - .. (These need to be in a separate section without a Stable ABI anotation.) + .. (These need to be in a separate section without a Stable ABI annotation.) To be used only as in the macro expansions listed :ref:`earlier in this section `. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 38db69e5c6db96d..dcc9e243c2f3147 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2975,13 +2975,13 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. @@ -3007,10 +3007,10 @@ Buffer Object Structures schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -3064,7 +3064,7 @@ Buffer Object Structures *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 059a7ef399ae0f5..401c99ebeb0fec6 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -762,7 +762,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1174,7 +1174,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1196,7 +1196,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs diff --git a/Doc/conf.py b/Doc/conf.py index e2dff74538a3422..a766bac70632555 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -78,7 +76,7 @@ # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -557,16 +555,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True diff --git a/Doc/data/python3.15.abi b/Doc/data/python3.15.abi new file mode 100644 index 000000000000000..d769d6a764c62eb --- /dev/null +++ b/Doc/data/python3.15.abi @@ -0,0 +1,33503 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 663b79e45eec17d..60c02aabeb89c51 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2040,6 +2040,7 @@ PySeqIter_New:PyObject*:seq:0: PySentinel_New:PyObject*::+1: PySentinel_New:const char*:name:: PySentinel_New:const char*:module_name:: +PySentinel_New:const char*:repr:: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 2d4278c9d97c859..86080fac7163838 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,7 @@ role,name,added,ifdef_note,struct_abi_kind macro,METH_CLASS,3.2,, macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.7,, +macro,METH_FASTCALL,3.10,, macro,METH_METHOD,3.7,, macro,METH_NOARGS,3.2,, macro,METH_O,3.2,, diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst index 894f5bdbb8f09c2..55a772e2aca24f5 100644 --- a/Doc/extending/first-extension-module.rst +++ b/Doc/extending/first-extension-module.rst @@ -164,7 +164,7 @@ Then, create ``meson.build`` containing the following: .. note:: - See `meson-python documentation `_ for details on + See the `meson-python documentation `_ for details on configuration. Now, build install the *project in the current directory* (``.``) via ``pip``: diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst index 3adfedbf410ecc8..7a7a87cb9584001 100644 --- a/Doc/howto/a-conceptual-overview-of-asyncio.rst +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -115,7 +115,7 @@ The terms "coroutine function" and "coroutine object" are often conflated as coroutine. That can be confusing! In this article, coroutine specifically refers to a coroutine object, or more -precisely, an instance of :data:`types.CoroutineType` (native coroutine). +precisely, an instance of :class:`types.CoroutineType` (native coroutine). Note that coroutines can also exist as instances of :class:`collections.abc.Coroutine` -- a distinction that matters for type checking. diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst new file mode 100644 index 000000000000000..ed7a324c4af6f0a --- /dev/null +++ b/Doc/howto/abi3t-migration.rst @@ -0,0 +1,614 @@ +.. highlight:: c + +.. _abi3t-migration-howto: + +****************************************************** +Migrating to Stable ABI for free threading (``abi3t``) +****************************************************** + +Starting with the 3.15 release, CPython supports a variant of the Stable ABI +that supports :term:`free-threaded ` Python: +Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +This document describes how to adapt C API extensions to support free threading. + +Why do this +=========== + +The typical reason to use Stable ABI is to reduce the number of artifacts that +you need to build and distribute for each version of your library. + +Without the Stable ABI, you must build a separate shared library, and typically +a *wheel* distribution, for each feature version of CPython you wish +to support. +For example, each tag in the following table represents a separate +library/wheel: + ++-----------------+-----------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+=======================+========================+ +| 3.12 | ``cpython-312`` | --- | ++-----------------+-----------------------+------------------------+ +| 3.13 | ``cpython-313`` | ``cpython-313t`` | ++-----------------+-----------------------+------------------------+ +| 3.14 | ``cpython-314`` | ``cpython-314t`` | ++-----------------+-----------------------+------------------------+ +| 3.15 | ``cpython-315`` | ``cpython-315t`` | ++-----------------+-----------------------+------------------------+ +| 3.16 | ``cpython-316`` | ``cpython-316t`` | ++-----------------+-----------------------+------------------------+ +| Later versions | :samp:`cpython-3{XX}` | :samp:`cpython-3{XX}t` | ++-----------------+-----------------------+------------------------+ + +That's a lot of builds, especially when multiplied by the number +of supported platforms. + +With the Stable ABI (``abi3``, introduced in CPython 3.2), a single extension +(per platform) can cover all *non-free-threaded* builds of CPython: + ++-----------------+-------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+========================+ +| 3.12 | ``abi3`` | --- | ++-----------------+ +------------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+ +------------------------+ +| 3.15 | | ``cpython-315t`` | ++-----------------+ +------------------------+ +| 3.16 | | ``cpython-316t`` | ++-----------------+ +------------------------+ +| Later versions | | :samp:`cpython-3{XX}t` | ++-----------------+-------------------+------------------------+ + +The Stable ABI for free-threaded builds (``abi3t``), introduced in +CPython 3.15, does the same for free-threaded builds. +And it's compatible with non-free-threaded ones as well: + ++-----------------+-------------------+------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+==================+ +| 3.12 | ``abi3`` * | --- | ++-----------------+ +------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+-------------------+------------------+ +| 3.15 | ``abi3t`` | ++-----------------+ + +| 3.16 | | ++-----------------+ + +| Later versions | | ++-----------------+-------------------+------------------+ + +\* (As above, the ``abi3`` extension is compatible with all non-free-threaded +builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) + +Why *not* do this +----------------- + +There are two main downsides to Stable ABI. + +First, you extension may become slower, since Stable ABI prioritizes +compatibility over performance. +The difference is usually not noticeable, and often can be mitigated by +using the same source to build both a Stable ABI build and a few +version-specific ones for "tier 1" CPython versions. + +Second, not all of the C API is available. +Extensions need to be ported to build for Stable ABI, which may be difficult +or, in rare cases, impossible. + +Specifically, ``abi3t`` requires APIs added in CPython 3.15. +If you want to build your extension for older versions of CPython from the +same source, you have two main options: + +- Use preprocessor conditionals. + + When following this guide, use ``#ifdef Py_TARGET_ABI3T`` blocks whenever + you are told to do a change that breaks the build on CPython versions you + care about. Keep the pre-existing code in ``#else`` blocks. + + For hand-written C extensions, this approach is reasonable down to + CPython 3.12, due to additions introduced in :pep:`697`. + Keeping compatibility with 3.11 and below may be worth it for code + generators (for example, Cython). + +- Do not port to ``abi3t``, and continue building separate extensions for + each version of CPython, until you can drop support for the older versions. + + This is a valid approach. Not all extensions need to switch to ``abi3t`` + right now. + + +Prerequisites +============= + +This guide assumes that you have an extension written directly in C (or C++), +which you want to port to ``abi3t``. + +If your extenstion uses a code generator (like Cython) or language binding +(like PyO3), it's best to wait until that tool has support for ``abi3t``. +If you maintain such a tool, you might be able to adapt the instructions +here for your tool. + +Non-free-threaded Stable ABI +---------------------------- + +Your extension should support the Stable ABI (``abi3t``). +If not, either port it first, or follow this guide but be prepared to fix +issues it does not mention. + +Free-threading support +---------------------- + +While it's technically not a hard prerequisite, you will most likely want to +prepare your extension for free threading before you port it to ``abi3t``. +See :ref:`freethreading-extensions-howto` for instructions. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `__: + A community-maintained porting guide for extension authors. + +Isolating extension modules +--------------------------- + +Your module should use :ref:`multi-phase initialization `, +and it should either be isolated or limit itself to be loaded at most once +per process. +If it is not your case, follow :ref:`isolating-extensions-howto` first. +(See the :ref:`opt-out section ` for a shortcut.) + +Avoiding variable-sized types +----------------------------- + +If your extension defines variable-sized types (using :c:macro:`Py_tp_itemsize` +or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to +``abi3t`` 3.15. + + +Setting up the build +==================== + +If you use a build tool (such as setuptools, meson-python, scikit-build-core), +search its documentation for a way to select ``abi3t``. +At the time of writing, not all of them have this; but if your tool does, +use it. +You may want to verify that it set the right flag by temporarily adding the +following just after ``#include ``:: + + #if Py_TARGET_ABI3T+0 <= 0x30f0000 + #error "abi3t define is not set!" + #endif + +This should result in a different error than "``abt3t`` define is not set". + +.. note:: + + If your build tool doesn't support ``abi3t`` yet, set the following macro + before including ``Python.h``:: + + #define Py_TARGET_ABI3T 0x30f0000 + + or specify it as a compiler flag, for example:: + + -DPy_TARGET_ABI3T=0x30f0000 + + Once your extension builds with this setting, it will be compatible with + CPython 3.15 and above. + + If you set this macro manually, you will later need to name and tag the + resulting extension manually as well. + This is covered in :ref:`abi3t-migration-tagging` below. + +This guide will ask you to make a series of changes. +After each one, verify that your extension still builds in the original +(non-``abi3t``) configuration, and ideally run tests on all Python +versions you support. +This will ensure that nothing breaks as you are porting. + + +Module export hook +================== + +Unless you've done this step already, your extension module defines a +:ref:`module initialization function ` +named :samp:`PyInit_{}`. +You will need to port it to a :ref:`module export hook `, +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:pep:`793`. + +Your existing init function should look like this (with your own names +for ```` and ````): + +.. code-block:: + :class: bad + + PyMODINIT_FUNC + PyInit_(void) + { + return PyModuleDef_Init(&); + } + +If there is some code before the ``return``, move it to +a :c:macro:`Py_mod_create` or :c:macro:`Py_mod_exec` slot function. +See the :ref:`PyInit documentation ` for related information. + +The function references a ``PyModuleDef`` object (```` in the code +above). +Its definition should be similar to the following, with different values +and perhaps some fields unnnamed or left out: + +.. code-block:: + :class: bad + + static PyModuleDef = { + PyModuleDef_HEAD_INIT, + .m_name = "my_module", + .m_doc = "my docstring", + .m_size = sizeof(my_state_struct), + .m_methods = my_methods, + .m_slots = my_slots, + .m_traverse = my_traverse, + .m_clear = my_clear, + .m_free = my_free, + }; + +Remove this definition and the ``PyInit`` function (or put them in +an ``#ifndef Py_TARGET_ABI3T`` block, to retain backwards compatibility), +and replace them with the following: + +.. code-block:: + :class: good + + PyABIInfo_VAR(abi_info); + + static PySlot my_slot_array[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "my_module"), + PySlot_STATIC_DATA(Py_mod_doc, "my docstring"), + PySlot_SIZE(Py_mod_state_size, sizeof(my_state_struct)), + PySlot_STATIC_DATA(Py_mod_methods, my_methods), + PySlot_STATIC_DATA(Py_mod_slots, my_slots), + PySlot_FUNC(Py_mod_state_traverse, my_traverse), + PySlot_FUNC(Py_mod_state_clear, my_clear), + PySlot_FUNC(Py_mod_state_free, my_free), + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_(void) + { + return my_slot_array; + } + +Leave out any fields that were missing (except the new :c:macro:`Py_mod_abi`), +and substitute your own values. + +See the :c:type:`PySlot` and :c:ref:`export hook ` +documentation for details on this API. + +Associated ``PyModuleDef`` +-------------------------- + +Since the new API does not use a :c:type:`!PyModuleDef` structure, a definition +will not be associated with the resulting module. +This changes the behavior of the following functions: + +- :c:func:`PyModule_GetDef` +- :c:func:`PyType_GetModuleByDef` + +Check your code for these. +If you do not use them, you can skip this section. + +These functions are typically used for two purposes: + +1. To get the definition the module was created with. + This is no longer possible using the new API. + Modules no longer keep a reference to the definition, so you will need to + figure out a different way to pass the relevant data around. + +.. _abi3t-migration-module-token: + +2. To check if a given module object is “yours”. + This use case is now served by :ref:`module tokens ` -- + opaque pointers that identify a module. + To use a token, declare (or reuse) a unique static variable, for example: + + .. code-block:: + :class: good + + static char my_token; + + and add a pointer to it in a new entry to your module's ``PySlot`` array: + + .. code-block:: + :class: good + :emphasize-lines: 3 + + static PySlot my_slot_array[] = { + ... + PySlot_STATIC_DATA(Py_mod_token, &my_token), + PySlot_END + } + + Then, switch from :c:func:`PyModule_GetDef` calls such as: + + .. code-block:: + :class: bad + + PyModuleDef *def = PyModule_GetDef(module); + + to :c:func:`PyModule_GetToken` (which uses an output argument and may fail + with an exception): + + .. code-block:: + :class: good + + void *token; + if (PyModule_GetToken(module, &token) < 0) { + /* handle error */ + } + + and from :c:func:`PyType_GetModuleByDef` calls such as: + + .. code-block:: + :class: bad + + PyObject *module = PyType_GetModuleByDef(type, my_def); + /* handle error; use module */ + + to :c:func:`PyType_GetModuleByToken` (which returns a strong reference): + + .. code-block:: + :class: good + + PyObject *module = PyType_GetModuleByToken(type, my_token); + /* handle error; use module */ + Py_XDECREF(module); + +``PyObject`` opaqueness +======================= + +The :c:type:`PyObject` and :c:type:`PyVarObject` structures are opaque +in ``abi3t``. + +Accessing their members is prohibited. +If you do this, switch to getter/setter functions mentioned in +their documentation: + +- :c:member:`PyObject.ob_type` +- :c:member:`PyObject.ob_refcnt` +- :c:member:`PyVarObject.ob_size` + +Also, the *size* of the :c:type:`PyObject` structures is +unknown to the compiler. +It can -- and *does* -- change between different CPython builds. + +.. note:: + + While the size is available at runtime (for example as + ``sys.getsizeof(object())`` in Python code), you should resist the + temptation to calculate pointer offsets from it. + The object memory layout is subject to change in future + ``abi3t`` implementations. + + +Custom type definitions +----------------------- + +Since :c:type:`!PyObject` is opaque, the traditional way of defining +custom types no longer works: + +.. code-block:: + :class: bad + + typedef struct { + PyObject_HEAD // expands to `PyObject ob_base;` which has unknown size + + int my_data; + } CustomObject; + + static PyType_Spec CustomType_spec = { + ... + .basicsize = sizeof(CustomObject), + ... + }; + +Most likely, all your class definitions, *and* all code that accesses +your classes' data, will need to be rewritten. +This will probably be the biggest change you need to support ``abi3t``. + +For each such type, instead of defining a ``struct`` for the entire instance, +define one with only the “additional” fields -- ones specific to your class, +not its superclasses: + +.. code-block:: + :class: good + + typedef struct { + int my_data; + } CustomObjectData; + +Change the name. +Almost all code that uses the struct will need to change +(notably, pointers to the new structure cannot be cast to/from ``PyObject*``), +and changing the name will highlight the usages as compiler errors. +(If you use ``typeof``, C++ ``auto``, or similar ways to avoid +typing the type name, this won't work. Be extra careful, and consider running +tools to detect undefined behavior.) + +Then, to create the class, use *negative* ``basicsize`` to indicate +“extra” storage space rather than *total* instance size: + +.. code-block:: + :class: good + + static PyType_Spec CustomType_spec = { + ... + .basicsize = -sizeof(CustomObjectData), /* note the minus sign */ + ... + }; + +If you use :c:macro:`Py_tp_members`, set the :c:macro:`Py_RELATIVE_OFFSET` +flag on each member and specify the :c:member:`~PyMemberDef.offset` +relative to your new struct. + + +Custom type data access +----------------------- + +Then comes the hard part: in all code that needs to access this struct, +you will need an additional :c:func:`PyObject_GetTypeData` call to +retrieve a ``CustomObjectData *`` pointer from ``PyObject *``: + +.. code-block:: + :class: good + + PyObject *obj = ...; + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + +Note that this call requires the *type object* for your class (``cls``). + +If your class is not subclassable (that is, it does not use the +:c:macro:`Py_TPFLAGS_BASETYPE` flag), ``cls`` will be ``Py_TYPE(obj)``. +Otherwise, **DO NOT USE** ``Py_TYPE`` with :c:func:`!PyObject_GetTypeData`: +it might return memory reserved to an unrelated subclass! +For example, if a user makes a subclass like this: + +.. code-block:: python + + class Sub(YourCustomClass): + __slots__ = ('a', 'b') + +then ``Py_TYPE(obj)`` is ``YourCustomClass``, and the underlying memory may +look like this: + +.. code-block:: text + + ╭─ PyObject *obj + │ ╭─ the pointer you want + │ │ ╭─ PyObject_GetTypeData(obj, Py_TYPE(obj)) + ▼ ▼ ▼ + ┌──────────┬───┬────────────────┬───┬─────────────┬───┬─────────────┐ + │ PyObject │...│ CustomTypeData │...│ PyObject *a │...│ PyObject *b │ + └──────────┴───┴────────────────┴───┴─────────────┴───┴─────────────┘ + +(Ellipses indicate possible padding. +Note that this memory layout is not guaranteed: future versions of Python may +add different padding or even switch the order of the structures.) + +There are two main ways to get the right class: + +- In instance methods, your implementation may use the :c:type:`PyCMethod` + signature (and the :c:macro:`METH_METHOD` bit in + :c:member:`PyMethodDef.ml_flags`), + and get the class as the ``defining_class`` argument. +- Otherwise, give your class a unique static token using the + :c:macro:`Py_tp_token` slot, and use: + + .. code-block:: + :class: good + + PyTypeObject cls; + if (PyType_GetBaseByToken(Py_TYPE(obj), my_tp_token, &cls) < 0) { + /* handle error */ + } + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + + Type tokens work similarly to module tokens covered :ref:`earlier in this + guide `. + + + +Avoid build-time conditionals +============================= + +Check your code for API that identifies the version of Python used to +*build* your extension. +This no longer corresponds to the Python your extension runs on, so code +that uses this information often needs changing. +The macros to check for are: + +- :c:macro:`PY_VERSION_HEX`, :c:macro:`PY_MAJOR_VERSION`, + :c:macro:`PY_MINOR_VERSION`: + + - to get the run-time version, use :c:data:`Py_Version`; + - to determine what C API is available, use :c:macro:`Py_TARGET_ABI3T`. + This macro is set to the minimum supported version. + +- :c:macro:`Py_GIL_DISABLED`: under ``abi3t``, this macro is always defined. + Code that works with free-threaded Python *should* also work with + the GIL enabled (since the GIL can be enabled at run time), + and usually *does* (unless it, for some reason, requires more than one + :term:`attached thread state ` at one time). + + +Further code changes +==================== + +If you are still left with compiler errors or warnings, find a way to fix them. +Alas, this guide is limited, and cannot cover all possible code +changes extensions may need. + +If you find a problem that other extension authors might run into, +consider :ref:`reporting an issue ` (or sending +a pull request) for this guide. + +It is possible your issue cannot be fixed for the current version of ``abi3t``. +In that case, reporting it may help it get prioritized for the next version +of CPython. + + +.. _abi3t-migration-tagging: + +Tagging and distribution +======================== + +If you are using a build tool with ``abi3t`` support, your extension is ready, +but you might want to check that it was built correctly. + +Extensions built with ``abi3t`` should have the following extension: + +- On Windows: ``.pyd`` (like any other extension); +- Linux, macOS, and other systems that use the ``.so`` suffix: ``.abi3t.so`` + (**not** ``.cpython-315t.so`` or ``.abi3.so``). + Note that both free-threaded and non-free-threaded builds will + load ``.abi3t.so`` extensions; +- Other systems: consult your distributor, and perhaps update this guide. + +If you distribute the extension as a *wheel*, use the following tags: + +* Python tag: :samp:`cp3{XX}`, where *XX* is the minimum Python version + the extension is built for. + (For example, ``cp315`` if you set ``Py_TARGET_ABI3T`` to ``0x30f0000``. + See :ref:`abi3-compiling` for more values.) +* ABI tag: ``abi3.abi3t``. This is a *compressed tag set* that indicates + support for both non-free-threaded and free-threaded builds. + +For example, the wheel filename may look like this: + +.. code-block:: text + + myproject-1.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl + +.. seealso:: `Platform Compatibility Tags `__ in the PyPA package distribution metadata. + +If the filename or tags are incorrect, fix them. + + +Testing +======= + +Note that when you build an extension compatible with multiple versions of +CPython, you should always *test* it with each version it supports (for +example, 3.15, 3.16, and so on). +Stable ABI only guarantees *ABI* compatibility; there may also be behavior +changes -- both intentional ones (covered by :pep:`387`) and bugs. + +Be sure to run tests on both free-threaded and non-free-threaded builds +of CPython. + +If they pass, congratulations! You have an ``abi3t`` extension. diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 380c2be04957d5e..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -165,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 552514063c95ab2..ebc7a100d91a646 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1042,7 +1042,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 81fc7e63f35bd79..57e2d6e0752447f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -37,6 +37,7 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + abi3t-migration.rst remote_debugging.rst General: @@ -61,6 +62,7 @@ Advanced development: * :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` +* :ref:`abi3t-migration-howto` * :ref:`python_2.3_mro` * :ref:`socket-howto` * :ref:`timerfd-howto` diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 47f5eabb6f2180f..13f463a1e95340e 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -162,58 +157,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/array.rst b/Doc/library/array.rst index ca7c055285aa822..da9b3fa2fe8a5de 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -134,6 +134,8 @@ The module defines the following type: :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 43977de273e61f6..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -101,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index cc833b80d52542b..64f0810777e41b9 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1230,6 +1230,9 @@ Task object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index a722607b2c1f198..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -284,19 +286,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -307,18 +318,22 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -330,10 +345,12 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -356,8 +373,11 @@ Refer to the documentation of the individual functions for more information. Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -372,8 +392,7 @@ Refer to the documentation of the individual functions for more information. .. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -392,11 +411,12 @@ Refer to the documentation of the individual functions for more information. .. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -414,8 +434,7 @@ Refer to the documentation of the individual functions for more information. .. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -499,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 8b4ba6ae9fb2549..ceb80a35a1a76bb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -133,8 +133,11 @@ The :mod:`!binascii` module defines the following functions: should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Ascii85 encoding. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -164,12 +167,16 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.15 @@ -213,8 +220,10 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. .. versionadded:: 3.15 @@ -358,9 +367,8 @@ The :mod:`!binascii` module defines the following functions: *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. .. versionchanged:: 3.15 Added the *ignorechars* parameter. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 2c29a5ec992737e..f532aa462565e40 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -200,7 +200,7 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score) + >>> def grade(score): ... i = bisect([60, 70, 80, 90], score) ... return "FDCBA"[i] ... diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 9259ab10d5850b5..059ed2c03acfa38 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1155,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1192,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 51853725b1b297c..10e3790717ed6ed 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -456,7 +456,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index e42bdc06be09fff..0c727b71cf4d4b7 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -484,6 +484,8 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: @@ -739,6 +741,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: @@ -1228,7 +1233,7 @@ variants of :func:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1243,10 +1248,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index d058ba206c6cd62..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -68,7 +68,7 @@ A small number of constants live in the built-in namespace. They are: The same as the ellipsis literal "``...``", an object frequently used to indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but assignment to ``...`` raises a :exc:`SyntaxError`. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 93d0c0d34bf039d..b0cc0be8e911bf0 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -42,6 +42,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -130,6 +133,9 @@ Context Variables Tokens support the :ref:`context manager protocol ` to automatically reset context variables. See :meth:`ContextVar.set`. + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. + .. versionadded:: 3.14 Added support for usage as a context manager. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 121c44a16ad43b9..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,9 +72,13 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 438afa04c6630d8..46b8dd8ef188149 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,14 @@ used to wrap these libraries in pure Python. .. include:: ../includes/optional-module.rst +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: @@ -198,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`!ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -3172,6 +3178,8 @@ Arrays and pointers subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 0bce3e5b762b8be..a09c28ad9791584 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -498,7 +498,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index e5afa1744135418..8b812c173b59536 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -728,18 +728,16 @@ Finally, we compare the two: >>> from pprint import pprint >>> pprint(result) - [ - ' 1. Beautiful is better than ugly.\n', - '- 2. Explicit is better than implicit.\n', - '- 3. Simple is better than complex.\n', - '+ 3. Simple is better than complex.\n', - '? ++\n', - '- 4. Complex is better than complicated.\n', - '? ^ ---- ^\n', - '+ 4. Complicated is better than complex.\n', - '? ++++ ^ ^\n', - '+ 5. Flat is better than nested.\n', - ] + [' 1. Beautiful is better than ugly.\n', + '- 2. Explicit is better than implicit.\n', + '- 3. Simple is better than complex.\n', + '+ 3. Simple is better than complex.\n', + '? ++\n', + '- 4. Complex is better than complicated.\n', + '? ^ ---- ^\n', + '+ 4. Complicated is better than complex.\n', + '? ++++ ^ ^\n', + '+ 5. Flat is better than nested.\n'] As a single multi-line string it looks like this: diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index c6924a0ac29c972..619c17c98e8d89c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -93,9 +93,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7fc6055aa9a8812..3775d5ac81a2736 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -984,6 +984,9 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + .. impl-detail:: The ``excs`` parameter may be any sequence, but lists and tuples are diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 1fed142d81b4f72..def2a211d1b3b4d 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,24 +19,25 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`sentinel` | -| | :func:`bool` | | | | | | :func:`slice` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`sorted` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | :func:`staticmethod` | -| | |func-bytes|_ | | :func:`globals` | | | | |func-str|_ | -| | | | | | **O** | | :func:`sum` | -| | **C** | | **H** | | :func:`object` | | :func:`super` | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | :func:`bin` | | |func-frozendict|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | |func-frozenset|_ | | | | :func:`slice` | +| | :func:`breakpoint` | | | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | **G** | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`getattr` | | | | |func-str|_ | +| | | | :func:`globals` | | **O** | | :func:`sum` | +| | **C** | | | | :func:`object` | | :func:`super` | +| | :func:`callable` | | **H** | | :func:`oct` | | | +| | :func:`chr` | | :func:`hasattr` | | :func:`open` | | **T** | +| | :func:`classmethod` | | :func:`hash` | | :func:`ord` | | |func-tuple|_ | +| | :func:`compile` | | :func:`help` | | | | :func:`type` | +| | :func:`complex` | | :func:`hex` | | **P** | | | +| | | | | | :func:`pow` | | **V** | +| | **D** | | **I** | | :func:`print` | | :func:`vars` | +| | :func:`delattr` | | :func:`id` | | :func:`property` | | | +| | |func-dict|_ | | :func:`input` | | | | **Z** | +| | :func:`dir` | | :func:`int` | | | | :func:`zip` | +| | :func:`divmod` | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -44,6 +45,7 @@ are always available. They are listed here in alphabetical order. used, with replacement texts to make the output in the table consistent .. |func-dict| replace:: ``dict()`` +.. |func-frozendict| replace:: ``frozendict()`` .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` @@ -485,8 +487,8 @@ are always available. They are listed here in alphabetical order. Create a new dictionary. The :class:`dict` object is the dictionary class. See :class:`dict` and :ref:`typesmapping` for documentation about this class. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built-in :class:`frozendict`, :class:`list`, + :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() @@ -864,6 +866,21 @@ are always available. They are listed here in alphabetical order. if *format_spec* is not an empty string. +.. _func-frozendict: +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + :noindex: + + Create a new frozen dictionary. The :class:`frozendict` object is a built-in class. + See :class:`frozendict` and :ref:`typesmapping` for documentation about this class. + + For other containers see the built-in :class:`dict`, :class:`list`, :class:`set`, + and :class:`tuple` classes, as well as the :mod:`collections` module. + + .. versionadded:: 3.15 + + .. _func-frozenset: .. class:: frozenset(iterable=(), /) :noindex: @@ -1827,15 +1844,21 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: sentinel(name, /) +.. class:: sentinel(name, /, *, repr=None) Return a new unique sentinel object. *name* must be a :class:`str`, and is - used as the returned object's representation:: + used by default as the returned object's representation:: >>> MISSING = sentinel("MISSING") >>> MISSING MISSING + The optional *repr* argument can be used to specify a different representation:: + + >>> MISSING = sentinel("MISSING", repr="") + >>> MISSING + + Sentinel objects are truthy and compare equal only to themselves. They are intended to be compared with the :keyword:`is` operator. @@ -1879,7 +1902,7 @@ are always available. They are listed here in alphabetical order. .. attribute:: __module__ - The name of the module where the sentinel was created. + The name of the module where the sentinel was created. This attribute is writable. .. versionadded:: 3.15 diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 2de16fe40362b3e..2ab7ba7df19cf14 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -51,19 +51,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -78,15 +78,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -223,20 +223,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -244,7 +244,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -322,7 +322,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -331,7 +331,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -352,7 +352,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -363,7 +363,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ed9fdaf1d727b08..2c667ddc522399c 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -108,9 +108,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 63de4f91f4ba5f5..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -105,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -224,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -252,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -465,6 +481,9 @@ The same applies for :func:`entry_points` and :func:`files`. .. attribute:: metadata :type: PackageMetadata + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. + There are all kinds of additional metadata available on :class:`!Distribution` instances as a :class:`PackageMetadata` instance:: diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 653fa61420be869..72db66f9f06f890 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -240,7 +240,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 8713765b8aebfbd..a0f7379b12a8a62 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -416,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -434,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -1572,10 +1614,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 494e57fe1c04743..d47b74efe22de9d 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,6 +38,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -65,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -103,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -478,8 +487,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -493,13 +505,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -513,6 +531,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -641,7 +666,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -661,10 +690,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -676,6 +701,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -824,9 +856,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b354e7ba534835f..383ccad9df041b5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -211,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9cc8c5d6886324c..41a9a0ab55d8fab 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -255,10 +255,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 0facacd50fd389e..c2ccabe3cc53892 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -348,7 +348,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -375,7 +375,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -397,7 +397,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 187143d02cd7bfb..2d13053915830b0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -100,10 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -The arguments to :class:`Process` usually need to be unpickleable from within -the child process. If you tried typing the above example directly into a REPL it -could lead to an :exc:`AttributeError` in the child process trying to locate the -*f* function in the ``__main__`` module. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -1723,7 +1723,10 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. The :mod:`!multiprocessing.sharedctypes` module diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index c0dab83977e427f..3d1c8cda13be381 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -110,7 +110,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -134,13 +134,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -152,7 +152,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -172,25 +172,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -209,7 +209,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -403,13 +403,18 @@ Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d2534b3e974f368..b65dbb4623af2a8 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -219,6 +219,14 @@ process and user. :data:`os.environ`, and when one of the :meth:`~dict.pop` or :meth:`~dict.clear` methods is called. + If the :manpage:`clearenv(3)` function is available, the :meth:`~dict.clear` method + uses it and emits a single ``os._clearenv`` audit event. Otherwise, it emits + an ``os.unsetenv`` event on each deleted variable. + + .. audit-event:: os.unsetenv key os.unsetenv + + .. audit-event:: os._clearenv "" os._clearenv + .. seealso:: The :func:`os.reload_environ` function. @@ -226,6 +234,10 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.15 + The :meth:`~dict.clear` method can now emit an ``os._clearenv`` audit + event. + .. data:: environb @@ -2549,7 +2561,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2567,6 +2580,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2593,6 +2612,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -2970,6 +2994,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2867015042ee162..45b5797058f6239 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1514,7 +1514,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1525,6 +1526,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1538,6 +1545,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f8975c2f4281d45..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -56,7 +56,7 @@ The :mod:`!pickle` module differs from :mod:`marshal` in several significant way * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -693,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index d62ef1f4d1e6b17..4f043fbb3a46dff 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -17,7 +17,7 @@ objects which are not representable as Python literals. The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width, -adjustable by the *width* parameter defaulting to 88 characters. +adjustable by the *width* parameter defaulting to 80 characters. .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. @@ -30,8 +30,9 @@ adjustable by the *width* parameter defaulting to 88 characters. Functions --------- -.. function:: pp(object, stream=None, indent=4, width=88, depth=None, *, \ - compact=False, sort_dicts=False, underscore_numbers=False) +.. function:: pp(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, expand=False, sort_dicts=False, \ + underscore_numbers=False) Prints the formatted representation of *object*, followed by a newline. This function may be used in the interactive interpreter @@ -66,11 +67,16 @@ Functions :param bool compact: Control the way long :term:`sequences ` are formatted. If ``False`` (the default), + each item of a sequence will be formatted on a separate line, + otherwise as many items as will fit within the *width* + will be formatted on each output line. + Incompatible with *expand*. + + :param bool expand: + If ``True``, opening parentheses and brackets will be followed by a newline and the following content will be indented by one level, similar to - pretty-printed JSON. - If ``True``, as many items as will fit within the *width* - will be formatted on each output line. + pretty-printed JSON. Incompatible with *compact*. :param bool sort_dicts: If ``True``, dictionaries will be formatted with @@ -85,25 +91,32 @@ Functions >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] >>> stuff.insert(0, stuff) - >>> pprint.pp(stuff, width=100) - [, 'spam', 'eggs', 'lumberjack', 'knights', 'ni'] + >>> pprint.pp(stuff) + [, + 'spam', + 'eggs', + 'lumberjack', + 'knights', + 'ni'] .. versionadded:: 3.8 -.. function:: pprint(object, stream=None, indent=4, width=88, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) +.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Alias for :func:`~pprint.pp` with *sort_dicts* set to ``True`` by default, which would automatically sort the dictionaries' keys, you might want to use :func:`~pprint.pp` instead where it is ``False`` by default. -.. function:: pformat(object, indent=4, width=88, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) +.. function:: pformat(object, indent=1, width=80, depth=None, *, \ + compact=False, expand=False, sort_dicts=True, \ + underscore_numbers=False) Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are + *width*, *depth*, *compact*, *expand*, *sort_dicts* and *underscore_numbers* are passed to the :class:`PrettyPrinter` constructor as formatting parameters and their meanings are as described in the documentation above. @@ -141,13 +154,13 @@ Functions .. _prettyprinter-objects: -PrettyPrinter objects +PrettyPrinter Objects --------------------- .. index:: single: ...; placeholder -.. class:: PrettyPrinter(indent=4, width=88, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, \ +.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ + compact=False, expand=False, sort_dicts=True, \ underscore_numbers=False) Construct a :class:`PrettyPrinter` instance. @@ -158,23 +171,21 @@ PrettyPrinter objects >>> import pprint >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter() + >>> pp = pprint.PrettyPrinter(indent=4) >>> pp.pprint(stuff) - [ - ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], + [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], 'spam', 'eggs', 'lumberjack', 'knights', - 'ni', - ] - >>> pp = pprint.PrettyPrinter(indent=1, width=41, compact=True) + 'ni'] + >>> pp = pprint.PrettyPrinter(width=41, compact=True) >>> pp.pprint(stuff) [['spam', 'eggs', 'lumberjack', 'knights', 'ni'], 'spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, indent=3) + >>> pp = pprint.PrettyPrinter(width=41, expand=True, indent=3) >>> pp.pprint(stuff) [ [ @@ -210,11 +221,7 @@ PrettyPrinter objects No longer attempts to write to :data:`!sys.stdout` if it is ``None``. .. versionchanged:: 3.15 - Changed default *indent* from 1 to 4 - and default *width* from 80 to 88. - The default ``compact=False`` layout is now similar to - pretty-printed JSON, with opening parentheses and brackets - followed by a newline and the contents indented by one level. + Added the *expand* parameter. :class:`PrettyPrinter` instances have the following methods: @@ -291,144 +298,219 @@ let's fetch information about a project from `PyPI `_:: In its basic form, :func:`~pprint.pp` shows the whole object:: >>> pprint.pp(project_info) - { - 'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Software Development :: Build Tools', - ], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of the project, basic\n' - 'usage examples, etc. Generally, including the project changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {'Download': 'UNKNOWN', 'Homepage': 'https://github.com/pypa/sampleproject'}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0', - } + {'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': ['Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Software Development :: Build Tools'], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ' + 'ReStructured Text. It\n' + 'will be used to generate the project webpage on PyPI, and ' + 'should be written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of ' + 'the project, basic\n' + 'usage examples, etc. Generally, including the project ' + 'changelog in here is not\n' + 'a good idea, although a simple "What\'s New" section for the ' + 'most recent version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': {'Download': 'UNKNOWN', + 'Homepage': 'https://github.com/pypa/sampleproject'}, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0'} The result can be limited to a certain *depth* (ellipsis is used for deeper contents):: >>> pprint.pp(project_info, depth=1) - { - 'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of the project, basic\n' - 'usage examples, etc. Generally, including the project changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0', - } + {'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [...], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ' + 'ReStructured Text. It\n' + 'will be used to generate the project webpage on PyPI, and ' + 'should be written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of ' + 'the project, basic\n' + 'usage examples, etc. Generally, including the project ' + 'changelog in here is not\n' + 'a good idea, although a simple "What\'s New" section for the ' + 'most recent version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {...}, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': {...}, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0'} Additionally, maximum character *width* can be suggested. If a long object cannot be split, the specified width will be exceeded:: >>> pprint.pp(project_info, depth=1, width=60) + {'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [...], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the ' + 'project.\n' + '\n' + 'The file should use UTF-8 encoding and be ' + 'written using ReStructured Text. It\n' + 'will be used to generate the project ' + 'webpage on PyPI, and should be written ' + 'for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would ' + 'include an overview of the project, ' + 'basic\n' + 'usage examples, etc. Generally, including ' + 'the project changelog in here is not\n' + 'a good idea, although a simple "What\'s ' + 'New" section for the most recent version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {...}, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': {...}, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0'} + +Lastly, we can format like pretty-printed JSON with the *expand* parameter. +Best results are achieved with a higher *indent* value:: + + >>> pprint.pp(project_info, indent=4, expand=True) { - 'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written ' - 'using ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, ' - 'and should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an ' - 'overview of the project, basic\n' - 'usage examples, etc. Generally, including the project ' - 'changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section ' - 'for the most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0', + 'author': 'The Python Packaging Authority', + 'author_email': 'pypa-dev@googlegroups.com', + 'bugtrack_url': None, + 'classifiers': [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Topic :: Software Development :: Build Tools', + ], + 'description': 'A sample Python project\n' + '=======================\n' + '\n' + 'This is the description file for the project.\n' + '\n' + 'The file should use UTF-8 encoding and be written using ReStructured ' + 'Text. It\n' + 'will be used to generate the project webpage on PyPI, and should be ' + 'written for\n' + 'that purpose.\n' + '\n' + 'Typical contents for this file would include an overview of the project, ' + 'basic\n' + 'usage examples, etc. Generally, including the project changelog in here ' + 'is not\n' + 'a good idea, although a simple "What\'s New" section for the most recent ' + 'version\n' + 'may be appropriate.', + 'description_content_type': None, + 'docs_url': None, + 'download_url': 'UNKNOWN', + 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, + 'dynamic': None, + 'home_page': 'https://github.com/pypa/sampleproject', + 'keywords': 'sample setuptools development', + 'license': 'MIT', + 'license_expression': None, + 'license_files': None, + 'maintainer': None, + 'maintainer_email': None, + 'name': 'sampleproject', + 'package_url': 'https://pypi.org/project/sampleproject/', + 'platform': 'UNKNOWN', + 'project_url': 'https://pypi.org/project/sampleproject/', + 'project_urls': { + 'Download': 'UNKNOWN', + 'Homepage': 'https://github.com/pypa/sampleproject', + }, + 'provides_extra': None, + 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', + 'requires_dist': None, + 'requires_python': None, + 'summary': 'A sample Python project', + 'version': '1.2.0', + 'yanked': False, + 'yanked_reason': None, } diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index f236eba84576575..a0cfb440a36ffa9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -68,6 +68,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 5ac72ef7604d50c..f5326aff7236bd6 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -76,6 +76,8 @@ The :mod:`!queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 diff --git a/Doc/library/re.rst b/Doc/library/re.rst index a46fd42458158cf..4745c1b98a45543 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1257,6 +1257,9 @@ Regular expression objects Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1419,6 +1422,9 @@ when there is no match, you can test whether there was a match with a simple Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 70541c5f3cb3676..037e27f031d0c82 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -117,9 +117,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 09563af14d018a4..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -94,18 +96,22 @@ The module defines the following: When CPython is built, this function may be disabled using :option:`--disable-epoll`. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -113,12 +119,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -190,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -285,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -362,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -396,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -421,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -489,7 +499,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -533,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -553,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -620,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -640,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -663,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d289ba58c240658..6a734966d1e0a45 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -696,7 +696,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -749,8 +749,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 656fbd142dfb0fc..11a5484c2b13362 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -356,7 +356,79 @@ Module contents This function used to be called unconditionally. -.. function:: addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) +.. function:: makepath(*paths) + + Join *paths* with :func:`os.path.join`, attempt to make the result + absolute with :func:`os.path.abspath`, and return a 2-tuple containing + the absolute path and its case-normalized form as produced by + :func:`os.path.normcase`. If :func:`os.path.abspath` raises + :exc:`OSError`, the joined path is used unchanged for the + case-normalization step. + + The second element of the returned tuple is the form used throughout the + :mod:`!site` module to compare paths on case-insensitive file systems, and + is what populates the ``known_paths`` sets that prevent duplicate + :data:`sys.path` entries in various APIs within this module. + + +.. class:: StartupState(known_paths=None) + + Instances of this class accumulate interpreter startup configuration data + from one or more site directories. They are the preferred interface for + batching the processing of :file:`.pth` and :file:`.start` files across + multiple site directories, so that every :data:`sys.path` extension is + visible before any startup code runs. + + The optional *known_paths* argument is a set of case-normalized paths + (which can be produced by :func:`makepath`) used to prevent duplicate + :data:`sys.path` entries. When ``None`` (the default), the set is built + from the current :data:`sys.path`. :func:`main` implicitly uses an + instance of this class. + + Typical use: + + .. code-block:: python + + state = site.StartupState() + for sitedir in site_dirs: + state.addsitedir(sitedir) + state.process() + + .. versionadded:: 3.15 + + .. method:: addsitedir(sitedir) + + Read the :file:`.pth` and :file:`.start` files in *sitedir* and + record their :data:`sys.path` extensions, deprecated :file:`.pth` + ``import`` lines, and :file:`.start` entry points on this state. + The recorded data is not applied until :meth:`process` is called. + + .. method:: addusersitepackages() + + Add the per-user site-packages directory, if enabled and if it exists. + The directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: addsitepackages(prefixes=None) + + Add global site-packages directories, computed from *prefixes* or from + the global :data:`PREFIXES` when *prefixes* is ``None``. Each + directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: process() + + Apply the accumulated state by first adding the path extensions to + :data:`sys.path`, then executing the :file:`.start` file entry points + and :file:`.pth` file ``import`` lines (:ref:`deprecated + `). + + This method is not idempotent and must not be called more than once + on the same instance. Doing so will apply the accumulated state + more than once, re-running entry points and ``import`` lines. + + +.. function:: addsitedir(sitedir, known_paths=None) Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` files found in that directory. Typically used in :mod:`sitecustomize` or @@ -366,17 +438,15 @@ Module contents used to prevent duplicate :data:`sys.path` entries. When ``None`` (the default), the set is built from the current :data:`sys.path`. - While :file:`.pth` and :file:`.start` files are always parsed, set - *defer_processing_start_files* to ``True`` to prevent processing the - startup data found in those files, so that you can process them explicitly - (this is typically used by the :func:`main` function). + For batched processing across multiple site directories, build a + :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir` + on it; that defers :file:`.pth` and :file:`.start` processing until a + single :meth:`StartupState.process` call, ensuring every :data:`sys.path` + extension is visible before any startup code runs. .. versionchanged:: 3.15 Also processes :file:`.start` files. See :ref:`site-start-files`. - All :file:`.pth` and :file:`.start` files are now read and - accumulated before any path extensions, ``import`` line execution, - or entry point invocations take place. .. function:: getsitepackages() @@ -447,4 +517,3 @@ value greater than 2 if there is an error. * :pep:`370` -- Per user site-packages directory * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 484260e63dd5f2f..36f080b56ffea7c 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1417,6 +1417,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.15 + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1426,6 +1429,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: 3.15 + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1709,6 +1715,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.15 + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index d9c736d27dcaecc..41a101e84ac4d75 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1121,7 +1121,7 @@ SSL sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -2076,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2473,79 +2473,67 @@ Visual inspection shows that the certificate does identify the desired service (that is, the HTTPS host ``www.python.org``):: >>> pprint.pprint(cert) - { - 'OCSP': ('http://ocsp.digicert.com',), - 'caIssuers': ('http://cacerts.digicert.com/DigiCertSHA2ExtendedValidationServerCA.crt',), - 'crlDistributionPoints': ( - 'http://crl3.digicert.com/sha2-ev-server-g1.crl', - 'http://crl4.digicert.com/sha2-ev-server-g1.crl', - ), - 'issuer': ( - (('countryName', 'US'),), - (('organizationName', 'DigiCert Inc'),), - (('organizationalUnitName', 'www.digicert.com'),), - (('commonName', 'DigiCert SHA2 Extended Validation Server CA'),), - ), - 'notAfter': 'Sep 9 12:00:00 2016 GMT', - 'notBefore': 'Sep 5 00:00:00 2014 GMT', - 'serialNumber': '01BB6F00122B177F36CAB49CEA8B6B26', - 'subject': ( - (('businessCategory', 'Private Organization'),), - (('1.3.6.1.4.1.311.60.2.1.3', 'US'),), - (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), - (('serialNumber', '3359300'),), - (('streetAddress', '16 Allen Rd'),), - (('postalCode', '03894-4801'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'NH'),), - (('localityName', 'Wolfeboro'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'www.python.org'),), - ), - 'subjectAltName': ( - ('DNS', 'www.python.org'), - ('DNS', 'python.org'), - ('DNS', 'pypi.org'), - ('DNS', 'docs.python.org'), - ('DNS', 'testpypi.org'), - ('DNS', 'bugs.python.org'), - ('DNS', 'wiki.python.org'), - ('DNS', 'hg.python.org'), - ('DNS', 'mail.python.org'), - ('DNS', 'packaging.python.org'), - ('DNS', 'pythonhosted.org'), - ('DNS', 'www.pythonhosted.org'), - ('DNS', 'test.pythonhosted.org'), - ('DNS', 'us.pycon.org'), - ('DNS', 'id.python.org'), - ), - 'version': 3, - } + {'OCSP': ('http://ocsp.digicert.com',), + 'caIssuers': ('http://cacerts.digicert.com/DigiCertSHA2ExtendedValidationServerCA.crt',), + 'crlDistributionPoints': ('http://crl3.digicert.com/sha2-ev-server-g1.crl', + 'http://crl4.digicert.com/sha2-ev-server-g1.crl'), + 'issuer': ((('countryName', 'US'),), + (('organizationName', 'DigiCert Inc'),), + (('organizationalUnitName', 'www.digicert.com'),), + (('commonName', 'DigiCert SHA2 Extended Validation Server CA'),)), + 'notAfter': 'Sep 9 12:00:00 2016 GMT', + 'notBefore': 'Sep 5 00:00:00 2014 GMT', + 'serialNumber': '01BB6F00122B177F36CAB49CEA8B6B26', + 'subject': ((('businessCategory', 'Private Organization'),), + (('1.3.6.1.4.1.311.60.2.1.3', 'US'),), + (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), + (('serialNumber', '3359300'),), + (('streetAddress', '16 Allen Rd'),), + (('postalCode', '03894-4801'),), + (('countryName', 'US'),), + (('stateOrProvinceName', 'NH'),), + (('localityName', 'Wolfeboro'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'www.python.org'),)), + 'subjectAltName': (('DNS', 'www.python.org'), + ('DNS', 'python.org'), + ('DNS', 'pypi.org'), + ('DNS', 'docs.python.org'), + ('DNS', 'testpypi.org'), + ('DNS', 'bugs.python.org'), + ('DNS', 'wiki.python.org'), + ('DNS', 'hg.python.org'), + ('DNS', 'mail.python.org'), + ('DNS', 'packaging.python.org'), + ('DNS', 'pythonhosted.org'), + ('DNS', 'www.pythonhosted.org'), + ('DNS', 'test.pythonhosted.org'), + ('DNS', 'us.pycon.org'), + ('DNS', 'id.python.org')), + 'version': 3} Now the SSL channel is established and the certificate verified, you can proceed to talk with the server:: >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: linuxfr.org\r\n\r\n") >>> pprint.pprint(conn.recv(1024).split(b"\r\n")) - [ - b'HTTP/1.1 200 OK', - b'Date: Sat, 18 Oct 2014 18:27:20 GMT', - b'Server: nginx', - b'Content-Type: text/html; charset=utf-8', - b'X-Frame-Options: SAMEORIGIN', - b'Content-Length: 45679', - b'Accept-Ranges: bytes', - b'Via: 1.1 varnish', - b'Age: 2188', - b'X-Served-By: cache-lcy1134-LCY', - b'X-Cache: HIT', - b'X-Cache-Hits: 11', - b'Vary: Cookie', - b'Strict-Transport-Security: max-age=63072000; includeSubDomains', - b'Connection: close', - b'', - b'', - ] + [b'HTTP/1.1 200 OK', + b'Date: Sat, 18 Oct 2014 18:27:20 GMT', + b'Server: nginx', + b'Content-Type: text/html; charset=utf-8', + b'X-Frame-Options: SAMEORIGIN', + b'Content-Length: 45679', + b'Accept-Ranges: bytes', + b'Via: 1.1 varnish', + b'Age: 2188', + b'X-Served-By: cache-lcy1134-LCY', + b'X-Cache: HIT', + b'X-Cache-Hits: 11', + b'Vary: Cookie', + b'Strict-Transport-Security: max-age=63072000; includeSubDomains', + b'Connection: close', + b'', + b''] See the discussion of :ref:`ssl-security` below. @@ -2891,11 +2879,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3d943566be34ff1..9ad4b27cf2fc879 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1403,6 +1403,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1494,6 +1496,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -2168,9 +2174,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2211,15 +2233,14 @@ expression support in the :mod:`re` module). >>> '0123456789'.isnumeric() True - >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine True >>> '⅕'.isnumeric() # Vulgar fraction one fifth True >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() (False, True, True) - See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are - a superset of decimal numbers. + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2610,7 +2631,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2628,7 +2651,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2640,7 +2665,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2649,7 +2676,7 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] - See also :meth:`join`. + See also :meth:`join` and :meth:`rsplit`. .. index:: @@ -2747,6 +2774,8 @@ expression support in the :mod:`re` module). The *chars* argument is not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by :meth:`str.isspace`. + For example: .. doctest:: @@ -4584,6 +4613,9 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -5277,6 +5309,8 @@ Note, the *elem* argument to the :meth:`~object.__contains__`, :meth:`~set.discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. +Sets and frozensets are :ref:`generic ` over the type of their elements. + .. seealso:: For detailed information on thread-safety guarantees for :class:`set` @@ -5380,6 +5414,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5717,6 +5754,9 @@ Frozen dictionaries :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly from ``object``. + Like dictionaries, frozendicts are :ref:`generic ` over two types, + signifying (respectively) the types of the frozendict's keys and values. + .. versionadded:: 3.15 @@ -5858,7 +5898,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5907,6 +5948,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 08ccdfa3f454f8d..be968a3c53d8430 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -603,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index a5b2d796aaf4b83..6e91850fdf59ca4 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -245,6 +245,8 @@ Types ... 3.0 | 1. + 2. | None | .2f + Interpolations are :ref:`generic ` over the types of their values. + .. rubric:: Attributes .. attribute:: value diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 6946eb6eeaa5fae..b4f410a020f86e1 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -919,8 +919,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports are potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) See also :func:`set_lazy_imports` and :pep:`810`. @@ -1483,6 +1481,21 @@ always available. Unless explicitly noted otherwise, all variables are read-only They hold the legacy representation of ``sys.last_exc``, as returned from :func:`exc_info` above. + +.. data:: lazy_modules + + A :class:`set` of fully qualified module name strings that have been lazily + imported in the current interpreter but not yet loaded. When a + lazily imported module is accessed for the first time, its name is removed + from this set. + + This attribute is intended for debugging and introspection. + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + .. data:: maxsize An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can @@ -1757,8 +1770,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports become potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) This function is intended for advanced users who need to control lazy imports across their entire application. Library developers should diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 6f1e01cf5aa6ee9..9b9783d8e58013f 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -144,7 +144,9 @@ Some facts and figures: For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the keyword argument *mtime* to create a gzip archive header with that mtime. By - default, the mtime is set to the time of creation of the archive. + default, the mtime is set to the time of creation of the archive. Use + *mtime* ``0`` to generate a compressed stream that does not depend on + creation time, for reproducible output. For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index eca3e76d84a1cfd..5d9a7b6314b1668 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -515,7 +515,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index b5464ac55ddfa92..aa48cea357cfd34 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -147,9 +147,7 @@ Module-Level Functions :ref:`traceback object ` *tb*. It is useful for alternate formatting of stack traces. The optional *limit* argument has the same meaning as for :func:`print_tb`. A "pre-processed" stack trace - entry is a :class:`FrameSummary` object containing attributes - :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, - :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the + entry is a :class:`FrameSummary` object with attributes representing the information that is usually printed for a stack trace. @@ -181,7 +179,7 @@ Module-Level Functions .. function:: format_exception_only(exc, /[, value], *, show_group=False) Format the exception part of a traceback using an exception value such as - given by :data:`sys.last_value`. The return value is a list of strings, each + given by :data:`sys.last_exc`. The return value is a list of strings, each ending in a newline. The list contains the exception's message, which is normally a single string; however, for :exc:`SyntaxError` exceptions, it contains several lines that (when printed) display detailed information @@ -347,7 +345,7 @@ the module-level functions described above. .. attribute:: exc_type - The class of the original traceback. + The class of the original exception. .. deprecated:: 3.13 @@ -391,7 +389,7 @@ the module-level functions described above. For syntax errors - the compiler error message. - .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False) + .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) Capture an exception for later rendering. *limit*, *lookup_lines* and *capture_locals* are as for the :class:`StackSummary` class. diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 74898baa521bd69..38a77119769d724 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -143,15 +143,15 @@ If you instantiate any of these types, note that signatures may vary between Pyt Standard names are defined for the following types: -.. data:: NoneType +.. class:: NoneType The type of :data:`None`. .. versionadded:: 3.10 -.. data:: FunctionType - LambdaType +.. class:: FunctionType + LambdaType The type of user-defined functions and functions created by :keyword:`lambda` expressions. @@ -162,13 +162,13 @@ Standard names are defined for the following types: and is not raised for normal compilation. -.. data:: GeneratorType +.. class:: GeneratorType The type of :term:`generator`-iterator objects, created by generator functions. -.. data:: CoroutineType +.. class:: CoroutineType The type of :term:`coroutine` objects, created by :keyword:`async def` functions. @@ -176,7 +176,7 @@ Standard names are defined for the following types: .. versionadded:: 3.5 -.. data:: AsyncGeneratorType +.. class:: AsyncGeneratorType The type of :term:`asynchronous generator`-iterator objects, created by asynchronous generator functions. @@ -196,7 +196,7 @@ Standard names are defined for the following types: required by the initializer. The audit event only occurs for direct instantiation of code objects, and is not raised for normal compilation. -.. data:: CellType +.. class:: CellType The type for cell objects: such objects are used as containers for a function's :term:`closure variables `. @@ -204,20 +204,20 @@ Standard names are defined for the following types: .. versionadded:: 3.8 -.. data:: MethodType +.. class:: MethodType The type of methods of user-defined class instances. -.. data:: BuiltinFunctionType - BuiltinMethodType +.. class:: BuiltinFunctionType + BuiltinMethodType The type of built-in functions like :func:`len` or :func:`sys.exit`, and methods of built-in classes. (Here, the term "built-in" means "written in C".) -.. data:: WrapperDescriptorType +.. class:: WrapperDescriptorType The type of methods of some built-in data types and base classes such as :meth:`object.__init__` or :meth:`object.__lt__`. @@ -225,7 +225,7 @@ Standard names are defined for the following types: .. versionadded:: 3.7 -.. data:: MethodWrapperType +.. class:: MethodWrapperType The type of *bound* methods of some built-in data types and base classes. For example it is the type of :code:`object().__str__`. @@ -233,21 +233,21 @@ Standard names are defined for the following types: .. versionadded:: 3.7 -.. data:: NotImplementedType +.. class:: NotImplementedType The type of :data:`NotImplemented`. .. versionadded:: 3.10 -.. data:: MethodDescriptorType +.. class:: MethodDescriptorType The type of methods of some built-in data types such as :meth:`str.join`. .. versionadded:: 3.7 -.. data:: ClassMethodDescriptorType +.. class:: ClassMethodDescriptorType The type of *unbound* class methods of some built-in data types such as ``dict.__dict__['fromkeys']``. @@ -273,7 +273,7 @@ Standard names are defined for the following types: creating :class:`!ModuleType` instances which ensures the various attributes are set appropriately. -.. data:: EllipsisType +.. class:: EllipsisType The type of :data:`Ellipsis`. @@ -327,13 +327,13 @@ Standard names are defined for the following types: dynamically. -.. data:: FrameType +.. class:: FrameType The type of :ref:`frame objects ` such as found in :attr:`tb.tb_frame ` if ``tb`` is a traceback object. -.. data:: FrameLocalsProxyType +.. class:: FrameLocalsProxyType The type of frame locals proxy objects, as found on the :attr:`frame.f_locals` attribute. @@ -343,7 +343,7 @@ Standard names are defined for the following types: .. seealso:: :pep:`667` -.. data:: LazyImportType +.. class:: LazyImportType The type of lazy import proxy objects. These objects are created when a module is lazily imported and serve as placeholders until the module is @@ -355,7 +355,7 @@ Standard names are defined for the following types: .. seealso:: :pep:`810` -.. data:: GetSetDescriptorType +.. class:: GetSetDescriptorType The type of objects defined in extension modules with ``PyGetSetDef``, such as :attr:`FrameType.f_locals ` or ``array.array.typecode``. @@ -364,7 +364,7 @@ Standard names are defined for the following types: :class:`property` type, but for classes defined in extension modules. -.. data:: MemberDescriptorType +.. class:: MemberDescriptorType The type of objects defined in extension modules with ``PyMemberDef``, such as ``datetime.timedelta.days``. This type is used as descriptor for simple C @@ -386,6 +386,10 @@ Standard names are defined for the following types: entries, which means that when the mapping changes, the view reflects these changes. + :class:`!MappingProxyType`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the underlying mapping's keys and + values. + .. versionadded:: 3.3 .. versionchanged:: 3.9 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index dca51b8014da5a4..ef6016d45c1f8bc 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -719,8 +719,8 @@ The :data:`Any` type ==================== A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. +every type as assignable to :data:`Any` and :data:`Any` as assignable to +every type. This means that it is possible to perform any operation or method call on a value of type :data:`Any` and assign it to any variable:: @@ -785,7 +785,7 @@ it as a return value) of a more specialized type is a type error. For example:: hash_a(42) hash_a("foo") - # Passes type checking, since Any is compatible with all types + # Passes type checking, since Any is assignable to all types hash_b(42) hash_b("foo") @@ -851,8 +851,8 @@ using ``[]``. Special type indicating an unconstrained type. - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. + * Every type is assignable to :data:`Any`. + * :data:`Any` is assignable to every type. .. versionchanged:: 3.11 :data:`Any` can now be used as a base class. This can be useful for @@ -1292,10 +1292,10 @@ These can be used as types in annotations. They all support subscription using :data:`ClassVar` accepts only types and cannot be further subscribed. - :data:`ClassVar` is not a class itself, and should not + :data:`ClassVar` is not a class itself, and cannot be used with :func:`isinstance` or :func:`issubclass`. :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker + it can be used by static type checkers. For example, a type checker might flag the following code as an error:: enterprise_d = Starship(3000) @@ -1365,7 +1365,7 @@ These can be used as types in annotations. They all support subscription using def mutate_movie(m: Movie) -> None: m["year"] = 1999 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. @@ -2292,8 +2292,8 @@ without the dedicated syntax, as documented below. * :data:`Concatenate` * :ref:`annotating-callables` -.. data:: ParamSpecArgs - ParamSpecKwargs +.. class:: ParamSpecArgs + ParamSpecKwargs Arguments and keyword arguments attributes of a :class:`ParamSpec`. The ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, @@ -2472,9 +2472,9 @@ types. Fields with a default value must come after any fields without a default. - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the + The types for each field name can be retrieved by calling + :func:`annotationlib.get_annotations` on the resulting class. (The field + names are in the ``_fields`` attribute and the default values are in the ``_field_defaults`` attribute, both of which are part of the :func:`~collections.namedtuple` API.) @@ -2535,7 +2535,7 @@ types. Helper class to create low-overhead :ref:`distinct types `. - A ``NewType`` is considered a distinct type by a typechecker. At runtime, + A ``NewType`` is considered a distinct type by a type checker. At runtime, however, calling a ``NewType`` returns its argument unchanged. Usage:: @@ -2616,7 +2616,7 @@ types. Mark a protocol class as a runtime protocol. Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This allows a simple-minded structural check, very similar to "one trick ponies" + This allows a simple-minded structural check, very similar to "one-trick ponies" in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: @runtime_checkable @@ -2855,7 +2855,7 @@ types. key: T group: list[T] - A ``TypedDict`` can be introspected via annotations dicts + A ``TypedDict`` can be introspected via :func:`annotationlib.get_annotations` (see :ref:`annotations-howto` for more information on annotations best practices) and the following attributes: @@ -2898,7 +2898,7 @@ types. For backwards compatibility with Python 3.10 and below, it is also possible to use inheritance to declare both required and - non-required keys in the same ``TypedDict`` . This is done by declaring a + non-required keys in the same ``TypedDict``. This is done by declaring a ``TypedDict`` with one value for the ``total`` argument and then inheriting from it in another ``TypedDict`` with a different value for ``total``: @@ -2982,34 +2982,34 @@ with :deco:`runtime_checkable`. .. class:: SupportsAbs - An ABC with one abstract method ``__abs__`` that is covariant + A protocol with one abstract method ``__abs__`` that is covariant in its return type. .. class:: SupportsBytes - An ABC with one abstract method ``__bytes__``. + A protocol with one abstract method ``__bytes__``. .. class:: SupportsComplex - An ABC with one abstract method ``__complex__``. + A protocol with one abstract method ``__complex__``. .. class:: SupportsFloat - An ABC with one abstract method ``__float__``. + A protocol with one abstract method ``__float__``. .. class:: SupportsIndex - An ABC with one abstract method ``__index__``. + A protocol with one abstract method ``__index__``. .. versionadded:: 3.8 .. class:: SupportsInt - An ABC with one abstract method ``__int__``. + A protocol with one abstract method ``__int__``. .. class:: SupportsRound - An ABC with one abstract method ``__round__`` + A protocol with one abstract method ``__round__`` that is covariant in its return type. .. _typing-io: @@ -3633,14 +3633,27 @@ Introspection helpers Determine if a type is a :class:`Protocol`. - For example:: + For example: + + .. testcode:: class P(Protocol): def a(self) -> str: ... b: int - is_protocol(P) # => True - is_protocol(int) # => False + assert is_protocol(P) + assert not is_protocol(int) + + This function only returns true for ``Protocol`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericP[T](Protocol): + def a(self) -> T: ... + b: int + + assert not is_protocol(GenericP[int]) .. versionadded:: 3.13 @@ -3663,6 +3676,17 @@ Introspection helpers # not a typed dict itself assert not is_typeddict(TypedDict) + This function only returns true for ``TypedDict`` classes, not for + :ref:`generic aliases ` of them: + + .. testcode:: + + class GenericFilm[T](TypedDict): + title: str + year: T + + assert not is_typeddict(GenericFilm[int]) + .. versionadded:: 3.10 .. class:: ForwardRef @@ -3739,7 +3763,7 @@ Constant .. data:: TYPE_CHECKING - A special constant that is assumed to be ``True`` by 3rd party static + A special constant that is assumed to be ``True`` by static type checkers. It's ``False`` at runtime. A module which is expensive to import, and which only contain types diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst index f5c11fd849f58b3..25bf872e0ab55a8 100644 --- a/Doc/library/unicodedata.rst +++ b/Doc/library/unicodedata.rst @@ -18,8 +18,7 @@ this database is compiled from the `UCD version 17.0.0 The module uses the same names and symbols as defined by Unicode Standard Annex #44, `"Unicode Character Database" -`_. It defines the -following functions: +`_. .. seealso:: @@ -27,6 +26,44 @@ following functions: this module. +============================================================ =========================================================== +**Lookup** +------------------------------------------------------------------------------------------------------------------------- +:func:`lookup(name) ` Look up character by name +:func:`name(chr) ` Return the name assigned to a character + +**Numeric values** +------------------------------------------------------------------------------------------------------------------------- +:func:`decimal(chr) ` Decimal value of a character +:func:`digit(chr) ` Digit value of a character +:func:`numeric(chr) ` Numeric value of a character + +**Properties** +------------------------------------------------------------------------------------------------------------------------- +:func:`bidirectional(chr) ` Bidirectional class of a character +:func:`block(chr) ` Unicode block of a character +:func:`category(chr) ` General category of a character +:func:`combining(chr) ` Canonical combining class of a character +:func:`decomposition(chr) ` Character decomposition mapping +:func:`east_asian_width(chr) ` East Asian width of a character +:func:`extended_pictographic(chr) ` Check if a character has the Extended_Pictographic property +:func:`grapheme_cluster_break(chr) ` Grapheme_Cluster_Break property of a character +:func:`indic_conjunct_break(chr) ` Indic_Conjunct_Break property of a character +:func:`isxidcontinue(chr) ` Check if a character is a valid identifier continuation +:func:`isxidstart(chr) ` Check if a character is a valid identifier start +:func:`mirrored(chr) ` Mirrored property of a character + +**Normalization** +------------------------------------------------------------------------------------------------------------------------- +:func:`normalize(form, unistr) ` Return the normalized form of a string +:func:`is_normalized(form, unistr) ` Check if a Unicode string is normalized + +**Text segmentation** +------------------------------------------------------------------------------------------------------------------------- +:func:`iter_graphemes(unistr) ` Iterate over grapheme clusters in a string +============================================================ =========================================================== + + .. function:: lookup(name, /) Look up character by name. If a character with the given name is found, return @@ -273,7 +310,7 @@ following functions: .. versionadded:: 3.15 -In addition, the module exposes the following constant: +In addition, the module exposes the following constants: .. data:: unidata_version diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 5b9f9eec93aa28d..2ff1015af7a86e0 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -2347,12 +2347,10 @@ chained call: >>> kall = call(1).method(arg='foo').other('bar')(2.0) >>> kall.call_list() - [ - call(1), - call().method(arg='foo'), - call().method().other('bar'), - call().method().other()(2.0), - ] + [call(1), + call().method(arg='foo'), + call().method().other('bar'), + call().method().other()(2.0)] >>> m.mock_calls == kall.call_list() True diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index c54f3e2792c3888..ff619f979233251 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -1262,10 +1262,10 @@ Test cases | :meth:`assertNotEndsWith(a, b) | ``not a.endswith(b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertHasAttr(a, b) | ``hastattr(a, b)`` | 3.14 | + | :meth:`assertHasAttr(a, b) | ``hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotHasAttr(a, b) | ``not hastattr(a, b)`` | 3.14 | + | :meth:`assertNotHasAttr(a, b) | ``not hasattr(a, b)`` | 3.14 | | ` | | | +---------------------------------------+--------------------------------+--------------+ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 64e915d042d4a03..03518d49d437ce5 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -1051,7 +1051,7 @@ AbstractBasicAuthHandler Objects *headers* should be the error headers. *host* is either an authority (e.g. ``"python.org"``) or a URL containing an - authority component (e.g. ``"http://python.org/"``). In either case, the + authority component (e.g. ``"https://python.org/"``). In either case, the authority must not contain a userinfo component (so, ``"python.org"`` and ``"python.org:80"`` are fine, ``"joe:password@python.org"`` is not). @@ -1247,10 +1247,14 @@ This example gets the python.org main page and displays the first 300 bytes of it:: >>> import urllib.request - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(300)) - ... - b'\n\n\n - >> import urllib.request - >>> f = urllib.request.urlopen('http://www.python.org/') + >>> f = urllib.request.urlopen('https://www.python.org/') >>> try: - ... print(f.read(100).decode('utf-8')) + ... enc = f.headers.get('Content-Encoding') + ... data = f.read() + ... if enc == 'gzip': + ... import gzip + ... data = gzip.decompress(data) + ... print(data[:100].decode('utf-8', errors='replace')) ... finally: ... f.close() - ... - - - """ % (output_encoded,) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 464126e2df06682..fc502f7fde17808 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -61,6 +61,7 @@ def set_filename_change_hook(self, hook): self.filename_change_hook = hook filename = None + file_timestamp = None dirname = None def set_filename(self, filename): @@ -127,6 +128,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = False except (UnicodeDecodeError, SyntaxError): # Wait for the editor window to appear @@ -142,6 +144,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = True except OSError as err: messagebox.showerror("I/O Error", str(err), parent=self.text) @@ -170,6 +173,7 @@ def loadfile(self, filename): self.text.insert("1.0", chars) self.reset_undo() self.set_filename(filename) + self.file_timestamp = file_timestamp if converted: # We need to save the conversion results first # before being able to execute the code @@ -206,7 +210,26 @@ def save(self, event): if not self.filename: self.save_as(event) else: + # Check the time of most recent content modification so the + # user doesn't accidentally overwrite a newer version of the file. + try: + file_timestamp = self.getmtime(self.filename) + except OSError: + pass + else: + if self.file_timestamp != file_timestamp: + confirm = messagebox.askokcancel( + title="File has changed", + message=( + "The file has changed on disk since reading it!\n\n" + "Do you really want to overwrite it?"), + default=messagebox.CANCEL, + parent=self.text) + if not confirm: + return "break" + if self.writefile(self.filename): + self.file_timestamp = self.getmtime(self.filename) self.set_saved(True) try: self.editwin.store_file_breaks() @@ -219,6 +242,7 @@ def save_as(self, event): filename = self.asksavefile() if filename: if self.writefile(filename): + self.file_timestamp = self.getmtime(filename) self.set_filename(filename) self.set_saved(1) try: @@ -251,6 +275,9 @@ def writefile(self, filename): parent=self.text) return False + def getmtime(self, filename): + return os.stat(filename).st_mtime + def fixnewlines(self): """Return text with os eols. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2fafd9322c609ee..497b5a60cecb083 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -712,7 +712,7 @@ def login(self, user, password): """ typ, dat = self._simple_command('LOGIN', user, self._quote(password)) if typ != 'OK': - raise self.error(dat[-1]) + raise self.error(dat[-1].decode('UTF-8', 'replace')) self.state = 'AUTH' return typ, dat diff --git a/Lib/inspect.py b/Lib/inspect.py index a96b3dc954ef0ca..dc5a6e3be883bb0 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2200,7 +2200,8 @@ def wrap_value(s): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 251025efac14b87..37a86831ff94838 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -142,8 +142,8 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -206,8 +206,8 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -292,12 +292,13 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. - ``array_hook`` is an optional function that will be called with the result - of any literal array decode (a ``list``). The return value of this function will - be used instead of the ``list``. This feature can be used along - ``object_pairs_hook`` to customize the resulting data structure - for example, - by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get - a deep immutable data structute from any JSON data. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -306,7 +307,7 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - array_hook=None, **kw) + array_hook=array_hook, **kw) def loads(s, *, cls=None, object_hook=None, parse_float=None, @@ -327,12 +328,13 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. - ``array_hook`` is an optional function that will be called with the result - of any literal array decode (a ``list``). The return value of this function will - be used instead of the ``list``. This feature can be used along - ``object_pairs_hook`` to customize the resulting data structure - for example, - by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get - a deep immutable data structute from any JSON data. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 7d637325240f1cb..811e796f7766e94 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -152,12 +152,14 @@ def splitdrive(p, /): It is always true that: result[0] + result[1] == p - If the path contained a drive letter, drive_or_unc will contain everything - up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + If the path contained a drive letter, drive_or_unc will contain + everything up to and including the colon. e.g. splitdrive("c:/dir") + returns ("c:", "/dir") - If the path contained a UNC path, the drive_or_unc will contain the host name - and share up to but not including the fourth directory separator character. - e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + If the path contained a UNC path, the drive_or_unc will contain the + host name and share up to but not including the fourth directory + separator character. e.g. splitdrive("//host/computer/dir") returns + ("//host/computer", "/dir") Paths cannot contain both a drive letter and a UNC path. @@ -222,8 +224,8 @@ def splitroot(p, /): def split(p, /): """Split a pathname. - Return tuple (head, tail) where tail is everything after the final slash. - Either part may be empty.""" + Return tuple (head, tail) where tail is everything after the final + slash. Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) d, r, p = splitroot(p) diff --git a/Lib/os.py b/Lib/os.py index 52cbc5bc85864e7..a5e1d8055569988 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -219,14 +219,17 @@ def _add(str, fn): # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) -def makedirs(name, mode=0o777, exist_ok=False): - """makedirs(name [, mode=0o777][, exist_ok=False]) - - Super-mkdir; create a leaf directory and all intermediate ones. Works like - mkdir, except that any intermediate path segment (not just the rightmost) - will be created if it does not exist. If the target directory already - exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. +def makedirs(name, mode=0o777, exist_ok=False, *, parent_mode=None): + """makedirs(name [, mode=0o777][, exist_ok=False][, parent_mode=None]) + + Super-mkdir; create a leaf directory and all intermediate ones. Works + like mkdir, except that any intermediate path segment (not just the + rightmost) will be created if it does not exist. If the target + directory already exists, raise an OSError if exist_ok is False. + Otherwise no exception is raised. If parent_mode is not None, it will + be used as the mode for any newly-created, intermediate-level + directories. Otherwise, intermediate directories are created with the + default permissions (respecting umask). This is recursive. """ head, tail = path.split(name) @@ -234,7 +237,11 @@ def makedirs(name, mode=0o777, exist_ok=False): head, tail = path.split(head) if head and tail and not path.exists(head): try: - makedirs(head, exist_ok=exist_ok) + if parent_mode is not None: + makedirs(head, mode=parent_mode, exist_ok=exist_ok, + parent_mode=parent_mode) + else: + makedirs(head, exist_ok=exist_ok) except FileExistsError: # Defeats race condition when another thread created the path pass @@ -314,12 +321,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of - the names of the subdirectories in dirpath (including symlinks to directories, - and excluding '.' and '..'). + the names of the subdirectories in dirpath (including symlinks to + directories, and excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. - Note that the names in the lists are just names, with no path components. - To get a full path (which begins with top) to a file or directory in - dirpath, do os.path.join(dirpath, name). + Note that the names in the lists are just names, with no path + components. To get a full path (which begins with top) to a file or + directory in dirpath, do os.path.join(dirpath, name). If optional arg 'topdown' is true or not specified, the triple for a directory is generated before the triples for any of its subdirectories @@ -329,13 +336,13 @@ def walk(top, topdown=True, onerror=None, followlinks=False): When topdown is true, the caller can modify the dirnames list in-place (e.g., via del or slice assignment), and walk will only recurse into the - subdirectories whose names remain in dirnames; this can be used to prune the - search, or to impose a specific order of visiting. Modifying dirnames when - topdown is false has no effect on the behavior of os.walk(), since the - directories in dirnames have already been generated by the time dirnames - itself is generated. No matter the value of topdown, the list of - subdirectories is retrieved before the tuples for the directory and its - subdirectories are generated. + subdirectories whose names remain in dirnames; this can be used to prune + the search, or to impose a specific order of visiting. Modifying + dirnames when topdown is false has no effect on the behavior of + os.walk(), since the directories in dirnames have already been generated + by the time dirnames itself is generated. No matter the value of + topdown, the list of subdirectories is retrieved before the tuples for + the directory and its subdirectories are generated. By default errors from the os.scandir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it @@ -462,9 +469,9 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= The advantage of fwalk() over walk() is that it's safe against symlink races (when follow_symlinks is False). - If dir_fd is not None, it should be a file descriptor open to a directory, - and top should be relative; top will then be relative to that directory. - (dir_fd is always supported for fwalk.) + If dir_fd is not None, it should be a file descriptor open to + a directory, and top should be relative; top will then be relative to + that directory. (dir_fd is always supported for fwalk.) Caution: Since fwalk() yields file descriptors, those are only valid until the diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index a32e4b5320ff6dd..8dd16c6225b927b 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -1202,7 +1202,7 @@ def touch(self, mode=0o666, exist_ok=True): fd = os.open(self, flags, mode) os.close(fd) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): + def mkdir(self, mode=0o777, parents=False, exist_ok=False, *, parent_mode=None): """ Create a new directory at this given path. """ @@ -1211,7 +1211,11 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True, exist_ok=True) + if parent_mode is not None: + self.parent.mkdir(mode=parent_mode, parents=True, exist_ok=True, + parent_mode=parent_mode) + else: + self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system diff --git a/Lib/pprint.py b/Lib/pprint.py index 1fd7e3ec95a0734..7355021998081dc 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -43,38 +43,23 @@ "PrettyPrinter", "pp"] -def pprint( - object, - stream=None, - indent=4, - width=88, - depth=None, - *, - compact=False, - sort_dicts=True, - underscore_numbers=False, -): +def pprint(object, stream=None, indent=1, width=80, depth=None, *, + compact=False, expand=False, sort_dicts=True, + underscore_numbers=False): """Pretty-print a Python object to a stream [default is sys.stdout].""" printer = PrettyPrinter( stream=stream, indent=indent, width=width, depth=depth, - compact=compact, sort_dicts=sort_dicts, + compact=compact, expand=expand, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers) printer.pprint(object) -def pformat( - object, - indent=4, - width=88, - depth=None, - *, - compact=False, - sort_dicts=True, - underscore_numbers=False, -): +def pformat(object, indent=1, width=80, depth=None, *, + compact=False, expand=False, sort_dicts=True, + underscore_numbers=False): """Format a Python object into a pretty-printed representation.""" return PrettyPrinter(indent=indent, width=width, depth=depth, - compact=compact, sort_dicts=sort_dicts, + compact=compact, expand=expand, sort_dicts=sort_dicts, underscore_numbers=underscore_numbers).pformat(object) @@ -127,17 +112,9 @@ def _safe_tuple(t): class PrettyPrinter: - def __init__( - self, - indent=4, - width=88, - depth=None, - stream=None, - *, - compact=False, - sort_dicts=True, - underscore_numbers=False, - ): + def __init__(self, indent=1, width=80, depth=None, stream=None, *, + compact=False, expand=False, sort_dicts=True, + underscore_numbers=False): """Handle pretty printing operations onto a stream using a set of configured parameters. @@ -156,6 +133,12 @@ def __init__( compact If true, several items will be combined in one line. + Incompatible with expand mode. + + expand + If true, the output will be formatted similar to + pretty-printed json.dumps() when ``indent`` is supplied. + Incompatible with compact mode. sort_dicts If true, dict keys are sorted. @@ -172,6 +155,8 @@ def __init__( raise ValueError('depth must be > 0') if not width: raise ValueError('width must be != 0') + if compact and expand: + raise ValueError('compact and expand are incompatible') self._depth = depth self._indent_per_level = indent self._width = width @@ -180,6 +165,7 @@ def __init__( else: self._stream = _sys.stdout self._compact = bool(compact) + self._expand = bool(expand) self._sort_dicts = sort_dicts self._underscore_numbers = underscore_numbers @@ -232,36 +218,36 @@ def _format(self, object, stream, indent, allowance, context, level): stream.write(rep) def _format_block_start(self, start_str, indent): - if self._compact: - return start_str - return f"{start_str}\n{' ' * indent}" + if self._expand: + return f"{start_str}\n{' ' * indent}" + return start_str def _format_block_end(self, end_str, indent): - if self._compact: - return end_str - return f"\n{' ' * indent}{end_str}" + if self._expand: + return f"\n{' ' * indent}{end_str}" + return end_str def _child_indent(self, indent, prefix_len): - if self._compact: - return indent + prefix_len - return indent + if self._expand: + return indent + return indent + prefix_len def _write_indent_padding(self, write): - if self._compact: - if self._indent_per_level > 1: - write((self._indent_per_level - 1) * " ") - elif self._indent_per_level > 0: - write(self._indent_per_level * " ") + if self._expand: + if self._indent_per_level > 0: + write(self._indent_per_level * " ") + elif self._indent_per_level > 1: + write((self._indent_per_level - 1) * " ") def _pprint_dataclass(self, object, stream, indent, allowance, context, level): # Lazy import to improve module import time from dataclasses import fields as dataclass_fields cls_name = object.__class__.__name__ - if self._compact: - indent += len(cls_name) + 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += len(cls_name) + 1 items = [(f.name, getattr(object, f.name)) for f in dataclass_fields(object) if f.repr] stream.write(self._format_block_start(cls_name + '(', indent)) self._format_namespace_items(items, stream, indent, allowance, context, level) @@ -384,7 +370,7 @@ def _pprint_list(self, object, stream, indent, allowance, context, level): def _pprint_tuple(self, object, stream, indent, allowance, context, level): stream.write(self._format_block_start('(', indent)) - if len(object) == 1 and self._compact: + if len(object) == 1 and not self._expand: endchar = ',)' else: endchar = ')' @@ -405,7 +391,7 @@ def _pprint_set(self, object, stream, indent, allowance, context, level): else: stream.write(self._format_block_start(typ.__name__ + '({', indent)) endchar = '})' - if self._compact: + if not self._expand: indent += len(typ.__name__) + 1 object = sorted(object, key=_safe_key) self._format_items(object, stream, indent, allowance + len(endchar), @@ -423,10 +409,10 @@ def _pprint_str(self, object, stream, indent, allowance, context, level): chunks = [] lines = object.splitlines(True) if level == 1: - if self._compact: - indent += 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += 1 allowance += 1 max_width1 = max_width = self._width - indent for i, line in enumerate(lines): @@ -479,10 +465,10 @@ def _pprint_bytes(self, object, stream, indent, allowance, context, level): return parens = level == 1 if parens: - if self._compact: - indent += 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += 1 allowance += 1 write(self._format_block_start('(', indent)) delim = '' @@ -499,11 +485,11 @@ def _pprint_bytes(self, object, stream, indent, allowance, context, level): def _pprint_bytearray(self, object, stream, indent, allowance, context, level): write = stream.write write(self._format_block_start('bytearray(', indent)) - if self._compact: - recursive_indent = indent + 10 - else: + if self._expand: write(' ' * self._indent_per_level) recursive_indent = indent + self._indent_per_level + else: + recursive_indent = indent + 10 self._pprint_bytes(bytes(object), stream, recursive_indent, allowance + 1, context, level + 1) write(self._format_block_end(')', indent)) @@ -531,10 +517,10 @@ def _pprint_simplenamespace(self, object, stream, indent, allowance, context, le cls_name = 'namespace' else: cls_name = object.__class__.__name__ - if self._compact: - indent += len(cls_name) + 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += len(cls_name) + 1 items = object.__dict__.items() stream.write(self._format_block_start(cls_name + '(', indent)) self._format_namespace_items(items, stream, indent, allowance, context, @@ -564,7 +550,7 @@ def _format_dict_items(self, items, stream, indent, allowance, context, ) if not last: write(delimnl) - elif not self._compact: + elif self._expand: write(',') def _format_namespace_items(self, items, stream, indent, allowance, context, level): @@ -590,7 +576,7 @@ def _format_namespace_items(self, items, stream, indent, allowance, context, lev ) if not last: write(delimnl) - elif not self._compact: + elif self._expand: write(',') def _format_items(self, items, stream, indent, allowance, context, level): @@ -632,7 +618,7 @@ def _format_items(self, items, stream, indent, allowance, context, level): self._format(ent, stream, indent, allowance if last else 1, context, level) - if last and not self._compact: + if last and self._expand: write(',') def _repr(self, object, context, level): @@ -657,11 +643,11 @@ def _pprint_default_dict(self, object, stream, indent, allowance, context, level return rdf = self._repr(object.default_factory, context, level) cls = object.__class__ - if self._compact: + if self._expand: + stream.write('%s(%s, ' % (cls.__name__, rdf)) + else: indent += len(cls.__name__) + 1 stream.write('%s(%s,\n%s' % (cls.__name__, rdf, ' ' * indent)) - else: - stream.write('%s(%s, ' % (cls.__name__, rdf)) self._pprint_dict(object, stream, indent, allowance + 1, context, level) stream.write(')') @@ -695,14 +681,14 @@ def _pprint_chain_map(self, object, stream, indent, allowance, context, level): cls = object.__class__ stream.write(self._format_block_start(cls.__name__ + '(', indent + self._indent_per_level)) - if self._compact: - indent += len(cls.__name__) + 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += len(cls.__name__) + 1 for i, m in enumerate(object.maps): if i == len(object.maps) - 1: self._format(m, stream, indent, allowance + 1, context, level) - if not self._compact: + if self._expand: stream.write(',') stream.write(self._format_block_end(')', indent - self._indent_per_level)) else: @@ -717,7 +703,7 @@ def _pprint_deque(self, object, stream, indent, allowance, context, level): return cls = object.__class__ stream.write(self._format_block_start(cls.__name__ + '([', indent)) - if self._compact: + if not self._expand: indent += len(cls.__name__) + 1 if object.maxlen is None: self._format_items(object, stream, indent, allowance + 2, @@ -727,10 +713,10 @@ def _pprint_deque(self, object, stream, indent, allowance, context, level): self._format_items(object, stream, indent, 2, context, level) rml = self._repr(object.maxlen, context, level) - if self._compact: - stream.write('],\n%smaxlen=%s)' % (' ' * indent, rml)) - else: + if self._expand: stream.write('%s], maxlen=%s)' % ('\n' + ' ' * indent, rml)) + else: + stream.write('],\n%smaxlen=%s)' % (' ' * indent, rml)) _dispatch[_collections.deque.__repr__] = _pprint_deque @@ -751,10 +737,10 @@ def _pprint_user_string(self, object, stream, indent, allowance, context, level) def _pprint_template(self, object, stream, indent, allowance, context, level): cls_name = object.__class__.__name__ - if self._compact: - indent += len(cls_name) + 1 - else: + if self._expand: indent += self._indent_per_level + else: + indent += len(cls_name) + 1 items = ( ("strings", object.strings), @@ -770,20 +756,7 @@ def _pprint_template(self, object, stream, indent, allowance, context, level): def _pprint_interpolation(self, object, stream, indent, allowance, context, level): cls_name = object.__class__.__name__ - if self._compact: - indent += len(cls_name) - items = ( - object.value, - object.expression, - object.conversion, - object.format_spec, - ) - stream.write(cls_name + "(") - self._format_items( - items, stream, indent, allowance, context, level - ) - stream.write(")") - else: + if self._expand: indent += self._indent_per_level items = ( ("value", object.value), @@ -798,6 +771,19 @@ def _pprint_interpolation(self, object, stream, indent, allowance, context, leve stream.write( self._format_block_end(")", indent - self._indent_per_level) ) + else: + indent += len(cls_name) + items = ( + object.value, + object.expression, + object.conversion, + object.format_spec, + ) + stream.write(cls_name + "(") + self._format_items( + items, stream, indent, allowance, context, level + ) + stream.write(")") t = t"{0}" _dispatch[type(t).__repr__] = _pprint_template diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index 1611bf754424c13..840acf2c27d1201 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -7,6 +7,7 @@ let invertedData = null; let currentThreadFilter = 'all'; let isInverted = false; let useModuleNames = true; +let zoomedNodeValue = null; // Heat colors are now defined in CSS variables (--heat-1 through --heat-8) // and automatically switch with theme changes - no JS color arrays needed! @@ -316,6 +317,7 @@ function createPythonTooltip(data) { const selfSamples = d.data.self || 0; const selfMs = (selfSamples / 1000).toFixed(2); const percentage = ((d.data.value / data.value) * 100).toFixed(2); + const relativePercentage = Math.min(100, ((d.data.value / (zoomedNodeValue ?? data.value)) * 100)).toFixed(2); const calls = d.data.calls || 0; const childCount = d.children ? d.children.length : 0; const source = d.data.source; @@ -439,6 +441,11 @@ function createPythonTooltip(data) { Percentage: ${percentage}% + ${relativePercentage != percentage && relativePercentage != "100.00" ? ` + Relative Percentage: + ${relativePercentage}% + ` : ''} + ${calls > 0 ? ` Function Calls: ${calls.toLocaleString()} @@ -620,6 +627,9 @@ function createFlamegraph(tooltip, rootValue, data) { const percentage = d.data.value / rootValue; const level = getHeatLevel(percentage); return heatColors[level]; + }) + .onClick(function (d) { + zoomedNodeValue = d.data.value; }); return chart; @@ -629,6 +639,7 @@ function renderFlamegraph(chart, data) { d3.select("#chart").datum(data).call(chart); window.flamegraphChart = chart; window.flamegraphData = data; + zoomedNodeValue = null; populateStats(data); } @@ -1269,6 +1280,7 @@ function filterDataByThread(data, threadId) { function resetZoom() { if (window.flamegraphChart) { + zoomedNodeValue = null; window.flamegraphChart.resetZoom(); } } diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index 2da1103b82a52a3..1f698779f3a46e3 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -84,7 +84,7 @@ function showNavigationMenu(button, items, title) { item.appendChild(funcDiv); item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); - item.addEventListener('click', () => window.location.href = linkData.link); + item.addEventListener('click', () => navigateToLine(linkData.link)); menu.appendChild(item); }); @@ -105,7 +105,7 @@ function handleNavigationClick(button, e) { const navData = button.getAttribute('data-nav'); if (navData) { - window.location.href = JSON.parse(navData).link; + navigateToLine(JSON.parse(navData).link); return; } @@ -117,11 +117,29 @@ function handleNavigationClick(button, e) { } } +function restartLineHighlight(target) { + target.style.animation = 'none'; + // Force style recalculation so restoring the animation restarts it. + void target.offsetWidth; + target.style.animation = ''; +} + +function navigateToLine(link) { + const url = new URL(link, window.location.href); + + if (url.href === window.location.href) { + scrollToTargetLine(); + } else { + window.location.href = link; + } +} + function scrollToTargetLine() { if (!window.location.hash) return; const target = document.querySelector(window.location.hash); if (target) { target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + restartLineHighlight(target); } } diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 0648713edc52af3..a5d9573ae6b6ddd 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -167,7 +167,9 @@ def _build_child_profiler_args(args): child_args.extend(["--mode", mode]) # Format options (skip pstats as it's the default) - if args.format != "pstats": + if args.format == "diff_flamegraph": + child_args.extend(["--diff-flamegraph", args.diff_baseline]) + elif args.format != "pstats": child_args.append(f"--{args.format}") return child_args diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 81ec6344ebdea4a..8e0f0c44c4f8f36 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -143,6 +143,8 @@ def iter_async_frames(awaited_info_list): class Collector(ABC): + aggregating = False + @abstractmethod def collect(self, stack_frames, timestamps_us=None): """Collect profiling data from stack frames. diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 8986194268b3ce4..361f6037f216fdc 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -1,8 +1,10 @@ import itertools +import io import json import os import platform import sys +import tempfile import threading import time @@ -61,8 +63,81 @@ PROCESS_TYPE_MAIN = 0 STACKWALK_DISABLED = 0 +# In-memory buffer before spilling to disk +DEFAULT_SPILL_BUFFER_BYTES = 128 * 1024 +_JSON_SEPARATORS = (",", ":") +_JSON_ENCODER = json.JSONEncoder( + separators=_JSON_SEPARATORS, allow_nan=False +) + + +class SpillColumn: + def __init__(self, directory, basename, *, + buffer_bytes=None): + self.path = os.path.join(directory, basename) + self.buffer = bytearray() + self._buffer_bytes = ( + DEFAULT_SPILL_BUFFER_BYTES if buffer_bytes is None + else buffer_bytes + ) + + def append(self, value): + self.buffer += (_JSON_ENCODER.encode(value) + "\n").encode("utf-8") + if len(self.buffer) >= self._buffer_bytes: + self.flush() + + def flush(self): + with open(self.path, "ab") as file: + file.write(self.buffer) + self.buffer.clear() + + def iter_tokens(self): + with open(self.path, encoding="utf-8") as file: + for line in file: + yield line.rstrip("\n") + + +class GeckoThreadSpill: + _COLUMNS = ( + ("samples_stack", "samples-stack.json"), + ("samples_time", "samples-time.json"), + ("markers_name", "markers-name.json"), + ("markers_start_time", "markers-start-time.json"), + ("markers_end_time", "markers-end-time.json"), + ("markers_phase", "markers-phase.json"), + ("markers_category", "markers-category.json"), + ("markers_data", "markers-data.json"), + ) + + def __init__(self, directory, tid): + prefix = f"thread-{tid}-" + for attr, basename in self._COLUMNS: + setattr(self, attr, SpillColumn(directory, prefix + basename)) + self.sample_count = 0 + self.marker_count = 0 + + def append_sample(self, stack_index, time_ms): + self.samples_stack.append(stack_index) + self.samples_time.append(time_ms) + self.sample_count += 1 + + def append_marker(self, name_idx, start_time, end_time, phase, category, data): + self.markers_name.append(name_idx) + self.markers_start_time.append(start_time) + self.markers_end_time.append(end_time) + self.markers_phase.append(phase) + self.markers_category.append(category) + self.markers_data.append(data) + self.marker_count += 1 + + def prepare_read(self): + for attr, _basename in self._COLUMNS: + getattr(self, attr).flush() + class GeckoCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle @@ -75,6 +150,8 @@ def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): # Per-thread data structures self.threads = {} # tid -> thread data + self.spill_dir = None + self.exported = False # Global tables self.libs = [] @@ -149,6 +226,9 @@ def collect(self, stack_frames, timestamps_us=None): stack_frames: List of interpreter/thread frame info timestamps_us: List of timestamps in microseconds (None for live sampling) """ + if self.exported: + raise RuntimeError("cannot append to GeckoCollector after export") + # Handle live sampling (no timestamps provided) if timestamps_us is None: current_time = (time.monotonic() * 1000) - self.start_time @@ -257,15 +337,9 @@ def collect(self, stack_frames, timestamps_us=None): stack_index = self._process_stack(thread_data, frames) # Add samples with timestamps - samples = thread_data["samples"] - samples_stack = samples["stack"] - samples_time = samples["time"] - samples_delay = samples["eventDelay"] - + thread_spill = thread_data["_spill"] for t in times: - samples_stack.append(stack_index) - samples_time.append(t) - samples_delay.append(None) + thread_spill.append_sample(stack_index, t) # Handle opcodes if self.opcodes_enabled and frames: @@ -292,6 +366,8 @@ def collect(self, stack_frames, timestamps_us=None): def _create_thread(self, tid, is_main_thread): """Create a new thread structure with processed profile format.""" + if self.spill_dir is None: + self.spill_dir = tempfile.TemporaryDirectory() thread = { "name": f"Thread-{tid}", @@ -305,15 +381,6 @@ def _create_thread(self, tid, is_main_thread): "tid": tid, "processType": "default", "processName": "Python Process", - # Sample data - processed format with direct arrays - "samples": { - "stack": [], - "time": [], - "eventDelay": [], - "weight": None, - "weightType": "samples", - "length": 0, # Will be updated on export - }, # Stack table - processed format "stackTable": { "frame": [], @@ -364,21 +431,12 @@ def _create_thread(self, tid, is_main_thread): "functionSize": [], "length": 0, }, - # Markers - processed format (arrays) - "markers": { - "data": [], - "name": [], - "startTime": [], - "endTime": [], - "phase": [], - "category": [], - "length": 0, - }, # Caches for deduplication "_stackCache": {}, "_frameCache": {}, "_funcCache": {}, "_resourceCache": {}, + "_spill": GeckoThreadSpill(self.spill_dir.name, tid), } return thread @@ -403,51 +461,42 @@ def _add_marker(self, tid, name, start_time, end_time, category): if tid not in self.threads: return - thread_data = self.threads[tid] duration = end_time - start_time name_idx = self._intern_string(name) - markers = thread_data["markers"] - markers["name"].append(name_idx) - markers["startTime"].append(start_time) - markers["endTime"].append(end_time) - markers["phase"].append(1) # 1 = interval marker - markers["category"].append(category) - markers["data"].append({ - "type": name.replace(" ", ""), - "duration": duration, - "tid": tid - }) - - def _add_opcode_interval_marker(self, tid, opcode, lineno, col_offset, funcname, start_time, end_time): + self.threads[tid]["_spill"].append_marker( + name_idx, start_time, end_time, 1, category, { + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid, + } + ) + + def _add_opcode_interval_marker(self, tid, opcode, lineno, col_offset, + funcname, start_time, end_time): """Add an interval marker for opcode execution span.""" if tid not in self.threads or opcode is None: return - thread_data = self.threads[tid] opcode_info = get_opcode_info(opcode) # Use formatted opcode name (with base opcode for specialized ones) formatted_opname = format_opcode(opcode) name_idx = self._intern_string(formatted_opname) - markers = thread_data["markers"] - markers["name"].append(name_idx) - markers["startTime"].append(start_time) - markers["endTime"].append(end_time) - markers["phase"].append(1) # 1 = interval marker - markers["category"].append(CATEGORY_OPCODES) - markers["data"].append({ - "type": "Opcode", - "opcode": opcode, - "opname": formatted_opname, - "base_opname": opcode_info["base_opname"], - "is_specialized": opcode_info["is_specialized"], - "line": lineno, - "column": col_offset if col_offset >= 0 else None, - "function": funcname, - "duration": end_time - start_time, - }) + self.threads[tid]["_spill"].append_marker( + name_idx, start_time, end_time, 1, CATEGORY_OPCODES, { + "type": "Opcode", + "opcode": opcode, + "opname": formatted_opname, + "base_opname": opcode_info["base_opname"], + "is_specialized": opcode_info["is_specialized"], + "line": lineno, + "column": col_offset if col_offset >= 0 else None, + "function": funcname, + "duration": end_time - start_time, + } + ) def _process_stack(self, thread_data, frames): """Process a stack and return the stack index.""" @@ -658,7 +707,6 @@ def _finalize_markers(self): def export(self, filename): """Export the profile to a Gecko JSON file.""" - if self.sample_count > 0 and self.last_sample_time > 0: self.interval = self.last_sample_time / self.sample_count @@ -679,19 +727,30 @@ def spin(): spinner_thread = threading.Thread(target=spin, daemon=True) spinner_thread.start() + temp_path = None + replaced = False try: - # Finalize any open markers before building profile - self._finalize_markers() - - profile = self._build_profile() - - with open(filename, "w") as f: - json.dump(profile, f, separators=(",", ":")) + self._prepare_for_serialization() + output_dir = os.path.dirname(os.path.abspath(filename)) or "." + with tempfile.NamedTemporaryFile( + "w", dir=output_dir, delete=False + ) as file: + temp_path = file.name + self._stream_profile(file) + os.replace(temp_path, filename) + replaced = True finally: + self.exported = True stop_spinner.set() spinner_thread.join(timeout=1.0) # Small delay to ensure the clear happens time.sleep(0.01) + if temp_path is not None and not replaced: + try: + os.unlink(temp_path) + except FileNotFoundError: + pass + self._cleanup_spills() print(f"Gecko profile written to {filename}") print( @@ -725,34 +784,17 @@ def _build_marker_schema(self): def _build_profile(self): """Build the complete profile structure in processed format.""" - # Convert thread data to final format - threads = [] - - for tid, thread_data in self.threads.items(): - # Update lengths - samples = thread_data["samples"] - stack_table = thread_data["stackTable"] - frame_table = thread_data["frameTable"] - func_table = thread_data["funcTable"] - resource_table = thread_data["resourceTable"] - - samples["length"] = len(samples["stack"]) - stack_table["length"] = len(stack_table["frame"]) - frame_table["length"] = len(frame_table["func"]) - func_table["length"] = len(func_table["name"]) - resource_table["length"] = len(resource_table["name"]) - thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) - - # Clean up internal caches - del thread_data["_stackCache"] - del thread_data["_frameCache"] - del thread_data["_funcCache"] - del thread_data["_resourceCache"] - - threads.append(thread_data) - - # Main profile structure in processed format - profile = { + try: + self._prepare_for_serialization() + file = io.StringIO() + self._stream_profile(file) + return json.loads(file.getvalue()) + finally: + self.exported = True + self._cleanup_spills() + + def _profile_head(self): + return { "meta": { "interval": self.interval, "startTime": self.start_time, @@ -782,7 +824,10 @@ def _build_profile(self): }, }, "libs": self.libs, - "threads": threads, + } + + def _profile_tail(self): + return { "pages": [], "shared": { "stringArray": self.global_strings, @@ -790,4 +835,146 @@ def _build_profile(self): }, } - return profile + def _prepare_for_serialization(self): + if self.exported: + raise RuntimeError("GeckoCollector has already been exported") + self._finalize_markers() + for thread_data in self.threads.values(): + thread_data["_spill"].prepare_read() + thread_data["stackTable"]["length"] = len(thread_data["stackTable"]["frame"]) + thread_data["frameTable"]["length"] = len(thread_data["frameTable"]["func"]) + thread_data["funcTable"]["length"] = len(thread_data["funcTable"]["name"]) + thread_data["resourceTable"]["length"] = len(thread_data["resourceTable"]["name"]) + + def _cleanup_spills(self): + if self.spill_dir is not None: + self.spill_dir.cleanup() + self.spill_dir = None + + def _stream_profile(self, file): + file.write("{") + first = True + for key, value in self._profile_head().items(): + first = _write_json_member(file, key, value, first) + + first = _write_member_name(file, "threads", first) + file.write("[") + for index, (tid, thread_data) in enumerate(self.threads.items()): + if index: + file.write(",") + self._stream_thread(file, tid, thread_data) + file.write("]") + + for key, value in self._profile_tail().items(): + first = _write_json_member(file, key, value, first) + file.write("}") + + def _stream_thread(self, file, tid, thread_data): + spill = thread_data["_spill"] + metadata = { + "name": thread_data["name"], + "isMainThread": thread_data["isMainThread"], + "processStartupTime": thread_data["processStartupTime"], + "processShutdownTime": thread_data["processShutdownTime"], + "registerTime": thread_data["registerTime"], + "unregisterTime": thread_data["unregisterTime"], + "pausedRanges": thread_data["pausedRanges"], + "pid": thread_data["pid"], + "tid": thread_data["tid"], + "processType": thread_data["processType"], + "processName": thread_data["processName"], + } + file.write("{") + first = True + for key, value in metadata.items(): + first = _write_json_member(file, key, value, first) + + first = _write_member_name(file, "samples", first) + self._stream_samples(file, spill) + for key in ( + "stackTable", + "frameTable", + "funcTable", + "resourceTable", + "nativeSymbols", + ): + first = _write_json_member(file, key, thread_data[key], first) + first = _write_member_name(file, "markers", first) + self._stream_markers(file, spill) + file.write("}") + + def _stream_samples(self, file, spill): + _stream_column_table( + file, + ( + ("stack", spill.samples_stack.iter_tokens()), + ("time", spill.samples_time.iter_tokens()), + ("eventDelay", ("null" for _ in range(spill.sample_count))), + ), + spill.sample_count, + ( + ("weight", None), + ("weightType", "samples"), + ("length", spill.sample_count), + ), + ) + + def _stream_markers(self, file, spill): + _stream_column_table( + file, + ( + ("data", spill.markers_data.iter_tokens()), + ("name", spill.markers_name.iter_tokens()), + ("startTime", spill.markers_start_time.iter_tokens()), + ("endTime", spill.markers_end_time.iter_tokens()), + ("phase", spill.markers_phase.iter_tokens()), + ("category", spill.markers_category.iter_tokens()), + ), + spill.marker_count, + (("length", spill.marker_count),), + ) + + +def _write_json(file, value): + for chunk in _JSON_ENCODER.iterencode(value): + file.write(chunk) + + +def _write_member_name(file, name, first): + if not first: + file.write(",") + _write_json(file, name) + file.write(":") + return False + + +def _write_json_member(file, name, value, first): + first = _write_member_name(file, name, first) + _write_json(file, value) + return first + + +def _stream_column_table(file, columns, expected_count, trailing_members=()): + file.write("{") + first = True + for name, token_iter in columns: + first = _write_member_name(file, name, first) + _stream_array(file, token_iter, expected_count, name) + for name, value in trailing_members: + first = _write_json_member(file, name, value, first) + file.write("}") + + +def _stream_array(file, token_iter, expected_count, label="array"): + file.write("[") + count = 0 + for token in token_iter: + if count: + file.write(",") + file.write(token) + count += 1 + if count != expected_count: + raise RuntimeError( + f"streamed {count} {label} items, expected {expected_count}" + ) + file.write("]") diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 5c36d78f5535e71..6e650ec08f410bc 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -452,7 +452,8 @@ def process_frames(self, frames, thread_id, weight=1): next_lineno = extract_lineno(next_frame[1]) self._record_call_relationship( (filename, lineno, funcname), - (next_frame[0], next_lineno, next_frame[2]) + (next_frame[0], next_lineno, next_frame[2]), + weight=weight, ) def _is_valid_frame(self, filename, lineno): @@ -561,7 +562,7 @@ def _get_bytecode_data_for_line(self, filename, lineno): result.sort(key=lambda x: (-x['samples'], x['opcode'])) return result - def _record_call_relationship(self, callee_frame, caller_frame): + def _record_call_relationship(self, callee_frame, caller_frame, weight=1): """Record caller/callee relationship between adjacent frames.""" callee_filename, callee_lineno, callee_funcname = callee_frame caller_filename, caller_lineno, caller_funcname = caller_frame @@ -587,7 +588,7 @@ def _record_call_relationship(self, callee_frame, caller_frame): # Count this call edge for path analysis edge_key = (caller_key, callee_key) - self.edge_samples[edge_key] += 1 + self.edge_samples[edge_key] += weight def export(self, output_path): """Export heatmap data as HTML files in a directory. diff --git a/Lib/profiling/sampling/jsonl_collector.py b/Lib/profiling/sampling/jsonl_collector.py index 7d26129b80de868..5aa42ef09024dc3 100644 --- a/Lib/profiling/sampling/jsonl_collector.py +++ b/Lib/profiling/sampling/jsonl_collector.py @@ -164,6 +164,7 @@ def export(self, filename): self._iter_final_agg_entries(), ) self._write_message(output, self._build_end_record()) + print(f"JSONL profile written to {filename}") def _build_meta_record(self): record = { diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 50500296c15acc9..43b1daf2a119d4e 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -8,6 +8,8 @@ class PstatsCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 5bbe24835813332..50ccc57566d70d3 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -47,9 +47,41 @@ def _pause_threads(unwinder, blocking): # If fewer samples are collected, we skip the TUI and just print a message MIN_SAMPLES_FOR_TUI = 200 +# Maximum number of consecutive identical samples to keep before flushing. +MAX_PENDING_SAMPLES = 8192 + + +def _resolve_python_pid(pid): + """On Windows, if pid is a venvlauncher process, return the child Python PID. + + The venvlauncher (used as python.exe in venvs) spawns the real Python + interpreter as a child process via CreateProcessW. The RemoteUnwinder + needs the child's PID, not the launcher's. + + Returns the original pid if not on Windows, not a venv launcher, + or no child process is found. + """ + if os.name != "nt" or sys.prefix == sys.base_prefix: + return pid + try: + children = _remote_debugging.get_child_pids(pid, recursive=False) + python_children = [ + child for child in children + if _remote_debugging.is_python_process(child) + ] + if len(python_children) == 1: + return python_children[0] + except (OSError, RuntimeError) as err: + raise SystemExit( + f"Failed to initialize profiler from virtualenv: {err}\n" + f"Try running with the base interpreter: {sys._base_executable}" + ) from err + return pid + + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): - self.pid = pid + self.pid = _resolve_python_pid(pid) self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads self.mode = mode # Store mode for later use @@ -58,10 +90,6 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD try: self.unwinder = self._new_unwinder(native, gc, opcodes, skip_non_matching_threads) except RuntimeError as err: - if os.name == "nt" and sys.executable.endswith("python.exe"): - raise SystemExit( - "Running profiling.sampling from virtualenv on Windows platform is not supported" - ) from err raise SystemExit(err) from err # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -109,6 +137,20 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + aggregating = getattr(collector, 'aggregating', False) is True + prev_stack = None + pending_count = 0 + pending_timestamps = [] if aggregating else None + + def flush_pending(): + nonlocal pending_count, pending_timestamps + if pending_count == 0: + return + pending_count = 0 + ts = pending_timestamps + pending_timestamps = [] + collector.collect(prev_stack, timestamps_us=ts) + try: while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop @@ -116,6 +158,7 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): break current_time = time.perf_counter() + current_time_us = int(current_time * 1_000_000) if next_time > current_time: sleep_time = (next_time - current_time) * 0.9 if sleep_time > 0.0001: @@ -125,13 +168,24 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): stack_frames = self._get_stack_trace( async_aware=async_aware ) - collector.collect(stack_frames) + if aggregating: + if stack_frames != prev_stack: + flush_pending() + prev_stack = stack_frames + pending_count += 1 + pending_timestamps.append(current_time_us) + if pending_count >= MAX_PENDING_SAMPLES: + flush_pending() + else: + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + flush_pending() collector.collect_failed_sample() errors += 1 + prev_stack = None except Exception as e: if not _is_process_running(self.pid): break @@ -163,6 +217,8 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): interrupted = True running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") + finally: + flush_pending() # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: @@ -296,6 +352,33 @@ def _print_unwinder_stats(self): print(f" Hits: {code_hits:n} ({ANSIColors.GREEN}{fmt(code_hits_pct)}%{ANSIColors.RESET})") print(f" Misses: {code_misses:n} ({ANSIColors.RED}{fmt(code_misses_pct)}%{ANSIColors.RESET})") + batched_attempts = stats.get('batched_read_attempts', 0) + batched_successes = stats.get('batched_read_successes', 0) + batched_misses = stats.get('batched_read_misses', 0) + segments_requested = stats.get('batched_read_segments_requested', 0) + segments_completed = stats.get('batched_read_segments_completed', 0) + if batched_attempts > 0: + batched_success_rate = stats.get('batched_read_success_rate', 0.0) + batched_miss_rate = 100.0 - batched_success_rate + segment_completion_rate = stats.get( + 'batched_read_segment_completion_rate', 0.0 + ) + + print(f" {ANSIColors.CYAN}Batched Reads:{ANSIColors.RESET}") + print(f" Attempts: {batched_attempts:n}") + print( + f" Successes: {batched_successes:n} " + f"({ANSIColors.GREEN}{fmt(batched_success_rate)}%{ANSIColors.RESET})" + ) + print( + f" Misses: {batched_misses:n} " + f"({ANSIColors.RED}{fmt(batched_miss_rate)}%{ANSIColors.RESET})" + ) + print( + f" Segments read: {segments_completed:n}/{segments_requested:n} " + f"({ANSIColors.GREEN}{fmt(segment_completion_rate)}%{ANSIColors.RESET})" + ) + # Memory operations memory_reads = stats.get('memory_reads', 0) memory_bytes = stats.get('memory_bytes_read', 0) diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 04622a8c1e89ef6..42281dc6454c83c 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -16,6 +16,8 @@ class StackTraceCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle @@ -698,6 +700,8 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path): func_key = self._extract_func_key(node, self._baseline_collector._string_table) current_path = path + (func_key,) if func_key else path + baseline_self = 0 + baseline_total = 0 if func_key and current_path in baseline_stats: baseline_data = baseline_stats[current_path] baseline_self = baseline_data["self"] * scale diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a1a6aad434ddf4d..497cc7d90a42456 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1845,6 +1845,7 @@ class Helper: 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'lazy': ('lazy', 'MODULES'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index 1a3126d3db95909..0505210b0bfe0df 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Thu May 7 16:26:23 2026 +# Autogenerated by Sphinx on Tue Jun 2 18:28:34 2026 # as part of the release process. module_docs = { diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 5f61001c46b79ce..3ab289ebed6a6f6 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Thu May 7 16:26:23 2026 +# Autogenerated by Sphinx on Tue Jun 2 18:28:34 2026 # as part of the release process. topics = { @@ -2344,9 +2344,9 @@ def foo(): The match statement is used for pattern matching. Syntax: match_stmt: 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT - subject_expr: `!star_named_expression` "," `!star_named_expressions`? - | `!named_expression` - case_block: 'case' patterns [guard] ":" `!block` + subject_expr: flexible_expression "," [flexible_expression_list [',']] + | assignment_expression + case_block: 'case' patterns [guard] ":" suite Note: @@ -2437,7 +2437,7 @@ def foo(): Guards ------ - guard: "if" `!named_expression` + guard: "if" assignment_expression A "guard" (which is part of the "case") must succeed for code inside the "case" block to execute. It takes the form: "if" followed by an @@ -4971,61 +4971,49 @@ def inner(x): 'dict': r'''Dictionary displays ******************* -A dictionary display is a possibly empty series of dict items -(key/value pairs) enclosed in curly braces: - - dict_display: "{" [dict_item_list | dict_comprehension] "}" - dict_item_list: dict_item ("," dict_item)* [","] - dict_comprehension: dict_item comp_for - dict_item: expression ":" expression | "**" or_expr - -A dictionary display yields a new dictionary object. - -If a comma-separated sequence of dict items is given, they are -evaluated from left to right to define the entries of the dictionary: -each key object is used as a key into the dictionary to store the -corresponding value. This means that you can specify the same key -multiple times in the dict item list, and the final dictionary’s value -for that key will be the last one given. - -A double asterisk "**" denotes *dictionary unpacking*. Its operand -must be a *mapping*. Each mapping item is added to the new -dictionary. Later values replace values already set by earlier dict -items and earlier dictionary unpackings. +A *dictionary display* is a possibly empty series of *dict items* +enclosed in curly braces. Each dict item is a colon-separated pair of +expressions: the *key* and its associated *value*. For example: + + >>> {1: 'one', 2: 'two'} + {1: 'one', 2: 'two'} + +At runtime, when a dictionary comprehension is evaluated, the +expressions are evaluated from left to right. Each key object is used +as a key into the dictionary to store the corresponding value. This +means that you can specify the same key multiple times in the +comprehension, and the final dictionary’s value for a given key will +be the last one given. For example: + + >>> { + ... 1: 'this will be overridden', + ... 2: 'two', + ... 1: 'also overridden', + ... 1: 'one', + ... } + {1: 'one', 2: 'two'} + +Instead of a key-value pair, a dict item may be an expression prefixed +by a double asterisk "**". This denotes *dictionary unpacking*. At +runtime, the expression must evaluate to a *mapping*; each item of the +mapping is added to the new dictionary. As with key-value pairs, later +values replace values already set by earlier items and unpackings. +This may be used to override a set of defaults: + + >>> defaults = {'color': 'blue', 'count': 8} + >>> overrides = {'color': 'yellow'} + >>> {**defaults, **overrides} + {'color': 'yellow', 'count': 8} Added in version 3.5: Unpacking into dictionary displays, originally proposed by **PEP 448**. -A dict comprehension may take one of two forms: - -* The first form uses two expressions separated with a colon followed - by the usual “for” and “if” clauses. When the comprehension is run, - the resulting key and value elements are inserted in the new - dictionary in the order they are produced. - -* The second form uses a single expression prefixed by the "**" - dictionary unpacking operator followed by the usual “for” and “if” - clauses. When the comprehension is evaluated, the expression is - evaluated and then unpacked, inserting zero or more key/value pairs - into the new dictionary. - -Both forms of dictionary comprehension retain the property that if the -same key is specified multiple times, the associated value in the -resulting dictionary will be the last one specified. - -Restrictions on the types of the key values are listed earlier in -section The standard type hierarchy. (To summarize, the key type -should be *hashable*, which excludes all mutable objects.) Clashes -between duplicate keys are not detected; the last value (textually -rightmost in the display) stored for a given key value prevails. +The formal grammar for dict displays is: -Changed in version 3.8: Prior to Python 3.8, in dict comprehensions, -the evaluation order of key and value was not well-defined. In -CPython, the value was evaluated before the key. Starting with 3.8, -the key is evaluated before the value, as proposed by **PEP 572**. - -Changed in version 3.15: Unpacking with the "**" operator is now -allowed in dictionary comprehensions. + dict: '{' [double_starred_kvpairs] '}' + double_starred_kvpairs: ','.double_starred_kvpair+ [','] + double_starred_kvpair: '**' or_expr | kvpair + kvpair: expression ':' expression ''', 'dynamic-features': r'''Interaction with dynamic features ********************************* @@ -5655,8 +5643,22 @@ class of the instance or a *non-virtual base class* thereof. The is the number of expressions in the list. The expressions are evaluated from left to right. -An asterisk "*" denotes *iterable unpacking*. Its operand must be an -*iterable*. The iterable is expanded into a sequence of items, which +A trailing comma is required only to create a one-item tuple, such as +"1,"; it is optional in all other cases. A single expression without a +trailing comma doesn’t create a tuple, but rather yields the value of +that expression. (To create an empty tuple, use an empty pair of +parentheses: "()".) + + +Iterable unpacking +================== + +In an expression list or tuple, list or set display, any expression +may be prefixed with an asterisk ("*"). This denotes *iterable +unpacking*. + +At runtime, the asterisk-prefixed expression must evaluate to an +*iterable*. The iterable is expanded into a sequence of items, which are included in the new tuple, list, or set, at the site of the unpacking. @@ -5665,12 +5667,6 @@ class of the instance or a *non-virtual base class* thereof. The Added in version 3.11: Any item in an expression list may be starred. See **PEP 646**. - -A trailing comma is required only to create a one-item tuple, such as -"1,"; it is optional in all other cases. A single expression without a -trailing comma doesn’t create a tuple, but rather yields the value of -that expression. (To create an empty tuple, use an empty pair of -parentheses: "()".) ''', 'floating': r'''Floating-point literals *********************** @@ -6015,7 +6011,8 @@ class of the instance or a *non-virtual base class* thereof. The | | is not supported. | +-----------+------------------------------------------------------------+ -For a locale aware separator, use the "'n'" presentation type instead. +For a locale-aware separator, use the "'n'" float presentation type or +integer presentation type instead. Changed in version 3.1: Added the "','" option (see also **PEP 378**). @@ -6061,7 +6058,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'d'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators. | + | | group separators. Note that the default locale is not the | + | | system locale. Depending on your use case, you may wish to | + | | set "LC_NUMERIC" with "locale.setlocale()" before using | + | | "'n'". | +-----------+------------------------------------------------------------+ | None | The same as "'d'". | +-----------+------------------------------------------------------------+ @@ -6135,7 +6135,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'g'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators for the integral part of a number. | + | | group separators for the integral part of a number. Note | + | | that the default locale is not the system locale. | + | | Depending on your use case, you may wish to set | + | | "LC_NUMERIC" with "locale.setlocale()" before using "'n'". | +-----------+------------------------------------------------------------+ | "'%'" | Percentage. Multiplies the number by 100 and displays in | | | fixed ("'f'") format, followed by a percent sign. | @@ -7231,21 +7234,113 @@ def (parameters): See section Function definitions for the syntax of parameter lists. Note that functions created with lambda expressions cannot contain statements or annotations. +''', + 'lazy': r'''Lazy imports +************ + +The "lazy" keyword is a soft keyword that only has special meaning +when it appears immediately before an "import" or "from" statement. +When an import statement is preceded by the "lazy" keyword, the import +becomes *lazy*: the module is not loaded immediately at the import +statement. Instead, a lazy proxy object is created and bound to the +name. The actual module is loaded on first use of that name. + +Lazy imports are only permitted at module scope. Using "lazy" inside a +function, class body, or "try"/"except"/"finally" block raises a +"SyntaxError". Star imports cannot be lazy ("lazy from module import +*" is a syntax error), and future statements cannot be lazy. + +When using "lazy from ... import", each imported name is bound to a +lazy proxy object. The first access to any of these names triggers +loading of the entire module and resolves only that specific name to +its actual value. Other names remain as lazy proxies until they are +accessed. + +Example: + + lazy import json + import sys + + print('json' in sys.modules) # False - json module not yet loaded + + # First use triggers loading + result = json.dumps({"hello": "world"}) + + print('json' in sys.modules) # True - now loaded + +If an error occurs during module loading (such as "ImportError" or +"SyntaxError"), it is raised at the point where the lazy import is +first used, not at the import statement itself. + +See **PEP 810** for the full specification of lazy imports. + +Added in version 3.15. + + +Compatibility via "__lazy_modules__" +==================================== + +As an alternative to using the "lazy" keyword, a module can opt into +lazy loading for specific imports by defining a module-level +"__lazy_modules__" variable. When present, it must be a container of +fully qualified module name strings. Any regular (non-"lazy") +"import" statement at module scope whose target appears in +"__lazy_modules__" is treated as a lazy import, exactly as if the +"lazy" keyword had been used. + +This provides a way to enable lazy loading for specific dependencies +without changing individual "import" statements. This is useful when +supporting Python versions older than 3.15 while using lazy imports in +3.15+: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the +lookup, so "__lazy_modules__" must always contain fully qualified +module names. + +For "from"-style imports, the relevant name is the module following +"from", not the names of its members: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or "try"/"except"/"finally" +blocks are always eager, regardless of "__lazy_modules__". + +Setting "-X lazy_imports=none" (or the "PYTHON_LAZY_IMPORTS" +environment variable to "none") overrides "__lazy_modules__" and +forces all imports to be eager. + +Added in version 3.15. ''', 'lists': r'''List displays ************* -A list display is a possibly empty series of expressions enclosed in -square brackets: +A *list display* is a possibly empty series of expressions enclosed in +square brackets. For example: + + >>> ["one", "two", "three"] + ['one', 'two', 'three'] + >>> ["one"] # One-element list + ['one'] + >>> [] # empty list + [] - list_display: "[" [flexible_expression_list | comprehension] "]" +See Container displays for general information on displays. -A list display yields a new list object, the contents being specified -by either a list of expressions or a comprehension. When a comma- -separated list of expressions is supplied, its elements are evaluated -from left to right and placed into the list object in that order. -When a comprehension is supplied, the list is constructed from the -elements resulting from the comprehension. +The formal grammar for list displays is: + + list: '[' [flexible_expression_list] ']' ''', 'naming': r'''Naming and binding ****************** @@ -11059,6 +11154,8 @@ class is used in a class pattern with positional arguments, each not a prefix or suffix; rather, all combinations of its values are stripped. + Whitespace characters are defined by "str.isspace()". + For example: >>> ' spacious '.strip() @@ -12447,7 +12544,7 @@ def foo(): target of assignments or "del" statements. The built-in function "len()" returns the number of items in a mapping. -There is currently a single intrinsic mapping type: +There are two intrinsic mapping types: Dictionaries @@ -12481,6 +12578,18 @@ def foo(): rather than a language guarantee. +Frozen dictionaries +------------------- + +These represent an immutable dictionary. They are created by the +built-in "frozendict()" constructor. A frozendict is *hashable* if +all of its keys and values are hashable, in which case it can be used +as an element of a set, or as a key in another mapping. "frozendict" +is not a subclass of "dict"; it inherits directly from "object". + +Added in version 3.15. + + Callable types ============== diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index e8cef29d00467f7..6c6d9bb6b34244e 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -179,14 +179,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue diff --git a/Lib/shutil.py b/Lib/shutil.py index 45cbe4c855b462b..d3ac5dc5c50a734 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -940,8 +940,8 @@ def move(src, dst, copy_function=copy2): return real_dst def _destinsrc(src, dst): - src = os.path.abspath(src) - dst = os.path.abspath(dst) + src = os.path.realpath(src) + dst = os.path.realpath(dst) if not src.endswith(os.path.sep): src += os.path.sep if not dst.endswith(os.path.sep): diff --git a/Lib/site.py b/Lib/site.py index 52dd9648734c3ec..b7f5c7f0246bc1b 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -154,13 +154,27 @@ def _init_pathinfo(): return d -# Accumulated entry points from .start files across all site-packages -# directories. Execution is deferred until all paths in .pth files have been -# appended to sys.path. Map the .pth/.start file the data is found in to the -# data. -_pending_entrypoints = {} -_pending_syspaths = {} -_pending_importexecs = {} +# PEP 829 implementation notes. +# +# Startup information (.pth and .start file information) can be processed in +# implicit or explicit batches. Implicit batches are self-contained +# site.addsitedir() calls: they create a per-call StartupState, populate it +# from the site directory's .pth and .start files, run process() on it, and +# then throw the state away. +# +# main() needs different semantics: it accumulates state across multiple +# StartupState.addsitedir() calls (user-site plus all global site-packages) so +# that every sys.path extension is visible *before* any startup code (.start +# entry points and .pth import lines) runs. Callers can opt into the same +# behavior by creating a StartupState directly and calling its addsitedir(), +# addusersitepackages(), and addsitepackages() methods, then invoking +# process() once at the end of the batch. +# +# Here's the CRITICAL reentrancy invariant: recursive site.addsitedir() calls +# reached from a .start entry point or an exec'd .pth import line must not +# mutate the StartupState currently being processed. Reentrant calls reach +# the module-level site.addsitedir() shim, which always builds a fresh +# per-call state. def _read_pthstart_file(sitedir, name, suffix): @@ -194,13 +208,13 @@ def _read_pthstart_file(sitedir, name, suffix): return None, filename try: - # Accept BOM markers in .start and .pth files as we do in source files (Windows PowerShell - # 5.1 makes it hard to emit UTF-8 files without a BOM). + # Accept BOM markers in .start and .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM). content = raw_content.decode("utf-8-sig") except UnicodeDecodeError: _trace(f"Cannot read {filename!r} as UTF-8.") - # For .pth files only, and then only until Python 3.20, fallback to locale encoding for - # backward compatibility. + # For .pth files only, and then only until Python 3.20, fall back to + # locale encoding for backward compatibility. _warn_future_us( ".pth files decoded to locale encoding as a fallback", remove=(3, 20) @@ -214,153 +228,342 @@ def _read_pthstart_file(sitedir, name, suffix): return content.splitlines(), filename -def _read_pth_file(sitedir, name, known_paths): - """Parse a .pth file, accumulating sys.path extensions and import lines. +class StartupState: + """Per-batch accumulator for .pth and .start file processing. - Errors on individual lines do not abort processing of the rest of the - file (PEP 829). - """ - lines, filename = _read_pthstart_file(sitedir, name, ".pth") - if lines is None: - return + A StartupState collects sys.path extensions, deprecated .pth import lines, + and .start entry points read from one or more site-packages directories. + Calling process() applies them in PEP 829 order: paths are added to + sys.path first, then import lines from .pth files (skipping any with a + matching .start), then entry points from .start files. - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue + State lives entirely on the instance; there is no module-level pending + state. This is what makes the module reentrancy-safe: a site.addsitedir() + call reached recursively from an exec'd import line or a .start entry + point operates on a different StartupState than the one being processed by + the outer call. - # In Python 3.18 and 3.19, `import` lines are silently ignored. In - # Python 3.20 and beyond, issue a warning when `import` lines in .pth - # files are detected. - if line.startswith(("import ", "import\t")): - _warn_future_us( - "import lines in .pth files are silently ignored", - remove=(3, 18) - ) - _warn_future_us( - "import lines in .pth files are noisily ignored", - remove=(3, 20) - ) - _pending_importexecs.setdefault(filename, []).append(line) - continue + The internal data is intentionally private. The lower-level write + methods (_record_sitedir(), _read_pth_file(), _read_start_file()) are + private to the site module; the public surface is addsitedir(), + addusersitepackages(), addsitepackages(), and process(). + """ + __slots__ = ( + '_known_paths', + '_processed_sitedirs', + '_path_entries', + '_importexecs', + '_entrypoints', + ) + def __init__(self, known_paths=None): + """Create an independent startup state. + + *known_paths* is a set of case-normalized paths already present + on sys.path, used to avoid duplicate path entries. When None + (the default), it is initialized from the current sys.path. + + A caller-supplied set is stored by reference and mutated in place + as new paths are recorded; pass a fresh set per StartupState if + isolation across instances is required. + """ + self._known_paths = ( + _init_pathinfo() + if known_paths is None + else known_paths) + self._processed_sitedirs = set() + # The sys.path append ledger. This is a list of 2-tuples of the form + # (pthfile, path) where `pthfile` is the .pth file which is extending + # the path, and `path` is the directory to add to sys.path. Note that + # to preserve the interleaving semantics (i.e. .pth file paths are + # added after the sitedir in which the .pth file is found), `path` + # could be a sitedir, in which case `pthfile` will always be None. + self._path_entries = [] + # Both dicts map "" -> list + # of items collected from that file. Mapping by filename lets us + # cross-reference a .pth and its matching .start (PEP 829 import + # suppression rule) and lets _print_error report the source file + # when an entry fails. + self._importexecs = {} + self._entrypoints = {} + + def addsitedir(self, sitedir): + """Add a site directory and accumulate its .pth and .start startup data. + + Read the .pth and .start files in *sitedir* and record their + sys.path extensions, deprecated .pth import lines, and .start entry + points on this state. The recorded data is not applied until + process() is called. + + Typically used to batch multiple site directories before a single + process() call, so that every sys.path extension is visible before + any startup code runs. Reentrant calls reached from a .start entry + point or an exec'd .pth import line must not mutate the state + currently being processed; for those cases, use site.addsitedir() + instead, which always creates a fresh per-call state. + """ + self._addsitedir(sitedir, process_known_sitedirs=True) + + def addusersitepackages(self): + """Add the per-user site-packages directory, if enabled. + + The user site directory is added only when user site-packages are + enabled and the directory exists. Its startup data is accumulated + for later processing by process(). + """ + _trace("Processing user site-packages") + user_site = getusersitepackages() + if ENABLE_USER_SITE and os.path.isdir(user_site): + self.addsitedir(user_site) + + def addsitepackages(self, prefixes=None): + """Add global site-packages directories, if they exist. + + Site-packages directories are computed from *prefixes*, or from the + global PREFIXES when *prefixes* is None. Each directory's startup + data is accumulated for later processing by process(). + """ + _trace("Processing global site-packages") + for sitedir in getsitepackages(prefixes): + if os.path.isdir(sitedir): + self.addsitedir(sitedir) + + def _addsitedir(self, sitedir, *, process_known_sitedirs): + """Internal addsitedir() implementation with full dedup control. + + The public addsitedir() always uses process_known_sitedirs=True + (gh-149819 semantics). The module-level legacy known_paths shim + uses process_known_sitedirs=False to preserve 3.14 idempotency + (gh-75723). + """ + sitedir = self._record_sitedir( + sitedir, process_known_sitedirs=process_known_sitedirs) + if sitedir is None: + return try: - dir_, dircase = makepath(sitedir, line) - except Exception as exc: - _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) - continue - - if dircase in known_paths: - _trace(f"In {filename!r}, line {n:d}: " - f"skipping duplicate sys.path entry: {dir_}") - else: - _pending_syspaths.setdefault(filename, []).append(dir_) - known_paths.add(dircase) - - -def _read_start_file(sitedir, name): - """Parse a .start file for a list of entry point strings.""" - lines, filename = _read_pthstart_file(sitedir, name, ".start") - if lines is None: - return - - # PEP 829: the *presence* of a matching .start file disables `import` - # line processing in the matched .pth file, regardless of whether the - # .start file produced any entry points. Register the filename as a - # key now so an empty (or comment-only) .start file still suppresses. - entrypoints = _pending_entrypoints.setdefault(filename, []) - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - # Syntax validation is deferred to entry-point execution time, - # where pkgutil.resolve_name(strict=True) enforces the - # pkg.mod:callable form. - entrypoints.append(line) - - -def _extend_syspath(): - # We've already filtered out duplicates, either in the existing sys.path - # or in all the .pth files we've seen. We've also abspath/normpath'd all - # the entries, so all that's left to do is to ensure that the path exists. - for filename, dirs in _pending_syspaths.items(): - for dir_ in dirs: - if os.path.exists(dir_): - _trace(f"Extending sys.path with {dir_} from {filename}") - sys.path.append(dir_) - else: - _print_error( - f"In {filename}: {dir_} does not exist; " - f"skipping sys.path append") - + names = os.listdir(sitedir) + except OSError: + return -def _exec_imports(): - # For all the `import` lines we've seen in .pth files, exec() them in - # order. However, if they come from a file with a matching .start, then - # we ignore these import lines. For the ones we do process, print a - # warning but only when -v was given. - for filename, imports in _pending_importexecs.items(): - name, dot, pth = filename.rpartition(".") - assert dot == "." and pth == "pth", f"Bad startup filename: {filename}" + # The following phases are defined by PEP 829. + # Phases 1-3: Read .pth files, accumulating paths and import lines. + pth_names = sorted( + name for name in names + if name.endswith(".pth") and not name.startswith(".") + ) + for name in pth_names: + self._read_pth_file(sitedir, name) + + # Phases 6-7: Discover .start files and accumulate their entry points. + # Import lines from .pth files with a matching .start file are + # discarded at flush time by _exec_imports(). + start_names = sorted( + name for name in names + if name.endswith(".start") and not name.startswith(".") + ) + for name in start_names: + self._read_start_file(sitedir, name) - if f"{name}.start" in _pending_entrypoints: - # Skip import lines in favor of entry points. - continue + def _record_sitedir(self, sitedir, *, process_known_sitedirs=True): + sitedir, sitedircase = makepath(sitedir) + # Have we already processed this sitedir? + if sitedircase in self._processed_sitedirs: + return None + # In legacy known_paths mode, a known sitedir means its startup files + # were already processed by an earlier addsitedir() call, so skip it + # to preserve idempotency (gh-75723). In explicit StartupState mode, + # known_paths only tracks sys.path entries; a sitedir may already be + # on sys.path (for example from $PYTHONPATH, gh-149819) but still need + # its .pth and .start files processed once. The separate + # _processed_sitedirs set is what lets explicit batches distinguish + # "already on sys.path" from "startup files already read". + if not process_known_sitedirs and sitedircase in self._known_paths: + return None + # Record that we've processed this sitedir. + self._processed_sitedirs.add(sitedircase) + if sitedircase not in self._known_paths: + self._known_paths.add(sitedircase) + # Add the sitedir to the sys.path extension ledger. There is no + # .pth file to record. + self._path_entries.append((None, sitedir)) + return sitedir + + def _read_pth_file(self, sitedir, name): + """Parse a .pth file, accumulating sys.path extensions and import lines. + + Errors on individual lines do not abort processing of the rest of + the file (PEP 829). Per-batch deduplication is done against + self._known_paths: any path already in it is skipped, and newly + accepted paths are added to it so that subsequent .pth files in + the same batch don't add them more than once. + """ + lines, filename = _read_pthstart_file(sitedir, name, ".pth") + if lines is None: + return + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): + continue - _trace( - f"import lines in {filename} are deprecated, " - f"use entry points in a {name}.start file instead." - ) + # In Python 3.18 and 3.19, `import` lines are silently + # ignored. In Python 3.20 and beyond, issue a warning when + # `import` lines in .pth files are detected. + if line.startswith(("import ", "import\t")): + _warn_future_us( + "import lines in .pth files are silently ignored", + remove=(3, 18), + ) + _warn_future_us( + "import lines in .pth files are noisily ignored", + remove=(3, 20), + ) + self._importexecs.setdefault(filename, []).append(line) + continue - for line in imports: try: - _trace(f"Exec'ing from {filename}: {line}") - exec(line) + dir_, dircase = makepath(sitedir, line) except Exception as exc: - _print_error( - f"Error in import line from {filename}: {line}", exc) - - -def _execute_start_entrypoints(): - """Execute all accumulated .start file entry points. + _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) + continue - Called after all site-packages directories have been processed so that - sys.path is fully populated before any entry point code runs. Uses - pkgutil.resolve_name(strict=True) which both validates the strict - pkg.mod:callable form and resolves the entry point in one step. - """ - for filename, entrypoints in _pending_entrypoints.items(): - for entrypoint in entrypoints: - try: - _trace(f"Executing entry point: {entrypoint} from {filename}") - callable_ = pkgutil.resolve_name(entrypoint, strict=True) - except ValueError as exc: - _print_error( - f"Invalid entry point syntax in {filename}: " - f"{entrypoint!r}", exc) + # PEP 829 dedup: skip paths already seen in this batch. + if dircase in self._known_paths: + _trace( + f"In {filename!r}, line {n:d}: " + f"skipping duplicate sys.path entry: {dir_}" + ) + else: + # Add this directory to the sys.path extension ledger, while + # also recording the .pth file it was found in. + self._path_entries.append((filename, dir_)) + self._known_paths.add(dircase) + + def _read_start_file(self, sitedir, name): + """Parse a .start file for a list of entry point strings.""" + lines, filename = _read_pthstart_file(sitedir, name, ".start") + if lines is None: + return + + # PEP 829: the *presence* of a matching .start file disables `import` + # line processing in the matched .pth file, regardless of whether this + # .start file contains any entry points. Register the filename as a + # key now so an empty (or comment-only) .start file still suppresses. + entrypoints = self._entrypoints.setdefault(filename, []) + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): continue - except Exception as exc: - _print_error( - f"Error resolving entry point {entrypoint} " - f"from {filename}", exc) + # Syntax validation is deferred to entry point execution + # time, where pkgutil.resolve_name(strict=True) enforces the + # pkg.mod:callable form. + entrypoints.append(line) + + def process(self): + """Apply accumulated state in PEP 829 order. + + Phase order matters: all .pth path extensions are applied to + sys.path *before* any import line or .start entry point runs, so + that an entry point may live in a module reachable only via a + .pth-extended path. + """ + self._extend_syspath() + self._exec_imports() + self._execute_start_entrypoints() + + def _extend_syspath(self): + # Duplicate path-extension specifications have already been filtered + # out upstream across .pth files within this batch (via known_paths), + # and ledger entries are already abspath/normpath'd. .pth-derived + # entries (filename is not None) are existence-checked and skipped + # with an error if missing. Sitedir entries (filename is None) are + # appended unconditionally: legacy addsitedir() added the sitedir to + # sys.path before attempting to list it, so an unreadable or + # non-existent sitedir still landed on sys.path. Deferring the + # append to here preserves that contract. + for filename, dir_ in self._path_entries: + # As a backstop, known_paths may not have been seeded from sys.path + # (callers can pass an empty set), and multiple StartupState + # instances against the same sys.path don't share state, so always + # do a final anti-duplication check. + if dir_ in sys.path: continue - try: - callable_() - except Exception as exc: + if filename is None or os.path.exists(dir_): + if filename is not None: + _trace(f"Extending sys.path with {dir_} from {filename}") + sys.path.append(dir_) + else: _print_error( - f"Error in entry point {entrypoint} from {filename}", - exc) - + f"In {filename}: {dir_} does not exist; " + f"skipping sys.path append" + ) + + def _exec_imports(self): + # For each `import` line we've seen in a .pth file, exec() it in + # order, unless the .pth has a matching .start file in this same + # batch. In that case, PEP 829 says the import lines are + # suppressed in favor of the .start's entry points. + for filename, imports in self._importexecs.items(): + # Given "/path/to/foo.pth", check whether "/path/to/foo.start" was + # registered in this same batch. + name, dot, pth = filename.rpartition(".") + assert dot == "." and pth == "pth", ( + f"Bad startup filename: {filename}" + ) + if f"{name}.start" in self._entrypoints: + _trace( + f"import lines in {filename} are suppressed " + f"due to matching {name}.start file." + ) + continue -def process_startup_files(): - """Flush all pending sys.path and entry points.""" - _extend_syspath() - _exec_imports() - _execute_start_entrypoints() - _pending_syspaths.clear() - _pending_importexecs.clear() - _pending_entrypoints.clear() + _trace( + f"import lines in {filename} are deprecated, " + f"use entry points in a {name}.start file instead." + ) + for line in imports: + try: + _trace(f"Exec'ing from {filename}: {line}") + exec(line) + except Exception as exc: + _print_error( + f"Error in import line from {filename}: {line}", + exc, + ) + + def _execute_start_entrypoints(self): + # Resolve each entry point string to a callable via + # pkgutil.resolve_name(strict=True), which both validates the + # required pkg.mod:callable form and performs the import in one + # step, then call it with no arguments. + for filename, entrypoints in self._entrypoints.items(): + for entrypoint in entrypoints: + try: + _trace( + f"Executing entry point: {entrypoint} from {filename}" + ) + callable_ = pkgutil.resolve_name(entrypoint, strict=True) + except ValueError as exc: + _print_error( + f"Invalid entry point syntax in {filename}: " + f"{entrypoint!r}", + exc, + ) + except Exception as exc: + _print_error( + f"Error resolving entry point {entrypoint} " + f"from {filename}", + exc, + ) + else: + try: + callable_() + except Exception as exc: + _print_error( + f"Error in entry point {entrypoint} from {filename}", + exc, + ) def addpackage(sitedir, name, known_paths): @@ -370,60 +573,31 @@ def addpackage(sitedir, name, known_paths): reset = True else: reset = False - _read_pth_file(sitedir, name, known_paths) - process_startup_files() - if reset: - known_paths = None - return known_paths + state = StartupState(known_paths) + state._read_pth_file(sitedir, name) + state.process() -def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False): - """Add 'sitedir' argument to sys.path if missing and handle startup - files.""" - _trace(f"Adding directory: {sitedir!r}") - if known_paths is None: - known_paths = _init_pathinfo() - reset = True - else: - reset = False - sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - known_paths.add(sitedircase) - try: - names = os.listdir(sitedir) - except OSError: - return - - # The following phases are defined by PEP 829. - # Phases 1-3: Read .pth files, accumulating paths and import lines. - pth_names = sorted( - name for name in names - if name.endswith(".pth") and not name.startswith(".") - ) - for name in pth_names: - _read_pth_file(sitedir, name, known_paths) - - # Phases 6-7: Discover .start files and accumulate their entry points. - # Import lines from .pth files with a matching .start file are discarded - # at flush time by _exec_imports(). - start_names = sorted( - name for name in names - if name.endswith(".start") and not name.startswith(".") - ) - for name in start_names: - _read_start_file(sitedir, name) + return None if reset else known_paths - # Generally, when addsitedir() is called explicitly, we'll want to process - # all the startup file data immediately. However, when called through - # main(), we'll want to batch up all the startup file processing. main() - # will set this flag to True to defer processing. - if not defer_processing_start_files: - process_startup_files() - if reset: - known_paths = None +def addsitedir(sitedir, known_paths=None): + """Add a site directory and process its startup files. + For batched processing across multiple site directories, build a + StartupState explicitly and call StartupState.addsitedir() on it; that + defers .pth/.start processing until a single StartupState.process() call. + """ + _trace(f"Adding directory: {sitedir!r}") + if known_paths is None: + state = StartupState(_init_pathinfo()) + state.addsitedir(sitedir) + else: + # Preserve gh-75723 idempotency for legacy known_paths mode: a + # sitedir already present in known_paths is skipped, not reprocessed. + state = StartupState(known_paths) + state._addsitedir(sitedir, process_known_sitedirs=False) + state.process() return known_paths @@ -536,21 +710,20 @@ def getusersitepackages(): return USER_SITE -def addusersitepackages(known_paths, *, defer_processing_start_files=False): - """Add a per user site-package to sys.path - Each user has its own python directory with site-packages in the - home directory. - """ - # get the per user site-package path - # this call will also make sure USER_BASE and USER_SITE are set - _trace("Processing user site-packages") - user_site = getusersitepackages() +def addusersitepackages(known_paths): + """Add the per-user site-packages directory, if enabled. - if ENABLE_USER_SITE and os.path.isdir(user_site): - addsitedir(user_site, known_paths, defer_processing_start_files=defer_processing_start_files) + The user site directory is added only when user site-packages are enabled + and the directory exists. Return *known_paths*, updated with any paths + added by addsitedir(). + """ + state = StartupState(known_paths) + state.addusersitepackages() + state.process() return known_paths + def getsitepackages(prefixes=None): """Returns a list containing all global site-packages directories. @@ -590,15 +763,20 @@ def getsitepackages(prefixes=None): sitepackages.append(os.path.join(prefix, "Lib", "site-packages")) return sitepackages -def addsitepackages(known_paths, prefixes=None, *, defer_processing_start_files=False): - """Add site-packages to sys.path""" - _trace("Processing global site-packages") - for sitedir in getsitepackages(prefixes): - if os.path.isdir(sitedir): - addsitedir(sitedir, known_paths, defer_processing_start_files=defer_processing_start_files) +def addsitepackages(known_paths, prefixes=None): + """Add global site-packages directories, if they exist. + + Site-packages directories are computed from *prefixes*, or from the global + prefixes when *prefixes* is None. Return *known_paths*, updated with any + paths added by addsitedir(). + """ + state = StartupState(known_paths) + state.addsitepackages(prefixes) + state.process() return known_paths + def setquit(): """Define new builtins 'quit' and 'exit'. @@ -765,6 +943,15 @@ def write_history(): def venv(known_paths): + """Process pyvenv.cfg and add the venv site-packages, if applicable.""" + state = StartupState(known_paths) + _venv(state) + state.process() + return known_paths + + +def _venv(state): + """State-driven implementation of venv(); used by main() for batching.""" global PREFIXES, ENABLE_USER_SITE env = os.environ @@ -804,20 +991,22 @@ def venv(known_paths): sys._home = value if sys.prefix != site_prefix: - _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + _warn( + f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', + RuntimeWarning) if sys.exec_prefix != site_prefix: - _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) + _warn( + f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', + RuntimeWarning) - # Doing this here ensures venv takes precedence over user-site - addsitepackages(known_paths, [sys.prefix]) + # Doing this here ensures venv takes precedence over user-site. + state.addsitepackages([sys.prefix]) if system_site == "true": PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: ENABLE_USER_SITE = False - return known_paths - def execsitecustomize(): """Run custom site specific code, if available.""" @@ -868,24 +1057,25 @@ def main(): global ENABLE_USER_SITE orig_path = sys.path[:] - known_paths = removeduppaths() + removeduppaths() if orig_path != sys.path: # removeduppaths() might make sys.path absolute. # Fix __file__ of already imported modules too. abs_paths() - known_paths = venv(known_paths) + state = StartupState(set()) + _venv(state) + if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() - known_paths = addusersitepackages(known_paths, defer_processing_start_files=True) - known_paths = addsitepackages(known_paths, defer_processing_start_files=True) + + state.addusersitepackages() + state.addsitepackages() # PEP 829: flush accumulated data from all .pth and .start files. # Paths are extended first, then deprecated import lines are exec'd, # and finally .start entry points are executed — ensuring sys.path is - # fully populated before any startup code runs. process_startup_files() - # also clears the pending state so a later addsitedir() call does - # not re-apply already-processed data. - process_startup_files() + # fully populated before any startup code runs. + state.process() setquit() setcopyright() sethelper() diff --git a/Lib/tarfile.py b/Lib/tarfile.py index d0e7dec5575047a..55e4a4e0c9a29c9 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -380,7 +380,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("bz2 module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() self.exception = OSError else: @@ -392,7 +391,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("lzma module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = lzma.LZMADecompressor() self.exception = lzma.LZMAError else: @@ -403,7 +401,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("compression.zstd module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = zstd.ZstdDecompressor() self.exception = zstd.ZstdError else: @@ -485,7 +482,6 @@ def _init_read_gz(self): """Initialize for reading a gzip compressed fileobj. """ self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" # taken from gzip.GzipFile with some alterations if self.__read(2) != b"\037\213": @@ -498,7 +494,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) @@ -543,26 +539,44 @@ def _read(self, size): if self.comptype == "tar": return self.__read(size) - c = len(self.dbuf) - t = [self.dbuf] + c = 0 + t = [] while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" + if self.comptype == "gz": + # zlib interface is different than others. + # It returns data in unconsumed_tail. + if self.buf: + cbuf = self.buf + self.buf = b"" + else: + cbuf = self.fileobj.read(self.bufsize) + if not cbuf: + break + + try: + dbuf = self.cmp.decompress(cbuf, size - c) + self.buf = self.cmp.unconsumed_tail + except self.exception as e: + raise ReadError("invalid compressed data") from e else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] + # Other decompressors have needs_input. + # decompress() can buffer data internally. + if self.cmp.needs_input: + cbuf = self.fileobj.read(self.bufsize) + if not cbuf: + break + else: + cbuf = b"" + + try: + dbuf = self.cmp.decompress(cbuf, size - c) + except self.exception as e: + raise ReadError("invalid compressed data") from e + + t.append(dbuf) + c += len(dbuf) + + return b"".join(t) def __read(self, size): """Return size bytes from stream. If internal buffer is empty, @@ -830,16 +844,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: @@ -893,11 +913,14 @@ class TarInfo(object): size = 'Size in bytes.', mtime = 'Time of last modification.', chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), + type = ('File type. type is usually one of these constants: ' + 'REGTYPE,\n' + 'AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' + 'CONTTYPE, CHRTYPE,\n' + 'BLKTYPE, GNUTYPE_SPARSE.'), linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), + 'in TarInfo\n' + 'objects of type LNKTYPE and SYMTYPE.'), uname = 'User name.', gname = 'Group name.', devmajor = 'Device major number.', @@ -905,7 +928,8 @@ class TarInfo(object): offset = 'The tar header starts here.', offset_data = "The file's data starts here.", pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), + 'associated pax\n' + 'extended header.'), sparse = 'Sparse member information.', _tarfile = None, _sparse_structs = None, @@ -2269,10 +2293,11 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If 'verbose' is False, only - the names of the members are printed. If it is True, an 'ls -l'-like - output is produced. 'members' is optional and must be a subset of the - list returned by getmembers(). + """Print a table of contents to sys.stdout. + + If 'verbose' is False, only the names of the members are printed. + If it is True, an 'ls -l'-like output is produced. 'members' is + optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, @@ -2363,10 +2388,12 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents - a non zero-size regular file, the 'fileobj' argument should be a binary file, - and tarinfo.size bytes are read from it and added to the archive. - You can create TarInfo objects directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. + + If 'tarinfo' represents a non zero-size regular file, the 'fileobj' + argument should be a binary file, and tarinfo.size bytes are read + from it and added to the archive. You can create TarInfo objects + directly, or by using gettarinfo(). """ self._check("awx") diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a893932169a089b..8be5bf8aa4f5469 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -208,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -225,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -258,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 958a915626ad241..d0759d2626989d6 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -145,7 +145,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2cac70f4ab2afbe..f9601655dfe157a 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -3323,3 +3323,18 @@ def control_characters_c0() -> list[str]: _ROOT_IN_POSIX = hasattr(os, 'geteuid') and os.geteuid() == 0 requires_root_user = unittest.skipUnless(_ROOT_IN_POSIX, "test needs root privilege") requires_non_root_user = unittest.skipIf(_ROOT_IN_POSIX, "test needs non-root account") + + +STATUS_DLL_INIT_FAILED = 0xC0000142 +def skip_on_low_desktop_heap_memory_subprocess(returncode): + if sys.platform not in ('win32', 'cygwin'): + return + # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could + # come from any of the DLLs being loaded when a new Python process is + # created. In practice, it's likely a memory allocation failure in the + # desktop heap memory which caused the DLL init failure, especially on + # process created with CREATE_NEW_CONSOLE creation flag. See the article: + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory + if returncode == STATUS_DLL_INIT_FAILED: + raise unittest.SkipTest('gh-150436: DLL init failed, likely because ' + 'of low desktop heap memory') diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 4ea5b6f53a04265..1dc3f538f4ad8ba 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -140,6 +140,48 @@ def test_parse_args(self): ) +class TestArgumentParserCopiable(unittest.TestCase): + def _get_parser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + return parser + + @force_not_colorized + def test_copiable(self): + import copy + parser = self._get_parser() + parser2 = copy.copy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test shallow copy also gets new arguments + parser.add_argument("--extra") + ns3 = parser2.parse_args(["--extra", "bar"]) + self.assertEqual(ns3.extra, "bar") + + @force_not_colorized + def test_deepcopiable(self): + import copy + parser = self._get_parser() + parser2 = copy.deepcopy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test deep copy does not get new arguments + parser.add_argument("--extra") + with self.assertRaises(argparse.ArgumentError): + parser2.parse_args(["--extra", "bar"]) + + class TestArgumentParserPickleable(unittest.TestCase): @force_not_colorized @@ -7863,12 +7905,25 @@ def fake_can_colorize(*, file=None): def test_fake_color_theme_matches_real(self): from argparse import _colorless_theme + + # Check the attributes match those of the 'real' theme _colorize_nocolor = _colorize.get_theme(force_no_color=True).argparse for k in _colorize_nocolor: self.assertEqual( getattr(_colorless_theme, k), getattr(_colorize_nocolor, k) ) + def test_fake_color_theme_raises(self): + from argparse import _colorless_theme + + # Make sure the _colorless_theme doesn't return empty strings + # for magic methods or private attributes + with self.assertRaises(AttributeError): + _colorless_theme.__unknown_dunder__ + + with self.assertRaises(AttributeError): + _colorless_theme._private_attribute + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index dcd963b3355ef86..7afd7de3bb936e6 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -228,6 +228,61 @@ def test_sock_sendfile_zero_size(self): self.assertEqual(ret, 0) self.assertEqual(self.file.tell(), 0) + def check_sock_sendfile_offset(self, data, offset, force_fallback=False): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + f.write(data) + f.flush() + self.assertEqual(f.tell(), len(data)) + + if force_fallback: + async def _sock_sendfile_fail(sock, file, offset, count): + raise asyncio.exceptions.SendfileNotAvailableError() + with support.swap_attr(self.loop, '_sock_sendfile_native', _sock_sendfile_fail): + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + else: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + + self.assertEqual(f.tell(), len(data)) + + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(data) - offset) + + + def test_sock_sendfile_offset(self): + data = b'abcdef' + for offset in (0, len(data) // 2, len(data)): + for force_fallback in (False, True): + with self.subTest(offset=offset, force_fallback=force_fallback): + self.check_sock_sendfile_offset(data, offset, force_fallback) + + def check_sendfile_offset(self, offset, fallback): + srv_proto, cli_proto = self.prepare_sendfile() + self.file.seek(123) + coro = self.loop.sendfile(cli_proto.transport, self.file, offset, fallback=fallback) + try: + ret = self.run_loop(coro) + except asyncio.SendfileNotAvailableError: + if fallback: + raise + cli_proto.transport.close() + self.run_loop(srv_proto.done) + return + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA) - offset) + self.assertEqual(srv_proto.nbytes, len(self.DATA) - offset) + self.assertEqual(srv_proto.data, self.DATA[offset:]) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_offset(self): + for offset in (0, len(self.DATA) // 2, len(self.DATA)): + for fallback in (False, True): + with self.subTest(offset=offset, fallback=fallback): + self.check_sendfile_offset(offset, fallback) + def test_sock_sendfile_mix_with_regular_send(self): buf = b"mix_regular_send" * (4 * 1024) # 64 KiB sock, proto = self.prepare_socksendfile() diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index f9ee2f4f68150a1..509697613475953 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 81967fb8a837404..d62a3a4f17f85e3 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1956,16 +1956,33 @@ def test_sentinel(self): with self.assertRaises(TypeError): class SubSentinel(sentinel): pass + + def test_sentinel_attributes(self): + missing = sentinel("MISSING") with self.assertRaises(TypeError): sentinel.attribute = "value" with self.assertRaises(AttributeError): - missing.__name__ = "CHANGED" + missing.attribute = "value" with self.assertRaises(AttributeError): - missing.__module__ = "changed" + missing.__name__ = "CHANGED" + missing.__module__ = "changed" + self.assertEqual(missing.__module__, "changed") with self.assertRaises(AttributeError): del missing.__name__ + del missing.__module__ with self.assertRaises(AttributeError): - del missing.__module__ + missing.__module__ + + def test_sentinel_repr(self): + with_repr = sentinel("WITH_REPR", repr="custom") + without_repr = sentinel("WITHOUT_REPR", repr=None) + self.assertEqual(repr(with_repr), "custom") + self.assertEqual(repr(without_repr), "WITHOUT_REPR") + self.assertEqual(str(with_repr), "custom") + self.assertEqual(str(without_repr), "WITHOUT_REPR") + + with self.assertRaisesRegex(TypeError, "repr.*str or None"): + sentinel("BAD_REPR", repr=42) def test_sentinel_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b1cdbe04765ed0d..e0e8dd4eccfb1b2 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2700,10 +2700,6 @@ def iconcat(b, a): # MODIFIES! b.wait() a += c - def irepeat(b, a): # MODIFIES! - b.wait() - a *= 2 - def subscript(b, a): b.wait() try: assert a[0] != 0xdd @@ -2837,9 +2833,10 @@ def check(funcs, a=None, *args): check([clear] + [repeat] * 10) check([clear] + [iconcat] * 10) - check([clear] + [irepeat] * 10) check([clear] + [ass_subscript] * 10) check([clear] + [repr_] * 10) + # gh-148605: Do not test "a *= 2" since it allocates up to 4 GiB using + # 10 threads # value errors diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index d8e3b671ec229f9..64293d757331d75 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1032,6 +1032,21 @@ def test_failure(self): # Previously, a second call could crash due to internal inconsistency self.assertRaises(Exception, bzd.decompress, self.BAD_DATA * 30) + def test_decompress_after_data_error(self): + data = bytes.fromhex( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegex(OSError, "Invalid data stream"): + bzd.decompress(data) + # Previously, a second call could crash due to internal inconsistency + self.assertFalse(bzd.needs_input) + self.assertFalse(bzd.eof) + with self.assertRaisesRegex(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @support.refcount_test def test_refleaks_in___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_c_stack_unwind.py similarity index 89% rename from Lib/test/test_frame_pointer_unwind.py rename to Lib/test/test_c_stack_unwind.py index faa012c9c00d8f9..91bf44e463473de 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_c_stack_unwind.py @@ -1,3 +1,12 @@ +"""Test in-process C stack unwinders against Python and JIT frames. + +The tests build a recursive Python call stack, ask each _testinternalcapi +unwinder for return addresses, and classify those addresses as Python, JIT, or +other frames. The backends include CPython's manual stack-chain unwinder and +GNU backtrace(), so this module is about in-process C stack unwinding rather +than a single unwind mechanism. GDB integration tests live in test_gdb. +""" + import json import os import platform @@ -20,7 +29,7 @@ STACK_DEPTH = 10 -def _frame_pointers_expected(machine): +def _manual_unwind_expected(machine): _Py_WITH_FRAME_POINTERS = getattr( _testinternalcapi, "_Py_WITH_FRAME_POINTERS", @@ -89,6 +98,21 @@ def _frame_pointers_expected(machine): return None +def _is_arm32_build(): + if sys.maxsize >= 2**32: + return False + + abi = " ".join( + value for value in ( + sysconfig.get_config_var("MULTIARCH"), + sysconfig.get_config_var("HOST_GNU_TYPE"), + sysconfig.get_config_var("SOABI"), + ) + if value + ).lower() + return "arm" in abi + + def _build_stack_and_unwind(unwinder): import operator @@ -180,7 +204,7 @@ def _annotate_unwind_after_executor_free(unwinder_name="gnu_backtrace_unwind"): def _run_unwind_helper(helper_name, unwinder_name, **env): code = ( - f"from test.test_frame_pointer_unwind import {helper_name}; " + f"from test.test_c_stack_unwind import {helper_name}; " f"print({helper_name}({unwinder_name!r}));" ) run_env = os.environ.copy() @@ -220,15 +244,17 @@ def _unwind_after_executor_free_result(unwinder_name, **env): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") -class FramePointerUnwindTests(unittest.TestCase): +class ManualStackUnwindTests(unittest.TestCase): def setUp(self): super().setUp() machine = platform.machine().lower() - expected = _frame_pointers_expected(machine) + expected = _manual_unwind_expected(machine) if expected is None: - self.skipTest(f"unsupported architecture for frame pointer check: {machine}") + self.skipTest( + f"unsupported architecture for manual stack unwind check: {machine}" + ) if expected == "crash": self.skipTest(f"test does crash on {machine}") @@ -236,12 +262,14 @@ def setUp(self): _testinternalcapi.manual_frame_pointer_unwind() except RuntimeError as exc: if "not supported" in str(exc): - self.skipTest("manual frame pointer unwinding not supported on this platform") + self.skipTest( + "manual stack unwinding not supported on this platform" + ) raise self.machine = machine - self.frame_pointers_expected = expected + self.manual_unwind_expected = expected - def test_manual_unwind_respects_frame_pointers(self): + def test_manual_unwind_finds_expected_frames(self): jit_available = hasattr(sys, "_jit") and sys._jit.is_available() envs = [({"PYTHON_JIT": "0"}, False)] if jit_available: @@ -253,7 +281,7 @@ def test_manual_unwind_respects_frame_pointers(self): jit_frames = result["jit_frames"] python_frames = result.get("python_frames", 0) jit_backend = result.get("jit_backend") - if self.frame_pointers_expected: + if self.manual_unwind_expected: self.assertGreaterEqual( python_frames, STACK_DEPTH, @@ -295,6 +323,10 @@ def test_manual_unwind_respects_frame_pointers(self): @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") @unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux") +@unittest.skipIf( + _is_arm32_build(), + "GNU backtrace unwinding skipped on Arm 32-bit", +) class GnuBacktraceUnwindTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 635deaa73f7efab..e5c50902a0118d4 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -71,6 +71,8 @@ def test_pysentinel_new(self): self.assertIs(type(marker), sentinel) self.assertTrue(_testcapi.pysentinel_check(marker)) self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertTrue(_testcapi.pysentinel_checkexact(marker)) + self.assertFalse(_testcapi.pysentinel_checkexact(object())) self.assertEqual(marker.__name__, "CAPI_SENTINEL") self.assertEqual(marker.__module__, __name__) self.assertEqual(repr(marker), "CAPI_SENTINEL") @@ -80,6 +82,12 @@ def test_pysentinel_new(self): self.assertEqual(no_module.__name__, "NO_MODULE") self.assertIs(no_module.__module__, None) + with_repr = _testcapi.pysentinel_new("WITH_REPR", __name__, "custom repr") + self.assertIs(type(with_repr), sentinel) + self.assertEqual(with_repr.__name__, "WITH_REPR") + self.assertEqual(with_repr.__module__, __name__) + self.assertEqual(repr(with_repr), "custom repr") + globals()["CAPI_SENTINEL"] = marker self.addCleanup(globals().pop, "CAPI_SENTINEL", None) self.assertIs(pickle.loads(pickle.dumps(marker)), marker) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d80fec9a8a0d2b0..9f0427172b5048e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -598,7 +598,8 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + self.assertNotIn("_ITER_NEXT_INLINE", uops) @requires_specialization @@ -5921,6 +5922,44 @@ def __next__(self): """), PYTHON_JIT="1", PYTHON_JIT_STRESS="1") self.assertEqual(result[0].rc, 0, result) + def test_149335_trace_buffer_guard(self): + # https://github.com/python/cpython/issues/149335 + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import sys + + def f1(): + for i_3178 in 0, 2, 10: + mv162 = 162 + + mv3 = mv1 = mv_165 = mv16 = \ + mv167 = mv168 = \ + mv169 = \ + mv_1403_170 = \ + 169 + + mv_1403_170 + + mv_172 = mv_3 = mv_4 = mv175 = mv176 = mv17 = mv178 = mv179 = mv0 = mv1 = mv182 = ( + mv3 + ) = mv4 = mv185 = mv186 = mv187 = mv18 = mv189 = mv0 = mv1 = mv192 = mv3 = mv4 = ( + mv195 + ) = mv196 = mv197 = mv_198 = mv19 = mv0 = mv1 = mv2 = mv3 = mv4 = mv05 = mv06 = ( + mv07 + ) = mv08 = mv09 = mv0 = mv1 = mv2 = mv3 = mv4 = mv15 = mv16 = mv17 = mv18 = mv19 = ( + mv0 + ) = mv1 = mv_2 = mv3 = mv4 = mv_25 = mv_26 = mv_27 = mv_28 = mv_29 = mv0 = mv1 = ( + mv2 + ) = mv_1403 = mv4 = mv35 = mv36 = mv37 = mv38 = mv39 = mv0 = -sys.maxsize / 3 + + mv1 = mv_12 = mv3 = mv_14 = mv45 = sys.float_info.epsilon + mv46 = sys.float_info.epsilon + + for i in range(15000): + f1() + """), PYTHON_JIT="1") + self.assertEqual(result[0].rc, 0, result) + def test_144068_daemon_thread_jit_cleanup(self): result = script_helper.run_python_until_end('-c', textwrap.dedent(""" import threading @@ -6137,6 +6176,20 @@ def __init__(self, x): C(0) if i else str(0) """)) + def test_load_special_type_guard_deopt(self): + script_helper.assert_python_ok("-s", "-c", textwrap.dedent(f""" + def f1(): + class Context: + def __enter__(self): ... + def __exit__(self, e, v, t): ... + + with Context(): + pass + + for _ in range({TIER2_THRESHOLD + 5}): + f1() + """), PYTHON_JIT="1") + def global_identity(x): return x diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7f9e44d70001b75..3b556ec31445dfb 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1036,6 +1036,7 @@ def test_python_legacy_windows_stdio(self): p = subprocess.run([sys.executable, "-c", code], creationflags=subprocess.CREATE_NEW_CONSOLE, env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. @@ -1044,6 +1045,7 @@ def test_python_legacy_windows_stdio(self): p = subprocess.run([sys.executable, "-c", code], creationflags=subprocess.CREATE_NEW_CONSOLE, env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 4e5362111687477..f4bf5a66ad21550 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -157,6 +157,10 @@ def ignore_byteswarning(): {}, {1: 7, 2: 8, 3: 9}, {1: [1], 2: (2,), 3: {3: 4}}, + # frozendict + frozendict(), + frozendict({1: 7, 2: 8, 3: 9}), + frozendict({1: [1], 2: (2,), 3: {3: 4}, 4: frozendict({5: 6})}), # set set(), {1, 2, 3}, diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 46f8ff93efa9152..6dadb7b410d7034 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,6 +1,7 @@ import sys import unittest import test.support +import ctypes from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, @@ -247,6 +248,13 @@ def test_parameter_repr(self): self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + if hasattr(ctypes, 'c_double_complex'): + self.assertRegex(repr(ctypes.c_double_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_float_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_longdouble_complex.from_param(0)), + r"^$") @test.support.cpython_only def test_from_param_result_refcount(self): diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py index a193eb10f16d178..cc78f01e3e2ebdc 100644 --- a/Lib/test/test_defaultdict.py +++ b/Lib/test/test_defaultdict.py @@ -186,6 +186,18 @@ def test_union(self): with self.assertRaises(TypeError): i |= None + # frozendict + i_fd = i | frozendict(s) + self.assertIs(type(i_fd), defaultdict) + self.assertIs(i_fd.default_factory, int) + self.assertDictEqual(i_fd, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(i_fd), [1, 2, 0]) + + fd_i = frozendict(s) | i + self.assertIs(type(fd_i), frozendict) + self.assertEqual(fd_i, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(fd_i), [0, 1, 2]) + def test_factory_conflict_with_set_value(self): key = "conflict_test" count = 0 diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 4e1a489205a6855..3c45032cda91387 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -287,6 +287,22 @@ def test_index(self): else: self.assertEqual(d.index(element, start, stop), target) + # Test stop argument + for elem in d: + index = d.index(elem) + self.assertEqual( + index, + d.index(elem, 0), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d)), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d) + 100), + ) + # Test large start argument d = deque(range(0, 10000, 10)) for step in range(100): diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 425fb85e93558d2..828440a993a975d 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -168,56 +168,54 @@ def merge(self, other): >>> import pprint >>> pprint.pprint(dir(list)) # like list.__dict__.keys(), but sorted - [ - '__add__', - '__class__', - '__class_getitem__', - '__contains__', - '__delattr__', - '__delitem__', - '__dir__', - '__doc__', - '__eq__', - '__format__', - '__ge__', - '__getattribute__', - '__getitem__', - '__getstate__', - '__gt__', - '__hash__', - '__iadd__', - '__imul__', - '__init__', - '__init_subclass__', - '__iter__', - '__le__', - '__len__', - '__lt__', - '__mul__', - '__ne__', - '__new__', - '__reduce__', - '__reduce_ex__', - '__repr__', - '__reversed__', - '__rmul__', - '__setattr__', - '__setitem__', - '__sizeof__', - '__str__', - '__subclasshook__', - 'append', - 'clear', - 'copy', - 'count', - 'extend', - 'index', - 'insert', - 'pop', - 'remove', - 'reverse', - 'sort', - ] + ['__add__', + '__class__', + '__class_getitem__', + '__contains__', + '__delattr__', + '__delitem__', + '__dir__', + '__doc__', + '__eq__', + '__format__', + '__ge__', + '__getattribute__', + '__getitem__', + '__getstate__', + '__gt__', + '__hash__', + '__iadd__', + '__imul__', + '__init__', + '__init_subclass__', + '__iter__', + '__le__', + '__len__', + '__lt__', + '__mul__', + '__ne__', + '__new__', + '__reduce__', + '__reduce_ex__', + '__repr__', + '__reversed__', + '__rmul__', + '__setattr__', + '__setitem__', + '__sizeof__', + '__str__', + '__subclasshook__', + 'append', + 'clear', + 'copy', + 'count', + 'extend', + 'index', + 'insert', + 'pop', + 'remove', + 'reverse', + 'sort'] The new introspection API gives more information than the old one: in addition to the regular methods, it also shows the methods that are diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b2f4363b23e7480..f26586809238f0e 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1868,6 +1868,11 @@ def test_merge(self): self.assertEqual(fd | {}, fd) self.assertEqual(frozendict() | fd, fd) + # gh-149676: Test hash(frozendict | frozendict) + a = frozendict({"a": 1}) + b = frozendict({"b": 2}) + self.assertEqual(hash(a | b), hash(frozendict({"a": 1, "b": 2}))) + def test_update(self): # test "a |= b" operator d = frozendict(x=1) @@ -1898,10 +1903,35 @@ def test_hash(self): self.assertEqual(hash(frozendict(x=1, y=2)), hash(frozendict(y=2, x=1))) + # Check that hash() computes the hash of (key, value) pairs + cases = [ + frozendict(a=False, b=True, c=True), + frozendict(a=True, b=False, c=True), + frozendict(a=True, b=True, c=False), + frozendict({False: "a", "b": True, "c": True}), + frozendict({"a": "b", False: True, True: "c"}), + ] + hashes = {hash(fd) for fd in cases} + self.assertEqual(len(hashes), len(cases)) + fd = frozendict(x=[1], y=[2]) with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): hash(fd) + @support.cpython_only + def test_hash_cpython(self): + # Check that hash(frozendict) implementation is: + # hash(frozenset(fd.items())) + for fd in ( + frozendict(), + frozendict(x=1, y=2), + frozendict(y=2, x=1), + frozendict(a=False, b=True, c=True), + frozendict.fromkeys('abc'), + ): + with self.subTest(fd=fd): + self.assertEqual(hash(fd), hash(frozenset(fd.items()))) + def test_fromkeys(self): self.assertEqual(frozendict.fromkeys('abc'), frozendict(a=None, b=None, c=None)) diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index aded44e85ee3368..9d9fe418ee4d067 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1060,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -2387,6 +2459,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index ca44f54c69b39bc..59013f087199e3a 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -58,6 +58,62 @@ def test_japanese_codecs(self): # TK: full decode comparison eq(str(h).encode(jcode), subject_bytes) + h = Header("Japanese") + s = '\u65e5\u672c\u8a9e' # 日本語 + h.append(s, Charset('euc-jp')) + h.append(s, Charset('iso-2022-jp')) + h.append(s, Charset('shift_jis')) + eq(h.encode(), """\ +Japanese =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= + =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?=""") + eq(decode_header(h.encode()), + [(b'Japanese ', None), + (b'\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B', 'iso-2022-jp'), + ]) + + def test_chinese_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Chinese") + s = '\u4e2d\u6587' # 中文 + h.append(s, Charset('gb2312')) + h.append(s, Charset('gbk')) + h.append(s, Charset('gb18030')) + h.append(s, Charset('hz')) + h.append(s, Charset('big5')) + h.append(s, Charset('big5hkscs')) + eq(h.encode(), """\ +Chinese =?gb2312?b?1tDOxA==?= =?gbk?b?1tDOxA==?= =?gb18030?b?1tDOxA==?= + =?hz-gb-2312?b?fntWUE5Efn0=?= =?big5?b?pKSk5Q==?= =?big5-hkscs?b?pKSk5Q==?=""") + eq(decode_header(h.encode()), + [(b'Chinese ', None), + (b'\xd6\xd0\xce\xc4', 'gb2312'), + (b'\xd6\xd0\xce\xc4', 'gbk'), + (b'\xd6\xd0\xce\xc4', 'gb18030'), + (b'~{VPND~}', 'hz-gb-2312'), + (b'\xa4\xa4\xa4\xe5', 'big5'), + (b'\xa4\xa4\xa4\xe5', 'big5-hkscs'), + ]) + + def test_korean_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Korean") + s = '\ud55c\uad6d\uc5b4' # 한국어 + h.append(s, Charset('euc-kr')) + h.append(s, Charset('ks_c_5601-1987')) + h.append(s, Charset('cp949')) + h.append(s, Charset('iso-2022-kr')) + h.append(s, Charset('johab')) + eq(h.encode(), """\ +Korean =?euc-kr?b?x9Gxub7u?= =?ks_c_5601-1987?b?x9Gxub7uIMfRsbm+7g==?= + =?iso-2022-kr?b?GyQpQw5HUTE5Pm4P?= =?johab?b?0GWKgrTh?=""") + eq(decode_header(h.encode()), + [(b'Korean ', None), + (b'\xc7\xd1\xb1\xb9\xbe\xee', 'euc-kr'), + (b'\xc7\xd1\xb1\xb9\xbe\xee \xc7\xd1\xb1\xb9\xbe\xee', 'ks_c_5601-1987'), + (b'\x1b$)C\x0eGQ19>n\x0f', 'iso-2022-kr'), + (b'\xd0e\x8a\x82\xb4\xe1', 'johab'), + ]) + def test_payload_encoding_utf8(self): jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index dceb54f15e48f4e..bc0e5d356181591 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -342,6 +342,19 @@ def test_set_text_charset_latin_1(self): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_charset_cp949(self): + m = self._make_message() + content = "\ud55c\uad6d\uc5b4\n\uac02\n" + raw_data_manager.set_content(m, content, charset='cp949') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="ks_c_5601-1987" + Content-Transfer-Encoding: base64 + + x9Gxub7uCoFBCg== + """)) + self.assertEqual(m.get_payload(decode=True).decode('ks_c_5601-1987'), content) + self.assertEqual(m.get_content(), content) + def test_set_text_plain_long_line_heuristics(self): m = self._make_message() content = ("Simple but long message that is over 78 characters" diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7778566492d8f44..19555d87085e176 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -4970,6 +4970,128 @@ def tearDown(self): except KeyError: pass + def test_attributes(self): + from email import charset + c = Charset() + self.assertEqual(c.input_charset, 'us-ascii') + self.assertEqual(c.header_encoding, None) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'us-ascii') + self.assertEqual(c.input_codec, None) + self.assertEqual(c.output_codec, None) + + c = Charset('us-ascii') + self.assertEqual(c.input_charset, 'us-ascii') + self.assertEqual(c.header_encoding, None) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'us-ascii') + self.assertEqual(c.input_codec, None) + self.assertEqual(c.output_codec, None) + + c = Charset('utf8') + self.assertEqual(c.input_charset, 'utf-8') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'utf-8') + self.assertEqual(c.input_codec, 'utf-8') + self.assertEqual(c.output_codec, 'utf-8') + + c = Charset('latin1') + self.assertEqual(c.input_charset, 'iso-8859-1') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'iso-8859-1') + self.assertEqual(c.input_codec, 'iso-8859-1') + self.assertEqual(c.output_codec, 'iso-8859-1') + + c = Charset('latin9') + self.assertEqual(c.input_charset, 'iso-8859-15') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'iso-8859-15') + self.assertEqual(c.input_codec, 'iso-8859-15') + self.assertEqual(c.output_codec, 'iso-8859-15') + + c = Charset('cyrillic') + self.assertEqual(c.input_charset, 'iso-8859-5') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'iso-8859-5') + self.assertEqual(c.input_codec, 'iso-8859-5') + self.assertEqual(c.output_codec, 'iso-8859-5') + + c = Charset('cp1251') + self.assertEqual(c.input_charset, 'windows-1251') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'windows-1251') + self.assertEqual(c.input_codec, 'windows-1251') + self.assertEqual(c.output_codec, 'windows-1251') + + c = Charset('cp1252') + self.assertEqual(c.input_charset, 'windows-1252') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'windows-1252') + self.assertEqual(c.input_codec, 'windows-1252') + self.assertEqual(c.output_codec, 'windows-1252') + + c = Charset('eucjp') + self.assertEqual(c.input_charset, 'euc-jp') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'iso-2022-jp') + self.assertEqual(c.input_codec, 'euc-jp') + self.assertEqual(c.output_codec, 'iso-2022-jp') + + c = Charset('cp949') + self.assertEqual(c.input_charset, 'ks_c_5601-1987') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'ks_c_5601-1987') + self.assertEqual(c.input_codec, 'ks_c_5601-1987') + self.assertEqual(c.output_codec, 'ks_c_5601-1987') + + c = Charset('gb2312') + self.assertEqual(c.input_charset, 'gb2312') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'gb2312') + self.assertEqual(c.input_codec, 'gb2312') + self.assertEqual(c.output_codec, 'gb2312') + + c = Charset('big5') + self.assertEqual(c.input_charset, 'big5') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'big5') + self.assertEqual(c.input_codec, 'big5') + self.assertEqual(c.output_codec, 'big5') + + def test_user_charsets(self): + from email import charset + c = Charset('fake0') + self.assertEqual(c.input_charset, 'fake0') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'fake0') + self.assertEqual(c.input_codec, 'fake0') + self.assertEqual(c.output_codec, 'fake0') + + charset.add_alias('fake1', 'mime-fake') + charset.add_alias('output-mime-fake', 'output-mime-fake-alias') + charset.add_codec('mime-fake', 'fakecodec') + charset.add_codec('output-mime-fake-alias', 'outputfakecodec') + charset.add_charset('mime-fake', charset.QP, None, 'output-mime-fake') + + c = Charset('fake1') + self.assertEqual(c.input_charset, 'mime-fake') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'output-mime-fake-alias') + self.assertEqual(c.input_codec, 'fakecodec') + self.assertEqual(c.output_codec, 'outputfakecodec') + def test_codec_encodeable(self): eq = self.assertEqual # Make sure us-ascii = no Unicode conversion @@ -4995,15 +5117,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. @@ -5017,6 +5132,11 @@ def test_unicode_charset_name(self): self.assertEqual(str(charset), 'us-ascii') self.assertRaises(errors.CharsetError, Charset, 'asc\xffii') + def test_bytes_charset_name(self): + charset = Charset(b'us-ascii') + self.assertEqual(str(charset), 'us-ascii') + self.assertRaises(errors.CharsetError, Charset, b'asc\xffii') + # Test multilingual MIME headers. diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 779457119e8f0ea..e0dcc6b8a519e7d 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -5021,8 +5021,8 @@ class Color(enum.Enum) | __members__ | Returns a mapping of member name->value. | - | This mapping lists all enum members, including aliases. Note that this - | is a read-only view of the internal mapping.""" + | This mapping lists all enum members, including aliases. Note that + | this is a read-only view of the internal mapping.""" expected_help_output_without_docs = """\ Help on class Color in module %s: diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index a29e6cdbbf6c785..6b1529aa173f01c 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -3767,6 +3767,13 @@ def test_get_stats(self): "frames_read_from_cache", "frames_read_from_memory", "frame_cache_hit_rate", + "batched_read_attempts", + "batched_read_successes", + "batched_read_misses", + "batched_read_segments_requested", + "batched_read_segments_completed", + "batched_read_success_rate", + "batched_read_segment_completion_rate", ] for key in expected_keys: self.assertIn(key, stats) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 00f1ab44b0a8fa8..5d322cb444cfb68 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -68,33 +68,20 @@ def testcommon(formatstr, args, output=None, limit=None, overflowok=False): testformat(b_format, args, b_output, limit, overflowok) testformat(ba_format, args, ba_output, limit, overflowok) -def test_exc(formatstr, args, exception, excmsg): - try: - testformat(formatstr, args) - except exception as exc: - if str(exc) == excmsg: - if verbose: - print("yes") - else: - if verbose: print('no') - print('Unexpected ', exception, ':', repr(str(exc))) - raise - except: - if verbose: print('no') - print('Unexpected exception') - raise - else: - raise TestFailed('did not get expected exception: %s' % excmsg) - -def test_exc_common(formatstr, args, exception, excmsg): - # test str and bytes - test_exc(formatstr, args, exception, excmsg) - if isinstance(args, dict): - args = {k.encode('ascii'): v for k, v in args.items()} - test_exc(formatstr.encode('ascii'), args, exception, excmsg) class FormatTest(unittest.TestCase): + def check_exc(self, formatstr, args, exception, excmsg): + with self.assertRaisesRegex(exception, re.escape(excmsg)): + testformat(formatstr, args) + + def check_exc_common(self, formatstr, args, exception, excmsg): + # test str and bytes + self.check_exc(formatstr, args, exception, excmsg) + if isinstance(args, dict): + args = {k.encode('ascii'): v for k, v in args.items()} + self.check_exc(formatstr.encode('ascii'), args, exception, excmsg) + def test_common_format(self): # test the format identifiers that work the same across # str, bytes, and bytearrays (integer, float, oct, hex) @@ -271,6 +258,7 @@ def test_common_format(self): if verbose: print('Testing exceptions') + test_exc_common = self.check_exc_common test_exc_common('abc %', (), ValueError, "stray % at position 4") test_exc_common('abc % %s', 1, ValueError, "stray % at position 4 or unexpected format character '%' at position 6") @@ -365,6 +353,7 @@ def test_str_format(self): # Test exception for unknown format characters, etc. if verbose: print('Testing exceptions') + test_exc = self.check_exc test_exc('abc %b', 1, ValueError, "unsupported format %b at position 4") test_exc("abc %\nd", 1, ValueError, @@ -468,6 +457,7 @@ def __bytes__(self): # Test exception for unknown format characters, etc. if verbose: print('Testing exceptions') + test_exc = self.check_exc test_exc(b"abc %\nd", 1, ValueError, "stray % at position 4 or unexpected format character with code 0x0a at position 5") test_exc(b"abc %'d", 1, ValueError, diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py index 3a413ccf396d4ba..849b0480e232fc2 100644 --- a/Lib/test/test_free_threading/test_collections.py +++ b/Lib/test/test_free_threading/test_collections.py @@ -24,6 +24,30 @@ def copy_loop(): threading_helper.run_concurrently([mutate, copy_loop]) + def test_index_race_in_ac(self): + # gh-150750: There was a c_default specified as `Py_SIZE(self)`, + # it was used without a critical section. + + d = deque(range(100)) + + def index(): + for _ in range(10000): + try: + d.index(50) + except ValueError: + pass + + def mutate(): + for _ in range(10000): + d.append(0) + d.clear() + d.extend(range(100)) + d.appendleft(-1) + + threading_helper.run_concurrently( + [index, *[mutate for _ in range(3)]], + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 55272a00c3ad501..dfe0634211d4b02 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -268,6 +268,34 @@ def watcher(): finally: _testcapi.clear_dict_watcher(wid) + def test_racing_split_dict_clear_and_lookup(self): + class C: + pass + + keys = [f"a{i}" for i in range(16)] + + def make_split_nonembedded(): + inst = C() + for key in keys: + setattr(inst, key, keys.index(key)) + # dict.copy() of a split instance dict yields a split table + # with non-embedded values + return inst.__dict__.copy() + + d = make_split_nonembedded() + + def clearer(): + for _ in range(1000): + d.clear() + d.update(make_split_nonembedded()) + + def reader(): + for _ in range(1000): + for k in keys: + d.get(k) + + threading_helper.run_concurrently([clearer, reader, reader]) + def test_racing_dict_update_and_method_lookup(self): # gh-144295: test race between dict modifications and method lookups. # Uses BytesIO because the race requires a type without Py_TPFLAGS_INLINE_VALUES diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 000000000000000..6a6843f9344f640 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py index 8b45b6e2150c288..cc1888dae48bc03 100644 --- a/Lib/test/test_free_threading/test_gc.py +++ b/Lib/test/test_free_threading/test_gc.py @@ -94,6 +94,36 @@ def evil(): thread.start() thread.join() + def test_set_threshold(self): + # GH-148613: Setting the GC threshold from another thread could cause a + # race between the `gc_should_collect` and `gc_set_threshold` functions. + NUM_THREADS = 8 + NUM_ITERS = 100_000 + barrier = threading.Barrier(NUM_THREADS) + + class CyclicReference: + def __init__(self): + self.r = self + + def allocator(): + barrier.wait() + for _ in range(NUM_ITERS): + CyclicReference() + + def setter(): + barrier.wait() + for i in range(NUM_ITERS): + gc.set_threshold(100 + (i % 100), 10 + (i % 10), 10 + (i % 10)) + + current_threshold = gc.get_threshold() + try: + threads = [Thread(target=allocator) for _ in range(NUM_THREADS - 1)] + threads.append(Thread(target=setter)) + with threading_helper.start_threads(threads): + pass + finally: + gc.set_threshold(*current_threshold) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a0065..44d3e9ccfdd14e0 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -12,7 +12,7 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 + NUMITEMS = 5000 NUMTHREADS = 5 NUMMUTATORS = 2 diff --git a/Lib/test/test_free_threading/test_pickle.py b/Lib/test/test_free_threading/test_pickle.py new file mode 100644 index 000000000000000..85a644dc72ecb44 --- /dev/null +++ b/Lib/test/test_free_threading/test_pickle.py @@ -0,0 +1,44 @@ +import pickle +import threading +import unittest + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestPickleFreeThreading(unittest.TestCase): + + def test_pickle_dumps_with_concurrent_dict_mutation(self): + # gh-146452: Pickling a dict while another thread mutates it + # used to segfault. batch_dict_exact() iterated dict items via + # PyDict_Next() which returns borrowed references, and a + # concurrent pop/replace could free the value before Py_INCREF + # got to it. + shared = {str(i): list(range(20)) for i in range(50)} + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except RuntimeError: + # "dictionary changed size during iteration" is expected + pass + + def mutator(): + for j in range(1000): + key = str(j % 50) + shared[key] = list(range(j % 20)) + if j % 10 == 0: + shared.pop(key, None) + shared[key] = [j] + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + with threading_helper.start_threads(threads): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9ed67e..f1eff9430f7351c 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -16,7 +16,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -1145,6 +1145,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a5969b7a47d948b..7816775620bc013 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -55,15 +55,14 @@ from unittest.case import _AssertRaisesContext from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref -import typing -from typing import Unpack try: from tkinter import Event except ImportError: Event = None from string.templatelib import Template, Interpolation -from typing import TypeVar +import typing +from typing import TypeVar, Unpack T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -621,6 +620,14 @@ def test_nested_paramspec_specialization(self): self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) self.assertEqual(deeply_nested_specialized.__parameters__, ()) + def test_gh150146(self): + # It used to crash: + for container in [memoryview, list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index e52e17b8dc73667..ed86802f069e0f8 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -1,5 +1,7 @@ """Test script for the grp module.""" +import random +import string import unittest from test.support import import_helper @@ -50,61 +52,51 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrgid, 3.14) + self.assertRaises(TypeError, grp.getgrgid, 0.0) + self.assertRaises(TypeError, grp.getgrgid, 0, 0) + # should be out of gid_t range + self.assertRaises(OverflowError, grp.getgrgid, 2**128) + self.assertRaises(OverflowError, grp.getgrgid, -2**128) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrnam, 42) - self.assertRaises(TypeError, grp.getgrall, 42) + self.assertRaises(TypeError, grp.getgrnam, b'root') + self.assertRaises(TypeError, grp.getgrnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, grp.getgrnam, 'roo\udc74') + self.assertRaises(KeyError, grp.getgrnam, '') + self.assertRaises(TypeError, grp.getgrall, 42) - # try to get some errors - bynames = {} - bygids = {} - for (n, p, g, mem) in grp.getgrall(): - if not n or n == '+': - continue # skip NIS entries etc. - bynames[n] = g - bygids[g] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue + # Find a non-existent group name. + # getgrall() will not necessarily report all existing groups + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + grp.getgrnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent group name') + + # Find a non-existent gid. + maxgid = 2**31 + for _ in range(30): + fakegid = random.randrange(maxgid) + try: + grp.getgrgid(fakegid) + except KeyError: + break + except OverflowError: + if maxgid == 2**31: + maxgid = 2**16-1 + elif maxgid == 2**16-1: + maxgid = 2**15 else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, grp.getgrnam, fakename) - - # Choose a non-existent gid. - fakegid = 4127 - while fakegid in bygids: - fakegid = (fakegid * 3) % 0x10000 - - self.assertRaises(KeyError, grp.getgrgid, fakegid) - - def test_noninteger_gid(self): - entries = grp.getgrall() - if not entries: - self.skipTest('no groups') - # Choose an existent gid. - gid = entries[0][2] - self.assertRaises(TypeError, grp.getgrgid, float(gid)) - self.assertRaises(TypeError, grp.getgrgid, str(gid)) + raise + else: + self.fail('Cannot find non-existent gid') if __name__ == "__main__": diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 442d30fc970fa94..cafac9d3c8be6e7 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -10,6 +10,7 @@ import sys import unittest from subprocess import PIPE, Popen +from unittest import mock from test.support import catch_unraisable_exception from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper @@ -350,6 +351,26 @@ def test_mtime(self): self.assertEqual(dataRead, data1) self.assertEqual(fRead.mtime, mtime) + def test_mtime_out_of_range(self): + for mtime in (-1, 2**32): + with gzip.GzipFile(self.filename, 'w', mtime=mtime) as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + datac = gzip.compress(data1, mtime=mtime) + with gzip.GzipFile(fileobj=io.BytesIO(datac)) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + + for mtime in (-1, 2**32): + with mock.patch('time.time', return_value=float(mtime)): + with gzip.GzipFile(self.filename, 'w') as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + def test_metadata(self): mtime = 123456789 @@ -795,6 +816,35 @@ def test_decompress_missing_trailer(self): compressed_data = gzip.compress(data1) self.assertRaises(EOFError, gzip.decompress, compressed_data[:-8]) + def test_truncated_header(self): + truncated_headers = [ + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00", # Missing OS byte + b"\x1f\x8b\x08\x02\x00\x00\x00\x00\x00\xff", # FHRC, but no checksum + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff", # FEXTRA, but no xlen + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff\xaa\x00", # FEXTRA, xlen, but no data + b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff", # FNAME but no fname + b"\x1f\x8b\x08\x10\x00\x00\x00\x00\x00\xff", # FCOMMENT, but no fcomment + ] + for header in truncated_headers: + with self.subTest(header=header): + with self.assertRaises(EOFError): + gzip.decompress(header) + + def test_corrupted_gzip_header(self): + header = (b"\x1f\x8b\x08\x1f\x00\x00\x00\x00\x00\xff" # All flags set + b"\x05\x00" # Xlen = 5 + b"extra" + b"name\x00" + b"comment\x00") + true_crc = zlib.crc32(header) & 0xFFFF + corrupted_crc = true_crc ^ 0xFFFF + corrupted_header = header + corrupted_crc.to_bytes(2, "little") + with self.assertRaises(gzip.BadGzipFile) as err: + gzip.decompress(corrupted_header) + self.assertEqual(str(err.exception), + f"Corrupted gzip header. Checksums do not " + f"match: {true_crc:04x} != {corrupted_crc:04x}") + def test_read_truncated(self): data = data1*50 # Drop the CRC (4 bytes) and file size (4 bytes). diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index cde268e32418509..d1df2ec42f0d146 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,11 +1,11 @@ # Simple test suite for http/cookies.py -import base64 import copy import unittest import doctest from http import cookies import pickle from test import support +import urllib.parse class CookieTests(unittest.TestCase): @@ -152,21 +152,21 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -271,21 +271,21 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -376,13 +376,14 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) - expected_encoded_cookie = base64.b64encode( - ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") - ).decode('ascii') + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ """ % (expected_encoded_cookie,) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 0b704d62655762c..fb256fb7cbcd344 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -434,6 +434,16 @@ def cmd_AUTHENTICATE(self, tag, args): r'\[AUTHENTICATIONFAILED\] invalid'): client.authenticate('MYAUTH', lambda x: b'fake') + def test_invalid_login(self): + class MyServer(SimpleIMAPHandler): + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'NO', '[LOGIN] failed') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[LOGIN\] failed'): + client.login('user', 'wrongpass') + def test_valid_authentication_bytes(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c905c0da0a12327..9f3df8010d32339 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -364,6 +364,15 @@ def test_import_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): import something_that_should_not_exist_anywhere + def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): + # gh-150633: module names containing null bytes should not + # lead to duplicates in sys.modules + before = set(sys.modules) + with self.assertRaises(ModuleNotFoundError): + __import__('zipimport\x00junk') + + self.assertEqual(set(sys.modules), before) + def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): from something_that_should_not_exist_anywhere import blah diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 9028d42c617fb4b..7351f97fd9a4b5c 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6255,8 +6255,7 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 482b183da23ffa2..3669ac0b038b71b 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -967,6 +967,25 @@ def test_setstate(self): memio.close() self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) + def test_write_str_subclass(self): + # Writing a str subclass should use the subclass's unicode data + # directly, not call __str__ on it (which may return a different + # value). gh-149047 + class MyStr(str): + def __str__(self): + return "WRONG" + + s = MyStr("correct") + memio = self.ioclass() + memio.write(s) + self.assertEqual(memio.getvalue(), "correct") + + # Also test the fast path where pos == string_size (STATE_ACCUMULATING) + memio2 = self.ioclass() + memio2.write(MyStr("hello ")) + memio2.write(MyStr("world")) + self.assertEqual(memio2.getvalue(), "hello world") + class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index d846c8af7ec4345..1d51fb2de0e69e4 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -87,6 +87,13 @@ def test_array_hook(self): self.assertEqual(self.loads('[]', array_hook=tuple), ()) + def test_load_array_hook(self): + # json.load must forward array_hook to loads + fp = StringIO('[10, 20, 30]') + result = self.json.load(fp, array_hook=tuple) + self.assertEqual(result, (10, 20, 30)) + self.assertEqual(type(result), tuple) + def test_decoder_optimizations(self): # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index 1d1d2e00bd733f4..c99c22491028d33 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -10,6 +10,7 @@ import unittest import tempfile import os +import contextlib from test import support from test.support.script_helper import assert_python_ok @@ -20,8 +21,10 @@ _testcapi = None -class LazyImportTests(unittest.TestCase): - """Tests for basic lazy import functionality.""" +class LazyImportTestCase(unittest.TestCase): + def setUp(self): + self.lazy_imports_filter = sys.get_lazy_imports_filter() + self.lazy_imports = sys.get_lazy_imports() def tearDown(self): """Clean up any test modules from sys.modules.""" @@ -29,16 +32,19 @@ def tearDown(self): if key.startswith('test.test_lazy_import.data'): del sys.modules[key] - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") + sys.set_lazy_imports_filter(self.lazy_imports_filter) + sys.set_lazy_imports(self.lazy_imports) sys.lazy_modules.clear() + +class LazyImportTests(LazyImportTestCase): + """Tests for basic lazy import functionality.""" + def test_basic_unused(self): """Lazy imported module should not be loaded if never accessed.""" import test.test_lazy_import.data.basic_unused self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) def test_sys_lazy_modules(self): try: @@ -48,7 +54,7 @@ def test_sys_lazy_modules(self): self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -88,8 +94,81 @@ def test_basic_used(self): import test.test_lazy_import.data.basic_used self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_raising(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import raising_attr + + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_module_getattr_missing(self): + """Lazy from import should respect module-level __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.module_with_getattr import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) -class GlobalLazyImportModeTests(unittest.TestCase): + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.module_with_getattr import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + +class GlobalLazyImportModeTests(LazyImportTestCase): """Tests for sys.set_lazy_imports() global mode control.""" def tearDown(self): @@ -100,10 +179,10 @@ def tearDown(self): sys.set_lazy_imports_filter(None) sys.set_lazy_imports("normal") - def test_global_off(self): - """Mode 'none' should disable lazy imports entirely.""" - import test.test_lazy_import.data.global_off - self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + def test_global_off_rejected(self): + """Mode 'none' is not supported.""" + with self.assertRaises(ValueError): + sys.set_lazy_imports("none") def test_global_on(self): """Mode 'all' should make regular imports lazy.""" @@ -131,17 +210,9 @@ def test_global_filter_from_true(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class CompatibilityModeTests(unittest.TestCase): +class CompatibilityModeTests(LazyImportTestCase): """Tests for __lazy_modules__ compatibility mode.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_compatibility_mode(self): """__lazy_modules__ should enable lazy imports for listed modules.""" import test.test_lazy_import.data.basic_compatibility_mode @@ -168,17 +239,9 @@ def test_compatibility_mode_relative(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class ModuleIntrospectionTests(unittest.TestCase): +class ModuleIntrospectionTests(LazyImportTestCase): """Tests for module dict and getattr behavior with lazy imports.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_modules_dict(self): """Accessing module.__dict__ should not trigger reification.""" import test.test_lazy_import.data.modules_dict @@ -195,17 +258,9 @@ def test_modules_getattr_other(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class LazyImportTypeTests(unittest.TestCase): +class LazyImportTypeTests(LazyImportTestCase): """Tests for the LazyImportType and its resolve() method.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_value_resolve(self): """resolve() method should force the lazy import to load.""" import test.test_lazy_import.data.lazy_get_value @@ -231,17 +286,9 @@ def test_lazy_import_type_attributes_accessible(self): self.assertIn(b" initial_count print("OK") """) @@ -510,35 +658,22 @@ def test_lazy_modules_tracks_lazy_imports(self): @support.requires_subprocess() -class ErrorHandlingTests(unittest.TestCase): +class ErrorHandlingTests(LazyImportTestCase): """Tests for error handling during lazy import reification. PEP 810: Errors during reification should show exception chaining with both the lazy import definition location and the access location. """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_import_error_shows_chained_traceback(self): - """ImportError during reification should chain to show both definition and access.""" - # Errors at reification must show where the lazy import was defined - # AND where the access happened, per PEP 810 "Reification" section + """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.nonexistent_module try: x = test.test_lazy_import.data.nonexistent_module - except ImportError as e: - # Should have __cause__ showing the original error - # The exception chain shows both where import was defined and where access happened - assert e.__cause__ is not None, "Expected chained exception" + except AttributeError as e: print("OK") """) result = subprocess.run( @@ -586,7 +721,7 @@ def test_reification_retries_on_failure(self): # First access - should fail try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: pass # The lazy object should still be a lazy proxy (not reified) @@ -596,7 +731,7 @@ def test_reification_retries_on_failure(self): # Second access - should also fail (retry the import) try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: print("OK - retry worked") """) result = subprocess.run( @@ -609,7 +744,6 @@ def test_reification_retries_on_failure(self): def test_error_during_module_execution_propagates(self): """Errors in module code during reification should propagate correctly.""" - # Module that raises during import should propagate with chaining code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.broken_module @@ -617,12 +751,8 @@ def test_error_during_module_execution_propagates(self): try: _ = test.test_lazy_import.data.broken_module print("FAIL - should have raised") - except ValueError as e: - # The ValueError from the module should be the cause - if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)): - print("OK") - else: - print(f"FAIL - wrong error: {e}") + except AttributeError: + print("OK") """) result = subprocess.run( [sys.executable, "-c", code], @@ -667,7 +797,7 @@ def hello(): @support.requires_subprocess() -class GlobalsAndDictTests(unittest.TestCase): +class GlobalsAndDictTests(LazyImportTestCase): """Tests for globals() and __dict__ behavior with lazy imports. PEP 810: "Calling globals() or accessing a module's __dict__ does not trigger @@ -675,14 +805,6 @@ class GlobalsAndDictTests(unittest.TestCase): through that dictionary still returns lazy proxy objects." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_globals_returns_lazy_proxy_when_accessed_from_function(self): """globals() accessed from a function should return lazy proxy without reification. @@ -837,7 +959,7 @@ def f(): @support.requires_subprocess() -class MultipleNameFromImportTests(unittest.TestCase): +class MultipleNameFromImportTests(LazyImportTestCase): """Tests for lazy from ... import with multiple names. PEP 810: "When using lazy from ... import, each imported name is bound to a @@ -846,14 +968,6 @@ class MultipleNameFromImportTests(unittest.TestCase): Other names remain as lazy proxies until they are accessed." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_accessing_one_name_leaves_others_as_proxies(self): """Accessing one name from multi-name import should leave others lazy.""" code = textwrap.dedent(""" @@ -918,20 +1032,12 @@ def test_all_names_reified_after_all_accessed(self): @support.requires_subprocess() -class SysLazyModulesTrackingTests(unittest.TestCase): +class SysLazyModulesTrackingTests(LazyImportTestCase): """Tests for sys.lazy_modules tracking behavior. PEP 810: "When the module is reified, it's removed from sys.lazy_modules" """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_module_added_to_lazy_modules_on_lazy_import(self): """Module should be added to sys.lazy_modules when lazily imported.""" # PEP 810 states lazy_modules tracks modules that have been lazily imported @@ -945,15 +1051,14 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): lazy import test.test_lazy_import.data.basic2 # Should be in lazy_modules after lazy import - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count # Trigger reification _ = test.test_lazy_import.data.basic2.x # Module should still be tracked (for diagnostics per PEP 810) - assert "test.test_lazy_import.data" not in sys.lazy_modules + assert "test.test_lazy_import.data.basic2" not in sys.lazy_modules print("OK") """) result = subprocess.run( @@ -966,8 +1071,8 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): def test_lazy_modules_is_per_interpreter(self): """Each interpreter should have independent sys.lazy_modules.""" - # Basic test that sys.lazy_modules exists and is a dict - self.assertIsInstance(sys.lazy_modules, dict) + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) def test_lazy_module_without_children_is_tracked(self): code = textwrap.dedent(""" @@ -976,10 +1081,6 @@ def test_lazy_module_without_children_is_tracked(self): assert "json" in sys.lazy_modules, ( f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" ) - assert sys.lazy_modules["json"] == set(), ( - f"expected empty set for sys.lazy_modules['json'], " - f"got {sys.lazy_modules['json']!r}" - ) print("OK") """) assert_python_ok("-c", code) @@ -1015,68 +1116,16 @@ def test_cli_lazy_imports_all_makes_regular_imports_lazy(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("LAZY", result.stdout) - def test_cli_lazy_imports_none_forces_all_imports_eager(self): - """-X lazy_imports=none should force all imports to be eager.""" - code = textwrap.dedent(""" - import sys - # Even explicit lazy imports should be eager in 'none' mode - lazy import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) + def test_cli_lazy_imports_none_is_rejected(self): + """-X lazy_imports=none should be rejected.""" result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], + [sys.executable, "-X", "lazy_imports=none", "-c", "pass"], capture_output=True, text=True ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) - - @support.requires_resource("cpu") - def test_cli_lazy_imports_modes_import_stdlib_modules(self): - """-X lazy_imports modes should import available stdlib modules.""" - # Do not smoke-test modules with intentional import-time effects. - import_side_effect_modules = {"antigravity", "this"} - importable = [] - - for module in sorted(sys.stdlib_module_names): - if module in import_side_effect_modules: - continue - - with self.subTest(module=module): - code = f"import {module}; print({module})" - baseline = subprocess.run( - [sys.executable, "-I", "-c", code], - capture_output=True, - text=True, - timeout=60, - ) - if baseline.returncode: - # sys.stdlib_module_names includes modules for other - # platforms and optional extension modules not built here. - continue - importable.append(module) - - for mode in ("normal", "none"): - with self.subTest(module=module, mode=mode): - result = subprocess.run( - [ - sys.executable, - "-I", - "-X", - f"lazy_imports={mode}", - "-c", - code, - ], - capture_output=True, - text=True, - timeout=60, - ) - self.assertEqual(result.returncode, 0, result.stderr) - - self.assertGreater(len(importable), 100) + self.assertNotEqual(result.returncode, 0) + self.assertIn("-X lazy_imports: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) def test_cli_lazy_imports_normal_respects_lazy_keyword_only(self): """-X lazy_imports=normal should respect lazy keyword only.""" @@ -1125,77 +1174,27 @@ def test_env_var_lazy_imports_all_enables_global_lazy(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("LAZY", result.stdout) - def test_env_var_lazy_imports_none_disables_all_lazy(self): - """PYTHON_LAZY_IMPORTS=none should disable all lazy imports.""" - code = textwrap.dedent(""" - import sys - lazy import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) + def test_env_var_lazy_imports_none_is_rejected(self): + """PYTHON_LAZY_IMPORTS=none should be rejected.""" import os env = os.environ.copy() env["PYTHON_LAZY_IMPORTS"] = "none" result = subprocess.run( - [sys.executable, "-c", code], + [sys.executable, "-c", "pass"], capture_output=True, text=True, env=env ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) - - def test_cli_lazy_imports_none_disables_dunder_lazy_modules(self): - """-X lazy_imports=none should override __lazy_modules__.""" - code = textwrap.dedent(""" - import sys - __lazy_modules__ = ["json"] - import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) - result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) - - def test_env_var_lazy_imports_none_disables_dunder_lazy_modules(self): - """PYTHON_LAZY_IMPORTS=none should override __lazy_modules__.""" - code = textwrap.dedent(""" - import sys - __lazy_modules__ = ["json"] - import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) - import os - - env = os.environ.copy() - env["PYTHON_LAZY_IMPORTS"] = "none" - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env=env, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PYTHON_LAZY_IMPORTS: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) def test_cli_overrides_env_var(self): """Command-line option should take precedence over environment variable.""" # PEP 810: -X lazy_imports takes precedence over PYTHON_LAZY_IMPORTS code = textwrap.dedent(""" import sys - lazy import json + import json if 'json' in sys.modules: print("EAGER") else: @@ -1203,23 +1202,23 @@ def test_cli_overrides_env_var(self): """) import os env = os.environ.copy() - env["PYTHON_LAZY_IMPORTS"] = "all" # env says all + env["PYTHON_LAZY_IMPORTS"] = "all" # env says all imports are lazy result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], # CLI says none + [sys.executable, "-X", "lazy_imports=normal", "-c", code], capture_output=True, text=True, env=env ) self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - # CLI should win - imports should be eager + # CLI should win, so a regular import should stay eager. self.assertIn("EAGER", result.stdout) def test_sys_set_lazy_imports_overrides_cli(self): """sys.set_lazy_imports() should take precedence over CLI option.""" code = textwrap.dedent(""" import sys - sys.set_lazy_imports("none") # Override CLI - lazy import json + sys.set_lazy_imports("normal") # Override CLI + import json if 'json' in sys.modules: print("EAGER") else: @@ -1235,20 +1234,12 @@ def test_sys_set_lazy_imports_overrides_cli(self): @support.requires_subprocess() -class FilterFunctionSignatureTests(unittest.TestCase): +class FilterFunctionSignatureTests(LazyImportTestCase): """Tests for the filter function signature per PEP 810. PEP 810: func(importer: str, name: str, fromlist: tuple[str, ...] | None) -> bool """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def _run_subprocess_with_modules(self, code, files): with tempfile.TemporaryDirectory() as tmpdir: for relpath, contents in files.items(): @@ -1518,17 +1509,9 @@ def my_filter(importer, name, fromlist): self.assertIn("OK", result.stdout) -class AdditionalSyntaxRestrictionTests(unittest.TestCase): +class AdditionalSyntaxRestrictionTests(LazyImportTestCase): """Additional syntax restriction tests per PEP 810.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_import_inside_class_raises_syntax_error(self): """lazy import inside class body should raise SyntaxError.""" # PEP 810: "The soft keyword is only allowed at the global (module) level, @@ -1538,7 +1521,7 @@ def test_lazy_import_inside_class_raises_syntax_error(self): @support.requires_subprocess() -class MixedLazyEagerImportTests(unittest.TestCase): +class MixedLazyEagerImportTests(LazyImportTestCase): """Tests for mixing lazy and eager imports of the same module. PEP 810: "If module foo is imported both lazily and eagerly in the same @@ -1546,14 +1529,6 @@ class MixedLazyEagerImportTests(unittest.TestCase): the same module object." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_eager_import_before_lazy_resolves_to_same_module(self): """Eager import before lazy should make lazy resolve to same module.""" code = textwrap.dedent(""" @@ -1599,17 +1574,9 @@ def test_lazy_import_before_eager_resolves_to_same_module(self): self.assertIn("OK", result.stdout) -class RelativeImportTests(unittest.TestCase): +class RelativeImportTests(LazyImportTestCase): """Tests for relative imports with lazy keyword.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_relative_lazy_import(self): """lazy from . import submodule should work.""" from test.test_lazy_import.data import relative_lazy @@ -1634,21 +1601,13 @@ def test_relative_lazy_from_import(self): self.assertIn("test.test_lazy_import.data.basic2", sys.modules) -class LazyModulesCompatibilityFromImportTests(unittest.TestCase): +class LazyModulesCompatibilityFromImportTests(LazyImportTestCase): """Tests for __lazy_modules__ with from imports. PEP 810: "When a module is made lazy this way, from-imports using that module are also lazy" """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_modules_makes_from_imports_lazy(self): """__lazy_modules__ should make from imports of listed modules lazy.""" from test.test_lazy_import.data import lazy_compat_from @@ -1663,7 +1622,7 @@ def test_lazy_modules_makes_from_imports_lazy(self): @support.requires_subprocess() -class ImportStateAtReificationTests(unittest.TestCase): +class ImportStateAtReificationTests(LazyImportTestCase): """Tests for import system state at reification time. PEP 810: "Reification still calls __import__ to resolve the import, which uses @@ -1672,14 +1631,6 @@ class ImportStateAtReificationTests(unittest.TestCase): statement was evaluated." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_sys_path_at_reification_time_is_used(self): """sys.path changes after lazy import should affect reification.""" code = textwrap.dedent(""" @@ -1722,17 +1673,9 @@ def test_sys_path_at_reification_time_is_used(self): @support.requires_subprocess() -class ThreadSafetyTests(unittest.TestCase): +class ThreadSafetyTests(LazyImportTestCase): """Tests for thread-safety of lazy imports.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_concurrent_lazy_import_reification(self): """Multiple threads racing to reify the same lazy import should succeed.""" from test.test_lazy_import.data import basic_unused @@ -1848,7 +1791,7 @@ def create_lazy_imports(idx): t.join() assert not errors, f"Errors: {errors}" - assert isinstance(sys.lazy_modules, dict), "sys.lazy_modules is not a dict" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a dict" print("OK") """) @@ -1994,16 +1937,13 @@ def test_normal_import_dis(self): @unittest.skipIf(_testcapi is None, 'need the _testcapi module') -class LazyCApiTests(unittest.TestCase): - def tearDown(self): - sys.set_lazy_imports("normal") - sys.set_lazy_imports_filter(None) - +class LazyCApiTests(LazyImportTestCase): def test_set_matches_sys(self): self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) - for mode in ("normal", "all", "none"): + for mode in ("normal", "all"): _testcapi.PyImport_SetLazyImportsMode(mode) self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) + self.assertRaises(ValueError, _testcapi.PyImport_SetLazyImportsMode, "none") def test_filter_matches_sys(self): self.assertEqual(_testcapi.PyImport_GetLazyImportsFilter(), sys.get_lazy_imports_filter()) diff --git a/Lib/test/test_lazy_import/__main__.py b/Lib/test/test_lazy_import/__main__.py new file mode 100644 index 000000000000000..d6c94efaf30833e --- /dev/null +++ b/Lib/test/test_lazy_import/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_lazy_import') diff --git a/Lib/test/test_lazy_import/data/global_off.py b/Lib/test/test_lazy_import/data/global_off.py deleted file mode 100644 index 95d1511dd932232..000000000000000 --- a/Lib/test/test_lazy_import/data/global_off.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys - -sys.set_lazy_imports("none") - -lazy import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 000000000000000..db3a2301075c2ee --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,12 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index 2d76abaa89f8937..5f7b8662596cac6 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -1 +1,14 @@ x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 2d618081521e10d..b49f05c66fcfbee 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -6,8 +6,9 @@ import unittest.mock from platform import win32_edition from test import support -from test.support import cpython_only, force_not_colorized, os_helper +from test.support import cpython_only, force_not_colorized, os_helper, requires_subprocess from test.support.import_helper import ensure_lazy_imports +from test.support.script_helper import assert_python_ok, assert_python_failure try: import _winapi @@ -508,5 +509,59 @@ def test_invocation_error(self): self.assertEqual(result, expected) +@requires_subprocess() +class CommandLineSubprocessTest(unittest.TestCase): + def test_help(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '--help') + self.assertIn(b'mimetypes', stdout) + self.assertIn(b'--extension', stdout) + self.assertIn(b'--lenient', stdout) + + def test_type_lookup(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf') + self.assertEqual(stdout.strip(), b'type: application/pdf encoding: None') + self.assertEqual(stderr, b'') + + def test_type_lookup_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', 'foo.unknownext12345') + self.assertEqual(stdout.strip(), b'error: media type unknown for foo.unknownext12345') + self.assertEqual(stderr, b'') + + def test_extension_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', 'image/jpeg') + self.assertEqual(stdout.strip(), b'.jpg') + self.assertEqual(stderr, b'') + + def test_extension_flag_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '-e', 'image/unknowntype12345') + self.assertEqual(stdout.strip(), b'error: unknown type image/unknowntype12345') + self.assertEqual(stderr, b'') + + def test_lenient_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', '--lenient', 'text/xul') + self.assertIn(b'.xul', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf', 'foo.png') + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'type: image/png encoding: None', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs_with_error(self): + rc, stdout, stderr = assert_python_failure( + '-m', 'mimetypes', 'foo.pdf', 'foo.unknownext12345' + ) + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'error: media type unknown for foo.unknownext12345', stdout) + self.assertEqual(stderr, b'') + + @force_not_colorized + def test_unknown_flag(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '--unknown-flag', 'foo.pdf') + self.assertEqual(stdout, b'') + self.assertIn(b'error: unrecognized arguments: --unknown-flag', stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 177fe45e8d97490..2e2ac147968dd4a 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -354,6 +354,8 @@ def test_find_end(self): self.assertEqual(m.find(b'one', 1, -1), 8) self.assertEqual(m.find(b'one', 1, -2), -1) self.assertEqual(m.find(bytearray(b'one')), 0) + self.assertEqual(m.find(b'', n + 1), -1) + self.assertEqual(m.rfind(b'', n + 1), -1) for i in range(-n-1, n+1): for j in range(-n-1, n+1): diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 1c6905bd1ee5864..fef86ce323e54d5 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -4,6 +4,7 @@ import unittest from textwrap import dedent +from test import support from test.support import os_helper, requires_resource from test.support.os_helper import TESTFN, TESTFN_ASCII @@ -67,8 +68,12 @@ def run_in_separated_process(self, code): # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] - subprocess.run(cmd, check=True, capture_output=True, - creationflags=subprocess.CREATE_NEW_CONSOLE) + try: + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + except subprocess.CalledProcessError as exc: + support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode) + raise def test_kbhit(self): code = dedent(''' diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 7e670e5a139d999..fef4f495fa56799 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -2137,6 +2137,94 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mode_with_parent_mode(self): + # Test the parent_mode parameter + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + # Specify mode for both leaf and parent directories + os.makedirs(path, 0o770, parent_mode=0o750) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # Leaf directory gets the mode parameter + self.assertEqual(os.stat(path).st_mode & 0o777, 0o770) + # Parent directory gets the parent_mode parameter + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o750) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + base = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3') + with os_helper.temp_umask(0o002): + os.makedirs(base, 0o755, parent_mode=0o700) + self.assertTrue(os.path.exists(base)) + if os.name != 'nt': + # Check that all parent directories have parent_mode + level1 = os.path.join(os_helper.TESTFN, 'dir1') + level2 = os.path.join(level1, 'dir2') + self.assertEqual(os.stat(level1).st_mode & 0o777, 0o700) + self.assertEqual(os.stat(level2).st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(os.stat(base).st_mode & 0o777, 0o755) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_same_as_mode(self): + # Test emulating Python 3.6 behavior by setting parent_mode=mode + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + os.makedirs(path, 0o705, parent_mode=0o705) + self.assertTrue(os.path.exists(path)) + if os.name != 'nt': + # Both directories should have the same mode + self.assertEqual(os.stat(path).st_mode & 0o777, 0o705) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o705) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o022): + os.makedirs(path, 0o777, parent_mode=0o777) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(os.stat(path).st_mode & 0o777, 0o755) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o755) + @unittest.skipIf( support.is_wasi, "WASI's umask is a stub." @@ -2210,15 +2298,9 @@ def test_win32_mkdir_700(self): ) def tearDown(self): - path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', - 'dir4', 'dir5', 'dir6') - # If the tests failed, the bottom-most directory ('../dir6') - # may not have been created, so we look for the outermost directory - # that exists. - while not os.path.exists(path) and path != os_helper.TESTFN: - path = os.path.dirname(path) - - os.removedirs(path) + # Remove the whole tree regardless of which sub-directories a test + # created and regardless of their permission bits. + os_helper.rmtree(os_helper.TESTFN) @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @@ -3707,7 +3789,6 @@ async def test_trailers(self): @requires_headers_trailers @requires_32b async def test_headers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, headers=[b"x" * 2**16] * 2**15) @@ -3716,7 +3797,6 @@ async def test_headers_overflow_32bits(self): @requires_headers_trailers @requires_32b async def test_trailers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, trailers=[b"x" * 2**16] * 2**15) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 09d1b5d725e5ba7..2cb4876f5c6400a 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -2492,6 +2492,116 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parents_umask(self): + # Test that parent directories respect umask when parent_mode is not set + p = self.cls(self.base, 'umasktest', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o002): + p.mkdir(0o755, parents=True) + self.assertTrue(p.exists()) + # Leaf directory gets the specified mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory respects umask (0o777 & ~0o002 = 0o775) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o775) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_with_parent_mode(self): + # Test the parent_mode parameter + p = self.cls(self.base, 'newdirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + # Specify different modes for parent and leaf directories + p.mkdir(0o755, parents=True, parent_mode=0o750) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + # Leaf directory gets the mode parameter + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory gets the parent_mode parameter + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o750) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + p = self.cls(self.base, 'level1PM', 'level2PM', 'level3PM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o755, parents=True, parent_mode=0o700) + self.assertTrue(p.exists()) + # Check that all parent directories have parent_mode + level1 = self.cls(self.base, 'level1PM') + level2 = level1 / 'level2PM' + self.assertEqual(level1.stat().st_mode & 0o777, 0o700) + self.assertEqual(level2.stat().st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + p = self.cls(self.base, 'umaskPM', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o777, parents=True, parent_mode=0o777) + self.assertTrue(p.exists()) + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_same_as_mode(self): + # Test setting parent_mode same as mode + p = self.cls(self.base, 'samedirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o705, parents=True, parent_mode=0o705) + self.assertTrue(p.exists()) + # Both directories should have the same mode + self.assertEqual(p.stat().st_mode & 0o777, 0o705) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o705) + @needs_symlinks def test_symlink_to(self): P = self.cls(self.base) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 597e65993520491..425c76dd01ed7c2 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -34,6 +34,21 @@ def supports_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") +def _perf_env(**env_vars): + env = os.environ.copy() + # Keep perf's output stable regardless of the builder's perf config. + env.update( + { + "DEBUGINFOD_URLS": "", + "PERF_CONFIG": os.devnull, + } + ) + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + return env + + class TestPerfTrampoline(unittest.TestCase): def setUp(self): super().setUp() @@ -63,13 +78,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -132,13 +146,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -198,13 +211,12 @@ def test_trampoline_works_after_fork_with_many_code_objects(self): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -242,13 +254,12 @@ def baz(): code = set_eval_hook + code with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -345,9 +356,12 @@ def perf_command_works(): "-c", 'print("hello")', ) - env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + cmd, + cwd=script_dir, + text=True, + stderr=subprocess.STDOUT, + env=_perf_env(), ) except (subprocess.SubprocessError, OSError): return False @@ -359,43 +373,49 @@ def perf_command_works(): def run_perf(cwd, *args, use_jit=False, **env_vars): - env = os.environ.copy() - if env_vars: - env.update(env_vars) - env["PYTHON_JIT"] = "0" + env = _perf_env(**env_vars) output_file = cwd + "/perf_output.perf" - if not use_jit: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=fp", - "-o", output_file, - "--" - ) + base_cmd = [ + "perf", + "record", + "--no-buildid", + "--no-buildid-cache", + "-g", + "--call-graph=dwarf,65528" if use_jit else "--call-graph=fp", + ] + if use_jit: + perf_commands = [] + # Some builders have low perf_event_mlock_kb limits. + mmap_sizes = ("4M", "2M", "1M", "512K", "256K", "128K", None) + for mmap_size in mmap_sizes: + command = base_cmd.copy() + if mmap_size is not None: + command += ["-F99", "-k1", "-m", mmap_size] + else: + command += ["-F99", "-k1"] + command += ["-o", output_file, "--"] + perf_commands.append(command) else: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=dwarf,65528", - "-F99", - "-k1", - "-o", - output_file, - "--", + perf_commands = [base_cmd + ["-o", output_file, "--"]] + + mmap_pages_error = "try again with a smaller value of -m/--mmap_pages" + for index, base_cmd in enumerate(perf_commands): + proc = subprocess.run( + base_cmd + list(args), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True, ) - proc = subprocess.run( - base_cmd + args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - text=True, - ) + if ( + proc.returncode + and use_jit + and index != len(perf_commands) - 1 + and mmap_pages_error in proc.stderr + ): + continue + break + if proc.returncode: print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") @@ -425,16 +445,34 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): class TestPerfProfilerMixin: - def run_perf(self, script_dir, perf_mode, script): + PERF_CAPTURE_ATTEMPTS = 3 + + def run_perf(self, script_dir, script, activate_trampoline=True): raise NotImplementedError() + def run_perf_with_retries( + self, script_dir, script, expected_symbols=(), activate_trampoline=True + ): + stdout = stderr = "" + for _ in range(self.PERF_CAPTURE_ATTEMPTS): + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=activate_trampoline + ) + if activate_trampoline and any( + symbol not in stdout for symbol in expected_symbols + ): + continue + break + return stdout, stderr + def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -442,23 +480,29 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf(script_dir, script) - self.assertEqual(stderr, "") + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] + stdout, _ = self.run_perf_with_retries( + script_dir, script, expected_symbols + ) - self.assertIn(f"py::foo:{script}", stdout) - self.assertIn(f"py::bar:{script}", stdout) - self.assertIn(f"py::baz:{script}", stdout) + for expected_symbol in expected_symbols: + self.assertIn(expected_symbol, stdout) def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -466,13 +510,12 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf( + stdout, _ = self.run_perf_with_retries( script_dir, script, activate_trampoline=False ) - self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) @@ -542,13 +585,12 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 55a3c654aa0a471..48375cf459ea0b1 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -786,7 +786,11 @@ def test_invocation(self): 'b': ('character string', b'byte string'), 'c': 'string' } - expect = "{'a': [1, 2.0, (3+4j)], 'b': ('character string', b'byte string'), 'c': 'string'}" + expect = ''' + {'a': [1, 2.0, (3+4j)], + 'b': ('character string', b'byte string'), + 'c': 'string'} + ''' self.set_pickle_data(data) with self.subTest(data=data): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9ee97b922ad48e1..63c130813ec4972 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -534,7 +534,7 @@ def test_ios_ver(self): def test_libc_ver(self): if support.is_emscripten: - assert platform.libc_ver() == ("emscripten", "4.0.12") + assert platform.libc_ver() == ("emscripten", "4.0.19") return # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index f439782f53e6fb9..041c2072b9e253a 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -3,7 +3,6 @@ import collections import contextlib import dataclasses -import functools import io import itertools import pprint @@ -16,10 +15,6 @@ from test.support import cpython_only from test.support.import_helper import ensure_lazy_imports -# Pin pre-3.15 width/indent for existing formatting tests. -# compact=True keeps the legacy non-JSON-style container wrapping. -_pformat = functools.partial(pprint.pformat, indent=1, width=80, compact=True) - # list, tuple and dict subclasses that do or don't overwrite __repr__ class list2(list): pass @@ -169,6 +164,7 @@ def test_init(self): self.assertRaises(ValueError, pprint.PrettyPrinter, depth=0) self.assertRaises(ValueError, pprint.PrettyPrinter, depth=-1) self.assertRaises(ValueError, pprint.PrettyPrinter, width=0) + self.assertRaises(ValueError, pprint.PrettyPrinter, compact=True, expand=True) def test_basic(self): # Verify .isrecursive() and .isreadable() w/o recursion @@ -288,10 +284,10 @@ def test_same_as_repr(self): True, False, None, ..., ): native = repr(simple) - self.assertEqual(_pformat(simple), native) - self.assertEqual(_pformat(simple, width=1, indent=0) + self.assertEqual(pprint.pformat(simple), native) + self.assertEqual(pprint.pformat(simple, width=1, indent=0) .replace('\n', ' '), native) - self.assertEqual(_pformat(simple, underscore_numbers=True), native) + self.assertEqual(pprint.pformat(simple, underscore_numbers=True), native) self.assertEqual(pprint.saferepr(simple), native) def test_container_repr_override_called(self): @@ -322,8 +318,8 @@ def test_container_repr_override_called(self): ): native = repr(cont) expected = '*' * len(native) - self.assertEqual(_pformat(cont), expected) - self.assertEqual(_pformat(cont, width=1, indent=0), expected) + self.assertEqual(pprint.pformat(cont), expected) + self.assertEqual(pprint.pformat(cont, width=1, indent=0), expected) self.assertEqual(pprint.saferepr(cont), expected) def test_basic_line_wrap(self): @@ -344,7 +340,7 @@ def test_basic_line_wrap(self): 'read_io_runtime_us': 0, 'write_io_runtime_us': 43690}""" for type in [dict, dict2]: - self.assertEqual(_pformat(type(o)), exp) + self.assertEqual(pprint.pformat(type(o)), exp) exp = """\ frozendict({'RPM_cal': 0, @@ -354,7 +350,7 @@ def test_basic_line_wrap(self): 'main_code_runtime_us': 0, 'read_io_runtime_us': 0, 'write_io_runtime_us': 43690})""" - self.assertEqual(_pformat(frozendict(o)), exp) + self.assertEqual(pprint.pformat(frozendict(o)), exp) exp = """\ frozendict2({'RPM_cal': 0, 'RPM_cal2': 48059, @@ -363,79 +359,79 @@ def test_basic_line_wrap(self): 'main_code_runtime_us': 0, 'read_io_runtime_us': 0, 'write_io_runtime_us': 43690})""" - self.assertEqual(_pformat(frozendict2(o)), exp) + self.assertEqual(pprint.pformat(frozendict2(o)), exp) o = range(100) exp = 'dict_keys([%s])' % ',\n '.join(map(str, o)) keys = dict.fromkeys(o).keys() - self.assertEqual(_pformat(keys, width=1), exp) + self.assertEqual(pprint.pformat(keys), exp) keys = frozendict.fromkeys(o).keys() - self.assertEqual(_pformat(keys, width=1), exp) + self.assertEqual(pprint.pformat(keys), exp) o = range(100) exp = 'dict_values([%s])' % ',\n '.join(map(str, o)) values = {v: v for v in o}.values() - self.assertEqual(_pformat(values, width=1), exp) + self.assertEqual(pprint.pformat(values), exp) values = frozendict({v: v for v in o}).values() - self.assertEqual(_pformat(values, width=1), exp) + self.assertEqual(pprint.pformat(values), exp) o = range(100) exp = 'dict_items([%s])' % ',\n '.join("(%s, %s)" % (i, i) for i in o) items = {v: v for v in o}.items() - self.assertEqual(_pformat(items, width=11), exp) + self.assertEqual(pprint.pformat(items), exp) items = frozendict({v: v for v in o}).items() - self.assertEqual(_pformat(items, width=11), exp) + self.assertEqual(pprint.pformat(items), exp) o = range(100) exp = 'odict_keys([%s])' % ',\n '.join(map(str, o)) keys = collections.OrderedDict.fromkeys(o).keys() - self.assertEqual(_pformat(keys, width=1), exp) + self.assertEqual(pprint.pformat(keys), exp) o = range(100) exp = 'odict_values([%s])' % ',\n '.join(map(str, o)) values = collections.OrderedDict({v: v for v in o}).values() - self.assertEqual(_pformat(values, width=1), exp) + self.assertEqual(pprint.pformat(values), exp) o = range(100) exp = 'odict_items([%s])' % ',\n '.join("(%s, %s)" % (i, i) for i in o) items = collections.OrderedDict({v: v for v in o}).items() - self.assertEqual(_pformat(items, width=11), exp) + self.assertEqual(pprint.pformat(items), exp) o = range(100) exp = 'KeysView({%s})' % (': None,\n '.join(map(str, o)) + ': None') keys_view = KeysView(dict.fromkeys(o)) - self.assertEqual(_pformat(keys_view), exp) + self.assertEqual(pprint.pformat(keys_view), exp) o = range(100) exp = 'ItemsView({%s})' % (': None,\n '.join(map(str, o)) + ': None') items_view = ItemsView(dict.fromkeys(o)) - self.assertEqual(_pformat(items_view), exp) + self.assertEqual(pprint.pformat(items_view), exp) o = range(100) exp = 'MappingView({%s})' % (': None,\n '.join(map(str, o)) + ': None') mapping_view = MappingView(dict.fromkeys(o)) - self.assertEqual(_pformat(mapping_view), exp) + self.assertEqual(pprint.pformat(mapping_view), exp) o = range(100) exp = 'ValuesView({%s})' % (': None,\n '.join(map(str, o)) + ': None') values_view = ValuesView(dict.fromkeys(o)) - self.assertEqual(_pformat(values_view), exp) + self.assertEqual(pprint.pformat(values_view), exp) o = range(100) exp = '[%s]' % ',\n '.join(map(str, o)) for type in [list, list2]: - self.assertEqual(_pformat(type(o), width=1), exp) + self.assertEqual(pprint.pformat(type(o)), exp) o = tuple(range(100)) exp = '(%s)' % ',\n '.join(map(str, o)) for type in [tuple, tuple2]: - self.assertEqual(_pformat(type(o), width=1), exp) + self.assertEqual(pprint.pformat(type(o)), exp) # indent parameter o = range(100) exp = '[ %s]' % ',\n '.join(map(str, o)) for type in [list, list2]: - self.assertEqual(_pformat(type(o), indent=4, width=1), exp) + self.assertEqual(pprint.pformat(type(o), indent=4), exp) def test_nested_indentations(self): o1 = list(range(10)) @@ -444,13 +440,13 @@ def test_nested_indentations(self): expected = """\ [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], {'first': 1, 'second': 2, 'third': 3}]""" - self.assertEqual(pprint.pformat(o, indent=4, width=42, compact=True), expected) + self.assertEqual(pprint.pformat(o, indent=4, width=42), expected) expected = """\ [ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], { 'first': 1, 'second': 2, 'third': 3}]""" - self.assertEqual(pprint.pformat(o, indent=4, width=41, compact=True), expected) + self.assertEqual(pprint.pformat(o, indent=4, width=41), expected) def test_width(self): expected = """\ @@ -464,15 +460,17 @@ def test_width(self): [[[[[1, 2, 3], '1 2']]]]]""" o = eval(expected) - self.assertEqual(_pformat(o, width=15), expected) - self.assertEqual(_pformat(o, width=16), expected) - self.assertEqual(_pformat(o, width=25), expected) - self.assertEqual(_pformat(o, width=14), """\ -[[[[[[1, 2, + self.assertEqual(pprint.pformat(o, width=15), expected) + self.assertEqual(pprint.pformat(o, width=16), expected) + self.assertEqual(pprint.pformat(o, width=25), expected) + self.assertEqual(pprint.pformat(o, width=14), """\ +[[[[[[1, + 2, 3], '1 ' '2']]]], - {1: [1, 2, + {1: [1, + 2, 3], 2: [12, 34]}, @@ -482,14 +480,15 @@ def test_width(self): 'ef',), set2({1, 23}), - [[[[[1, 2, + [[[[[1, + 2, 3], '1 ' '2']]]]]""") def test_integer(self): - self.assertEqual(_pformat(1234567), '1234567') - self.assertEqual(_pformat(1234567, underscore_numbers=True), '1_234_567') + self.assertEqual(pprint.pformat(1234567), '1234567') + self.assertEqual(pprint.pformat(1234567, underscore_numbers=True), '1_234_567') class Temperature(int): def __new__(cls, celsius_degrees): @@ -497,7 +496,7 @@ def __new__(cls, celsius_degrees): def __repr__(self): kelvin_degrees = self + 273.15 return f"{kelvin_degrees:.2f}°K" - self.assertEqual(_pformat(Temperature(1000)), '1273.15°K') + self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): # Starting in Python 2.5, pprint sorts dict displays by key regardless @@ -505,8 +504,8 @@ def test_sorted_dict(self): # Before the change, on 32-bit Windows pformat() gave order # 'a', 'c', 'b' here, so this test failed. d = {'a': 1, 'b': 1, 'c': 1} - self.assertEqual(_pformat(d), "{'a': 1, 'b': 1, 'c': 1}") - self.assertEqual(_pformat([d, d]), + self.assertEqual(pprint.pformat(d), "{'a': 1, 'b': 1, 'c': 1}") + self.assertEqual(pprint.pformat([d, d]), "[{'a': 1, 'b': 1, 'c': 1}, {'a': 1, 'b': 1, 'c': 1}]") # The next one is kind of goofy. The sorted order depends on the @@ -514,42 +513,63 @@ def test_sorted_dict(self): # Python 2.5, this was in the test_same_as_repr() test. It's worth # keeping around for now because it's one of few tests of pprint # against a crazy mix of types. - self.assertEqual(_pformat({"xy\tab\n": (3,), 5: [[]], (): {}}), + self.assertEqual(pprint.pformat({"xy\tab\n": (3,), 5: [[]], (): {}}), r"{5: [[]], 'xy\tab\n': (3,), (): {}}") def test_sort_dict(self): d = dict.fromkeys('cba') - self.assertEqual(_pformat(d, sort_dicts=False), "{'c': None, 'b': None, 'a': None}") - self.assertEqual(_pformat([d, d], sort_dicts=False), + self.assertEqual(pprint.pformat(d, sort_dicts=False), "{'c': None, 'b': None, 'a': None}") + self.assertEqual(pprint.pformat([d, d], sort_dicts=False), "[{'c': None, 'b': None, 'a': None}, {'c': None, 'b': None, 'a': None}]") def test_ordered_dict(self): d = collections.OrderedDict() - self.assertEqual(_pformat(d, width=1), 'OrderedDict()') + self.assertEqual(pprint.pformat(d, width=1), 'OrderedDict()') d = collections.OrderedDict([]) - self.assertEqual(_pformat(d, width=1), 'OrderedDict()') + self.assertEqual(pprint.pformat(d, width=1), 'OrderedDict()') words = 'the quick brown fox jumped over a lazy dog'.split() d = collections.OrderedDict(zip(words, itertools.count())) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ -OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), ('jumped', 4), - ('over', 5), ('a', 6), ('lazy', 7), ('dog', 8)])""") - self.assertEqual( - _pformat(d.keys(), sort_dicts=False), - "odict_keys(['the', 'quick', 'brown', 'fox', 'jumped', 'over', 'a', 'lazy', 'dog'])", - ) - self.assertEqual(_pformat(d.items(), sort_dicts=False), +OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])""") + self.assertEqual(pprint.pformat(d.keys(), sort_dicts=False), +"""\ +odict_keys(['the', + 'quick', + 'brown', + 'fox', + 'jumped', + 'over', + 'a', + 'lazy', + 'dog'])""") + self.assertEqual(pprint.pformat(d.items(), sort_dicts=False), """\ -odict_items([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), ('jumped', 4), ('over', 5), - ('a', 6), ('lazy', 7), ('dog', 8)])""") - self.assertEqual(_pformat(d.values(), sort_dicts=False), +odict_items([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])""") + self.assertEqual(pprint.pformat(d.values(), sort_dicts=False), "odict_values([0, 1, 2, 3, 4, 5, 6, 7, 8])") def test_mapping_proxy(self): words = 'the quick brown fox jumped over a lazy dog'.split() d = dict(zip(words, itertools.count())) m = types.MappingProxyType(d) - self.assertEqual(_pformat(m), """\ + self.assertEqual(pprint.pformat(m), """\ mappingproxy({'a': 6, 'brown': 2, 'dog': 8, @@ -561,81 +581,49 @@ def test_mapping_proxy(self): 'the': 0})""") d = collections.OrderedDict(zip(words, itertools.count())) m = types.MappingProxyType(d) - self.assertEqual(_pformat(m), """\ -mappingproxy(OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), - ('jumped', 4), ('over', 5), ('a', 6), ('lazy', 7), + self.assertEqual(pprint.pformat(m), """\ +mappingproxy(OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), ('dog', 8)]))""") def test_dict_views(self): for dict_class in (dict, collections.OrderedDict, collections.Counter): empty = dict_class({}) short = dict_class(dict(zip('edcba', 'edcba'))) - lengths = {"empty": empty, "short": short} + long = dict_class(dict((chr(x), chr(x)) for x in range(90, 64, -1))) + lengths = {"empty": empty, "short": short, "long": long} prefix = "odict" if dict_class is collections.OrderedDict else "dict" for name, d in lengths.items(): with self.subTest(length=name, prefix=prefix): + is_short = len(d) < 6 + joiner = ", " if is_short else ",\n " k = d.keys() v = d.values() i = d.items() - self.assertEqual(_pformat(k, sort_dicts=True), + self.assertEqual(pprint.pformat(k, sort_dicts=True), prefix + "_keys([%s])" % - ", ".join(repr(key) for key in sorted(k))) - self.assertEqual(_pformat(v, sort_dicts=True), + joiner.join(repr(key) for key in sorted(k))) + self.assertEqual(pprint.pformat(v, sort_dicts=True), prefix + "_values([%s])" % - ", ".join(repr(val) for val in sorted(v))) - self.assertEqual(_pformat(i, sort_dicts=True), + joiner.join(repr(val) for val in sorted(v))) + self.assertEqual(pprint.pformat(i, sort_dicts=True), prefix + "_items([%s])" % - ", ".join(repr(item) for item in sorted(i))) - self.assertEqual(_pformat(k, sort_dicts=False), + joiner.join(repr(item) for item in sorted(i))) + self.assertEqual(pprint.pformat(k, sort_dicts=False), prefix + "_keys([%s])" % - ", ".join(repr(key) for key in k)) - self.assertEqual(_pformat(v, sort_dicts=False), + joiner.join(repr(key) for key in k)) + self.assertEqual(pprint.pformat(v, sort_dicts=False), prefix + "_values([%s])" % - ", ".join(repr(val) for val in v)) - self.assertEqual(_pformat(i, sort_dicts=False), + joiner.join(repr(val) for val in v)) + self.assertEqual(pprint.pformat(i, sort_dicts=False), prefix + "_items([%s])" % - ", ".join(repr(item) for item in i)) - - # Long case: views wrap with compact-mode packing. - long = dict((chr(x), chr(x)) for x in range(90, 64, -1)) - sorted_keys = ( - "['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',\n" - " 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']" - ) - unsorted_keys = ( - "['Z', 'Y', 'X', 'W', 'V', 'U', 'T', 'S', 'R', 'Q', 'P', 'O', 'N', 'M', 'L', 'K',\n" - " 'J', 'I', 'H', 'G', 'F', 'E', 'D', 'C', 'B', 'A']" - ) - sorted_items = ( - "[('A', 'A'), ('B', 'B'), ('C', 'C'), ('D', 'D'), ('E', 'E'), ('F', 'F'),\n" - " ('G', 'G'), ('H', 'H'), ('I', 'I'), ('J', 'J'), ('K', 'K'), ('L', 'L'),\n" - " ('M', 'M'), ('N', 'N'), ('O', 'O'), ('P', 'P'), ('Q', 'Q'), ('R', 'R'),\n" - " ('S', 'S'), ('T', 'T'), ('U', 'U'), ('V', 'V'), ('W', 'W'), ('X', 'X'),\n" - " ('Y', 'Y'), ('Z', 'Z')]" - ) - unsorted_items = ( - "[('Z', 'Z'), ('Y', 'Y'), ('X', 'X'), ('W', 'W'), ('V', 'V'), ('U', 'U'),\n" - " ('T', 'T'), ('S', 'S'), ('R', 'R'), ('Q', 'Q'), ('P', 'P'), ('O', 'O'),\n" - " ('N', 'N'), ('M', 'M'), ('L', 'L'), ('K', 'K'), ('J', 'J'), ('I', 'I'),\n" - " ('H', 'H'), ('G', 'G'), ('F', 'F'), ('E', 'E'), ('D', 'D'), ('C', 'C'),\n" - " ('B', 'B'), ('A', 'A')]" - ) - for dict_class in (dict, collections.OrderedDict, collections.Counter): - d = dict_class(long) - prefix = "odict" if dict_class is collections.OrderedDict else "dict" - with self.subTest(length="long", prefix=prefix): - self.assertEqual(_pformat(d.keys(), sort_dicts=True), - f"{prefix}_keys({sorted_keys})") - self.assertEqual(_pformat(d.values(), sort_dicts=True), - f"{prefix}_values({sorted_keys})") - self.assertEqual(_pformat(d.items(), sort_dicts=True), - f"{prefix}_items({sorted_items})") - self.assertEqual(_pformat(d.keys(), sort_dicts=False), - f"{prefix}_keys({unsorted_keys})") - self.assertEqual(_pformat(d.values(), sort_dicts=False), - f"{prefix}_values({unsorted_keys})") - self.assertEqual(_pformat(d.items(), sort_dicts=False), - f"{prefix}_items({unsorted_items})") + joiner.join(repr(item) for item in i)) def test_abc_views(self): empty = {} @@ -653,55 +641,55 @@ class MV(MappingView): pass s = sorted(i) joined_items = "({%s})" % joiner.join(["%r: %r" % (k, v) for (k, v) in i]) sorted_items = "({%s})" % joiner.join(["%r: %r" % (k, v) for (k, v) in s]) - self.assertEqual(_pformat(KeysView(d), sort_dicts=True), + self.assertEqual(pprint.pformat(KeysView(d), sort_dicts=True), KeysView.__name__ + sorted_items) - self.assertEqual(_pformat(ItemsView(d), sort_dicts=True), + self.assertEqual(pprint.pformat(ItemsView(d), sort_dicts=True), ItemsView.__name__ + sorted_items) - self.assertEqual(_pformat(MappingView(d), sort_dicts=True), + self.assertEqual(pprint.pformat(MappingView(d), sort_dicts=True), MappingView.__name__ + sorted_items) - self.assertEqual(_pformat(MV(d), sort_dicts=True), + self.assertEqual(pprint.pformat(MV(d), sort_dicts=True), MV.__name__ + sorted_items) - self.assertEqual(_pformat(ValuesView(d), sort_dicts=True), + self.assertEqual(pprint.pformat(ValuesView(d), sort_dicts=True), ValuesView.__name__ + sorted_items) - self.assertEqual(_pformat(KeysView(d), sort_dicts=False), + self.assertEqual(pprint.pformat(KeysView(d), sort_dicts=False), KeysView.__name__ + joined_items) - self.assertEqual(_pformat(ItemsView(d), sort_dicts=False), + self.assertEqual(pprint.pformat(ItemsView(d), sort_dicts=False), ItemsView.__name__ + joined_items) - self.assertEqual(_pformat(MappingView(d), sort_dicts=False), + self.assertEqual(pprint.pformat(MappingView(d), sort_dicts=False), MappingView.__name__ + joined_items) - self.assertEqual(_pformat(MV(d), sort_dicts=False), + self.assertEqual(pprint.pformat(MV(d), sort_dicts=False), MV.__name__ + joined_items) - self.assertEqual(_pformat(ValuesView(d), sort_dicts=False), + self.assertEqual(pprint.pformat(ValuesView(d), sort_dicts=False), ValuesView.__name__ + joined_items) def test_nested_views(self): d = {1: MappingView({1: MappingView({1: MappingView({1: 2})})})} self.assertEqual(repr(d), "{1: MappingView({1: MappingView({1: MappingView({1: 2})})})}") - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), "{1: MappingView({1: MappingView({1: MappingView({1: 2})})})}") - self.assertEqual(_pformat(d, depth=2), + self.assertEqual(pprint.pformat(d, depth=2), "{1: MappingView({1: {...}})}") d = {} d1 = {1: d.values()} d2 = {1: d1.values()} d3 = {1: d2.values()} - self.assertEqual(_pformat(d3), + self.assertEqual(pprint.pformat(d3), "{1: dict_values([dict_values([dict_values([])])])}") - self.assertEqual(_pformat(d3, depth=2), + self.assertEqual(pprint.pformat(d3, depth=2), "{1: dict_values([{...}])}") def test_unorderable_items_views(self): """Check that views with unorderable items have stable sorting.""" d = dict((((3+1j), 3), ((1+1j), (1+0j)), (1j, 0j), (500, None), (499, None))) iv = ItemsView(d) - self.assertEqual(_pformat(iv), - _pformat(iv)) - self.assertTrue(_pformat(iv).endswith(", 499: None, 500: None})"), - _pformat(iv)) - self.assertEqual(_pformat(d.items()), # Won't be equal unless _safe_tuple - _pformat(d.items())) # is used in _safe_repr - self.assertTrue(_pformat(d.items()).endswith(", (499, None), (500, None)])")) + self.assertEqual(pprint.pformat(iv), + pprint.pformat(iv)) + self.assertTrue(pprint.pformat(iv).endswith(", 499: None, 500: None})"), + pprint.pformat(iv)) + self.assertEqual(pprint.pformat(d.items()), # Won't be equal unless _safe_tuple + pprint.pformat(d.items())) # is used in _safe_repr + self.assertTrue(pprint.pformat(d.items()).endswith(", (499, None), (500, None)])")) def test_mapping_view_subclass_no_mapping(self): class BMV(MappingView): @@ -710,7 +698,7 @@ def __init__(self, d): self.mapping = self._mapping del self._mapping - self.assertRaises(AttributeError, _pformat, BMV({})) + self.assertRaises(AttributeError, pprint.pformat, BMV({})) def test_mapping_subclass_repr(self): """Test that mapping ABC views use their ._mapping's __repr__.""" @@ -734,10 +722,10 @@ def __repr__(self): self.assertEqual(repr(m), "MyMapping(['test', 1])") short_view_repr = "%s(MyMapping(['test', 1]))" self.assertEqual(repr(m.keys()), short_view_repr % "KeysView") - self.assertEqual(_pformat(m.items()), short_view_repr % "ItemsView") - self.assertEqual(_pformat(m.keys()), short_view_repr % "KeysView") - self.assertEqual(_pformat(MappingView(m)), short_view_repr % "MappingView") - self.assertEqual(_pformat(m.values()), short_view_repr % "ValuesView") + self.assertEqual(pprint.pformat(m.items()), short_view_repr % "ItemsView") + self.assertEqual(pprint.pformat(m.keys()), short_view_repr % "KeysView") + self.assertEqual(pprint.pformat(MappingView(m)), short_view_repr % "MappingView") + self.assertEqual(pprint.pformat(m.values()), short_view_repr % "ValuesView") alpha = "abcdefghijklmnopqrstuvwxyz" m = MyMapping(alpha) @@ -745,19 +733,19 @@ def __repr__(self): long_view_repr = "%%s(MyMapping([%s]))" % alpha_repr self.assertEqual(repr(m), "MyMapping([%s])" % alpha_repr) self.assertEqual(repr(m.keys()), long_view_repr % "KeysView") - self.assertEqual(_pformat(m.items()), long_view_repr % "ItemsView") - self.assertEqual(_pformat(m.keys()), long_view_repr % "KeysView") - self.assertEqual(_pformat(MappingView(m)), long_view_repr % "MappingView") - self.assertEqual(_pformat(m.values()), long_view_repr % "ValuesView") + self.assertEqual(pprint.pformat(m.items()), long_view_repr % "ItemsView") + self.assertEqual(pprint.pformat(m.keys()), long_view_repr % "KeysView") + self.assertEqual(pprint.pformat(MappingView(m)), long_view_repr % "MappingView") + self.assertEqual(pprint.pformat(m.values()), long_view_repr % "ValuesView") def test_empty_simple_namespace(self): ns = types.SimpleNamespace() - formatted = _pformat(ns) + formatted = pprint.pformat(ns) self.assertEqual(formatted, "namespace()") def test_small_simple_namespace(self): ns = types.SimpleNamespace(a=1, b=2) - formatted = _pformat(ns) + formatted = pprint.pformat(ns) self.assertEqual(formatted, "namespace(a=1, b=2)") def test_simple_namespace(self): @@ -772,7 +760,7 @@ def test_simple_namespace(self): lazy=7, dog=8, ) - formatted = pprint.pformat(ns, width=60, indent=4, compact=True) + formatted = pprint.pformat(ns, width=60, indent=4) self.assertEqual(formatted, """\ namespace(the=0, quick=1, @@ -797,7 +785,7 @@ class AdvancedNamespace(types.SimpleNamespace): pass lazy=7, dog=8, ) - formatted = _pformat(ns, width=60) + formatted = pprint.pformat(ns, width=60) self.assertEqual(formatted, """\ AdvancedNamespace(the=0, quick=1, @@ -811,17 +799,17 @@ class AdvancedNamespace(types.SimpleNamespace): pass def test_empty_dataclass(self): dc = dataclasses.make_dataclass("MyDataclass", ())() - formatted = _pformat(dc) + formatted = pprint.pformat(dc) self.assertEqual(formatted, "MyDataclass()") def test_small_dataclass(self): dc = dataclass1("text", 123) - formatted = _pformat(dc) + formatted = pprint.pformat(dc) self.assertEqual(formatted, "dataclass1(field1='text', field2=123, field3=False)") def test_larger_dataclass(self): dc = dataclass1("some fairly long text", int(1e10), True) - formatted = pprint.pformat([dc, dc], width=60, indent=4, compact=True) + formatted = pprint.pformat([dc, dc], width=60, indent=4) self.assertEqual(formatted, """\ [ dataclass1(field1='some fairly long text', field2=10000000000, @@ -832,12 +820,12 @@ def test_larger_dataclass(self): def test_dataclass_with_repr(self): dc = dataclass2() - formatted = _pformat(dc, width=20) + formatted = pprint.pformat(dc, width=20) self.assertEqual(formatted, "custom repr that doesn't fit within pprint width") def test_dataclass_no_repr(self): dc = dataclass3() - formatted = _pformat(dc, width=10) + formatted = pprint.pformat(dc, width=10) self.assertRegex( formatted, fr"<{re.escape(__name__)}.dataclass3 object at \w+>", @@ -846,7 +834,7 @@ def test_dataclass_no_repr(self): def test_recursive_dataclass(self): dc = dataclass4(None) dc.a = dc - formatted = _pformat(dc, width=10) + formatted = pprint.pformat(dc, width=10) self.assertEqual(formatted, """\ dataclass4(a=..., b=1)""") @@ -856,7 +844,7 @@ def test_cyclic_dataclass(self): dc6 = dataclass6(None) dc5.a = dc6 dc6.c = dc5 - formatted = _pformat(dc5, width=10) + formatted = pprint.pformat(dc5, width=10) self.assertEqual(formatted, """\ dataclass5(a=dataclass6(c=..., d=1), @@ -870,7 +858,7 @@ def test_subclassing(self): {'names with spaces': 'should be presented using repr()', others.should.not.be: like.this}""" - dotted_printer = DottedPrettyPrinter(indent=1, compact=True) + dotted_printer = DottedPrettyPrinter() self.assertEqual(dotted_printer.pformat(o), exp) # length(repr(obj)) < width @@ -882,29 +870,47 @@ def test_subclassing(self): self.assertEqual(dotted_printer.pformat(o2), exp2) def test_set_reprs(self): - self.assertEqual(_pformat(set()), 'set()') - self.assertEqual(_pformat(set(range(3))), '{0, 1, 2}') - self.assertEqual(_pformat(set(range(7)), width=20), '''\ -{0, 1, 2, 3, 4, 5, + self.assertEqual(pprint.pformat(set()), 'set()') + self.assertEqual(pprint.pformat(set(range(3))), '{0, 1, 2}') + self.assertEqual(pprint.pformat(set(range(7)), width=20), '''\ +{0, + 1, + 2, + 3, + 4, + 5, 6}''') - self.assertEqual(_pformat(set2(range(7)), width=20), '''\ -set2({0, 1, 2, 3, 4, - 5, 6})''') - self.assertEqual(_pformat(set3(range(7)), width=20), + self.assertEqual(pprint.pformat(set2(range(7)), width=20), '''\ +set2({0, + 1, + 2, + 3, + 4, + 5, + 6})''') + self.assertEqual(pprint.pformat(set3(range(7)), width=20), 'set3({0, 1, 2, 3, 4, 5, 6})') - self.assertEqual(_pformat(frozenset()), 'frozenset()') - self.assertEqual(_pformat(frozenset(range(3))), + self.assertEqual(pprint.pformat(frozenset()), 'frozenset()') + self.assertEqual(pprint.pformat(frozenset(range(3))), 'frozenset({0, 1, 2})') - self.assertEqual(_pformat(frozenset(range(7)), width=20), '''\ -frozenset({0, 1, 2, - 3, 4, 5, + self.assertEqual(pprint.pformat(frozenset(range(7)), width=20), '''\ +frozenset({0, + 1, + 2, + 3, + 4, + 5, 6})''') - self.assertEqual(_pformat(frozenset2(range(7)), width=20), '''\ -frozenset2({0, 1, 2, - 3, 4, 5, + self.assertEqual(pprint.pformat(frozenset2(range(7)), width=20), '''\ +frozenset2({0, + 1, + 2, + 3, + 4, + 5, 6})''') - self.assertEqual(_pformat(frozenset3(range(7)), width=20), + self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), 'frozenset3({0, 1, 2, 3, 4, 5, 6})') def test_set_of_sets_reprs(self): @@ -936,21 +942,21 @@ def test_set_of_sets_reprs(self): fs0 = frozenset() fs1 = frozenset(('abc', 'xyz')) data = frozenset((fs0, fs1)) - self.assertEqual(_pformat(data), + self.assertEqual(pprint.pformat(data), 'frozenset({%r, %r})' % (fs0, fs1)) - self.assertEqual(_pformat(data), repr(data)) + self.assertEqual(pprint.pformat(data), repr(data)) fs2 = frozenset(('one', 'two')) data = {fs2: frozenset((fs0, fs1))} - self.assertEqual(_pformat(data), + self.assertEqual(pprint.pformat(data), "{%r: frozenset({%r, %r})}" % (fs2, fs0, fs1)) - self.assertEqual(_pformat(data), repr(data)) + self.assertEqual(pprint.pformat(data), repr(data)) # Single-line, unordered: fs1 = frozenset(("xyz", "qwerty")) fs2 = frozenset(("abcd", "spam")) fs = frozenset((fs1, fs2)) - self.assertEqual(_pformat(fs), repr(fs)) + self.assertEqual(pprint.pformat(fs), repr(fs)) # Multiline, unordered: def check(res, invariants): @@ -960,7 +966,7 @@ def check(res, invariants): fs1 = frozenset(('regular string', 'other string')) fs2 = frozenset(('third string', 'one more string')) check( - _pformat(frozenset((fs1, fs2))), + pprint.pformat(frozenset((fs1, fs2))), [ """ frozenset({%r, @@ -975,7 +981,7 @@ def check(res, invariants): # Everything is multiline, unordered: check( - _pformat( + pprint.pformat( frozenset(( frozenset(( "xyz very-very long string", @@ -1022,16 +1028,16 @@ def test_depth(self): nested_tuple = (1, (2, (3, (4, (5, 6))))) nested_dict = {1: {2: {3: {4: {5: {6: 6}}}}}} nested_list = [1, [2, [3, [4, [5, [6, []]]]]]] - self.assertEqual(_pformat(nested_tuple), repr(nested_tuple)) - self.assertEqual(_pformat(nested_dict), repr(nested_dict)) - self.assertEqual(_pformat(nested_list), repr(nested_list)) + self.assertEqual(pprint.pformat(nested_tuple), repr(nested_tuple)) + self.assertEqual(pprint.pformat(nested_dict), repr(nested_dict)) + self.assertEqual(pprint.pformat(nested_list), repr(nested_list)) lv1_tuple = '(1, (...))' lv1_dict = '{1: {...}}' lv1_list = '[1, [...]]' - self.assertEqual(_pformat(nested_tuple, depth=1), lv1_tuple) - self.assertEqual(_pformat(nested_dict, depth=1), lv1_dict) - self.assertEqual(_pformat(nested_list, depth=1), lv1_list) + self.assertEqual(pprint.pformat(nested_tuple, depth=1), lv1_tuple) + self.assertEqual(pprint.pformat(nested_dict, depth=1), lv1_dict) + self.assertEqual(pprint.pformat(nested_list, depth=1), lv1_list) def test_sort_unorderable_values(self): # Issue 3976: sorted pprints fail for unorderable values. @@ -1041,24 +1047,24 @@ def test_sort_unorderable_values(self): skeys = sorted(keys, key=id) clean = lambda s: s.replace(' ', '').replace('\n','') - self.assertEqual(clean(_pformat(set(keys))), + self.assertEqual(clean(pprint.pformat(set(keys))), '{' + ','.join(map(repr, skeys)) + '}') - self.assertEqual(clean(_pformat(frozenset(keys))), + self.assertEqual(clean(pprint.pformat(frozenset(keys))), 'frozenset({' + ','.join(map(repr, skeys)) + '})') - self.assertEqual(clean(_pformat(dict.fromkeys(keys))), + self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys))), '{' + ','.join('%r:None' % k for k in skeys) + '}') - self.assertEqual(clean(_pformat(dict.fromkeys(keys).keys())), + self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys).keys())), 'dict_keys([' + ','.join('%r' % k for k in skeys) + '])') - self.assertEqual(clean(_pformat(dict.fromkeys(keys).items())), + self.assertEqual(clean(pprint.pformat(dict.fromkeys(keys).items())), 'dict_items([' + ','.join('(%r,None)' % k for k in skeys) + '])') # Issue 10017: TypeError on user-defined types as dict keys. - self.assertEqual(_pformat({Unorderable: 0, 1: 0}), + self.assertEqual(pprint.pformat({Unorderable: 0, 1: 0}), '{1: 0, ' + repr(Unorderable) +': 0}') # Issue 14998: TypeError on tuples with NoneTypes as dict keys. keys = [(1,), (None,)] - self.assertEqual(_pformat(dict.fromkeys(keys, 0)), + self.assertEqual(pprint.pformat(dict.fromkeys(keys, 0)), '{%r: 0, %r: 0}' % tuple(sorted(keys, key=id))) def test_sort_orderable_and_unorderable_values(self): @@ -1071,24 +1077,24 @@ def test_sort_orderable_and_unorderable_values(self): self.assertEqual(sorted([b, a]), [a, b]) self.assertEqual(sorted([a, b]), [a, b]) # set - self.assertEqual(_pformat(set([b, a]), width=1), + self.assertEqual(pprint.pformat(set([b, a]), width=1), '{%r,\n %r}' % (a, b)) - self.assertEqual(_pformat(set([a, b]), width=1), + self.assertEqual(pprint.pformat(set([a, b]), width=1), '{%r,\n %r}' % (a, b)) # dict - self.assertEqual(_pformat(dict.fromkeys([b, a]), width=1), + self.assertEqual(pprint.pformat(dict.fromkeys([b, a]), width=1), '{%r: None,\n %r: None}' % (a, b)) - self.assertEqual(_pformat(dict.fromkeys([a, b]), width=1), + self.assertEqual(pprint.pformat(dict.fromkeys([a, b]), width=1), '{%r: None,\n %r: None}' % (a, b)) def test_str_wrap(self): # pprint tries to wrap strings intelligently fox = 'the quick brown fox jumped over a lazy dog' - self.assertEqual(_pformat(fox, width=19), """\ + self.assertEqual(pprint.pformat(fox, width=19), """\ ('the quick brown ' 'fox jumped over ' 'a lazy dog')""") - self.assertEqual(_pformat({'a': 1, 'b': fox, 'c': 2}, + self.assertEqual(pprint.pformat({'a': 1, 'b': fox, 'c': 2}, width=25), """\ {'a': 1, 'b': 'the quick brown ' @@ -1101,28 +1107,28 @@ def test_str_wrap(self): # - non-ASCII is allowed # - an apostrophe doesn't disrupt the pprint special = "Portons dix bons \"whiskys\"\nà l'avocat goujat\t qui fumait au zoo" - self.assertEqual(_pformat(special, width=68), repr(special)) - self.assertEqual(_pformat(special, width=31), """\ + self.assertEqual(pprint.pformat(special, width=68), repr(special)) + self.assertEqual(pprint.pformat(special, width=31), """\ ('Portons dix bons "whiskys"\\n' "à l'avocat goujat\\t qui " 'fumait au zoo')""") - self.assertEqual(_pformat(special, width=20), """\ + self.assertEqual(pprint.pformat(special, width=20), """\ ('Portons dix bons ' '"whiskys"\\n' "à l'avocat " 'goujat\\t qui ' 'fumait au zoo')""") - self.assertEqual(_pformat([[[[[special]]]]], width=35), """\ + self.assertEqual(pprint.pformat([[[[[special]]]]], width=35), """\ [[[[['Portons dix bons "whiskys"\\n' "à l'avocat goujat\\t qui " 'fumait au zoo']]]]]""") - self.assertEqual(_pformat([[[[[special]]]]], width=25), """\ + self.assertEqual(pprint.pformat([[[[[special]]]]], width=25), """\ [[[[['Portons dix bons ' '"whiskys"\\n' "à l'avocat " 'goujat\\t qui ' 'fumait au zoo']]]]]""") - self.assertEqual(_pformat([[[[[special]]]]], width=23), """\ + self.assertEqual(pprint.pformat([[[[[special]]]]], width=23), """\ [[[[['Portons dix ' 'bons "whiskys"\\n' "à l'avocat " @@ -1131,14 +1137,14 @@ def test_str_wrap(self): 'zoo']]]]]""") # An unwrappable string is formatted as its repr unwrappable = "x" * 100 - self.assertEqual(_pformat(unwrappable, width=80), repr(unwrappable)) - self.assertEqual(_pformat(''), "''") + self.assertEqual(pprint.pformat(unwrappable, width=80), repr(unwrappable)) + self.assertEqual(pprint.pformat(''), "''") # Check that the pprint is a usable repr special *= 10 for width in range(3, 40): - formatted = _pformat(special, width=width) + formatted = pprint.pformat(special, width=width) self.assertEqual(eval(formatted), special) - formatted = _pformat([special] * 2, width=width) + formatted = pprint.pformat([special] * 2, width=width) self.assertEqual(eval(formatted), [special] * 2) def test_compact(self): @@ -1151,7 +1157,7 @@ def test_compact(self): 14, 15], [], [0], [0, 1], [0, 1, 2], [0, 1, 2, 3], [0, 1, 2, 3, 4]]""" - self.assertEqual(_pformat(o, width=47, compact=True), expected) + self.assertEqual(pprint.pformat(o, width=47, compact=True), expected) def test_compact_width(self): levels = 20 @@ -1160,117 +1166,117 @@ def test_compact_width(self): for i in range(levels - 1): o = [o] for w in range(levels * 2 + 1, levels + 3 * number - 1): - lines = _pformat(o, width=w, compact=True).splitlines() + lines = pprint.pformat(o, width=w, compact=True).splitlines() maxwidth = max(map(len, lines)) self.assertLessEqual(maxwidth, w) self.assertGreater(maxwidth, w - 3) def test_bytes_wrap(self): - self.assertEqual(_pformat(b'', width=1), "b''") - self.assertEqual(_pformat(b'abcd', width=1), "b'abcd'") + self.assertEqual(pprint.pformat(b'', width=1), "b''") + self.assertEqual(pprint.pformat(b'abcd', width=1), "b'abcd'") letters = b'abcdefghijklmnopqrstuvwxyz' - self.assertEqual(_pformat(letters, width=29), repr(letters)) - self.assertEqual(_pformat(letters, width=19), """\ + self.assertEqual(pprint.pformat(letters, width=29), repr(letters)) + self.assertEqual(pprint.pformat(letters, width=19), """\ (b'abcdefghijkl' b'mnopqrstuvwxyz')""") - self.assertEqual(_pformat(letters, width=18), """\ + self.assertEqual(pprint.pformat(letters, width=18), """\ (b'abcdefghijkl' b'mnopqrstuvwx' b'yz')""") - self.assertEqual(_pformat(letters, width=16), """\ + self.assertEqual(pprint.pformat(letters, width=16), """\ (b'abcdefghijkl' b'mnopqrstuvwx' b'yz')""") special = bytes(range(16)) - self.assertEqual(_pformat(special, width=61), repr(special)) - self.assertEqual(_pformat(special, width=48), """\ + self.assertEqual(pprint.pformat(special, width=61), repr(special)) + self.assertEqual(pprint.pformat(special, width=48), """\ (b'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat(special, width=32), """\ + self.assertEqual(pprint.pformat(special, width=32), """\ (b'\\x00\\x01\\x02\\x03' b'\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat(special, width=1), """\ + self.assertEqual(pprint.pformat(special, width=1), """\ (b'\\x00\\x01\\x02\\x03' b'\\x04\\x05\\x06\\x07' b'\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat({'a': 1, 'b': letters, 'c': 2}, + self.assertEqual(pprint.pformat({'a': 1, 'b': letters, 'c': 2}, width=21), """\ {'a': 1, 'b': b'abcdefghijkl' b'mnopqrstuvwx' b'yz', 'c': 2}""") - self.assertEqual(_pformat({'a': 1, 'b': letters, 'c': 2}, + self.assertEqual(pprint.pformat({'a': 1, 'b': letters, 'c': 2}, width=20), """\ {'a': 1, 'b': b'abcdefgh' b'ijklmnop' b'qrstuvwxyz', 'c': 2}""") - self.assertEqual(_pformat([[[[[[letters]]]]]], width=25), """\ + self.assertEqual(pprint.pformat([[[[[[letters]]]]]], width=25), """\ [[[[[[b'abcdefghijklmnop' b'qrstuvwxyz']]]]]]""") - self.assertEqual(_pformat([[[[[[special]]]]]], width=41), """\ + self.assertEqual(pprint.pformat([[[[[[special]]]]]], width=41), """\ [[[[[[b'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07' b'\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f']]]]]]""") # Check that the pprint is a usable repr for width in range(1, 64): - formatted = _pformat(special, width=width) + formatted = pprint.pformat(special, width=width) self.assertEqual(eval(formatted), special) - formatted = _pformat([special] * 2, width=width) + formatted = pprint.pformat([special] * 2, width=width) self.assertEqual(eval(formatted), [special] * 2) def test_bytearray_wrap(self): - self.assertEqual(_pformat(bytearray(), width=1), "bytearray(b'')") + self.assertEqual(pprint.pformat(bytearray(), width=1), "bytearray(b'')") letters = bytearray(b'abcdefghijklmnopqrstuvwxyz') - self.assertEqual(_pformat(letters, width=40), repr(letters)) - self.assertEqual(_pformat(letters, width=28), """\ + self.assertEqual(pprint.pformat(letters, width=40), repr(letters)) + self.assertEqual(pprint.pformat(letters, width=28), """\ bytearray(b'abcdefghijkl' b'mnopqrstuvwxyz')""") - self.assertEqual(_pformat(letters, width=27), """\ + self.assertEqual(pprint.pformat(letters, width=27), """\ bytearray(b'abcdefghijkl' b'mnopqrstuvwx' b'yz')""") - self.assertEqual(_pformat(letters, width=25), """\ + self.assertEqual(pprint.pformat(letters, width=25), """\ bytearray(b'abcdefghijkl' b'mnopqrstuvwx' b'yz')""") special = bytearray(range(16)) - self.assertEqual(_pformat(special, width=72), repr(special)) - self.assertEqual(_pformat(special, width=57), """\ + self.assertEqual(pprint.pformat(special, width=72), repr(special)) + self.assertEqual(pprint.pformat(special, width=57), """\ bytearray(b'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat(special, width=41), """\ + self.assertEqual(pprint.pformat(special, width=41), """\ bytearray(b'\\x00\\x01\\x02\\x03' b'\\x04\\x05\\x06\\x07\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat(special, width=1), """\ + self.assertEqual(pprint.pformat(special, width=1), """\ bytearray(b'\\x00\\x01\\x02\\x03' b'\\x04\\x05\\x06\\x07' b'\\x08\\t\\n\\x0b' b'\\x0c\\r\\x0e\\x0f')""") - self.assertEqual(_pformat({'a': 1, 'b': letters, 'c': 2}, + self.assertEqual(pprint.pformat({'a': 1, 'b': letters, 'c': 2}, width=31), """\ {'a': 1, 'b': bytearray(b'abcdefghijkl' b'mnopqrstuvwx' b'yz'), 'c': 2}""") - self.assertEqual(_pformat([[[[[letters]]]]], width=37), """\ + self.assertEqual(pprint.pformat([[[[[letters]]]]], width=37), """\ [[[[[bytearray(b'abcdefghijklmnop' b'qrstuvwxyz')]]]]]""") - self.assertEqual(_pformat([[[[[special]]]]], width=50), """\ + self.assertEqual(pprint.pformat([[[[[special]]]]], width=50), """\ [[[[[bytearray(b'\\x00\\x01\\x02\\x03\\x04\\x05\\x06\\x07' b'\\x08\\t\\n\\x0b\\x0c\\r\\x0e\\x0f')]]]]]""") def test_default_dict(self): d = collections.defaultdict(int) - self.assertEqual(_pformat(d, width=1), "defaultdict(, {})") + self.assertEqual(pprint.pformat(d, width=1), "defaultdict(, {})") words = 'the quick brown fox jumped over a lazy dog'.split() d = collections.defaultdict(int, zip(words, itertools.count())) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ defaultdict(, {'a': 6, @@ -1285,15 +1291,15 @@ def test_default_dict(self): def test_counter(self): d = collections.Counter() - self.assertEqual(_pformat(d, width=1), "Counter()") + self.assertEqual(pprint.pformat(d, width=1), "Counter()") d = collections.Counter('senselessness') - self.assertEqual(_pformat(d, width=40), + self.assertEqual(pprint.pformat(d, width=40), """\ Counter({'s': 6, 'e': 4, 'n': 2, 'l': 1})""") - self.assertEqual(_pformat(d, indent=2, width=1), + self.assertEqual(pprint.pformat(d, indent=2, width=1), """\ Counter({ 's': 6, 'e': 4, @@ -1302,11 +1308,11 @@ def test_counter(self): def test_chainmap(self): d = collections.ChainMap() - self.assertEqual(_pformat(d, width=1), "ChainMap({})") + self.assertEqual(pprint.pformat(d, width=1), "ChainMap({})") words = 'the quick brown fox jumped over a lazy dog'.split() items = list(zip(words, itertools.count())) d = collections.ChainMap(dict(items)) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ ChainMap({'a': 6, 'brown': 2, @@ -1318,7 +1324,7 @@ def test_chainmap(self): 'quick': 1, 'the': 0})""") d = collections.ChainMap(dict(items), collections.OrderedDict(items)) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ ChainMap({'a': 6, 'brown': 2, @@ -1329,10 +1335,16 @@ def test_chainmap(self): 'over': 5, 'quick': 1, 'the': 0}, - OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), - ('jumped', 4), ('over', 5), ('a', 6), ('lazy', 7), + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), ('dog', 8)]))""") - self.assertEqual(_pformat(d.keys()), + self.assertEqual(pprint.pformat(d.keys()), """\ KeysView(ChainMap({'a': 6, 'brown': 2, @@ -1343,10 +1355,16 @@ def test_chainmap(self): 'over': 5, 'quick': 1, 'the': 0}, - OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), - ('jumped', 4), ('over', 5), ('a', 6), ('lazy', 7), + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), ('dog', 8)])))""") - self.assertEqual(_pformat(d.items()), + self.assertEqual(pprint.pformat(d.items()), """\ ItemsView(ChainMap({'a': 6, 'brown': 2, @@ -1357,10 +1375,16 @@ def test_chainmap(self): 'over': 5, 'quick': 1, 'the': 0}, - OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), - ('jumped', 4), ('over', 5), ('a', 6), ('lazy', 7), + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), ('dog', 8)])))""") - self.assertEqual(_pformat(d.values()), + self.assertEqual(pprint.pformat(d.values()), """\ ValuesView(ChainMap({'a': 6, 'brown': 2, @@ -1371,34 +1395,52 @@ def test_chainmap(self): 'over': 5, 'quick': 1, 'the': 0}, - OrderedDict([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), - ('jumped', 4), ('over', 5), ('a', 6), ('lazy', 7), + OrderedDict([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), ('dog', 8)])))""") def test_deque(self): d = collections.deque() - self.assertEqual(_pformat(d, width=1), "deque([])") + self.assertEqual(pprint.pformat(d, width=1), "deque([])") d = collections.deque(maxlen=7) - self.assertEqual(_pformat(d, width=1), "deque([], maxlen=7)") + self.assertEqual(pprint.pformat(d, width=1), "deque([], maxlen=7)") words = 'the quick brown fox jumped over a lazy dog'.split() d = collections.deque(zip(words, itertools.count())) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ -deque([('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), ('jumped', 4), - ('over', 5), ('a', 6), ('lazy', 7), ('dog', 8)])""") +deque([('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)])""") d = collections.deque(zip(words, itertools.count()), maxlen=7) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ -deque([('brown', 2), ('fox', 3), ('jumped', 4), ('over', 5), ('a', 6), - ('lazy', 7), ('dog', 8)], +deque([('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)], maxlen=7)""") def test_user_dict(self): d = collections.UserDict() - self.assertEqual(_pformat(d, width=1), "{}") + self.assertEqual(pprint.pformat(d, width=1), "{}") words = 'the quick brown fox jumped over a lazy dog'.split() d = collections.UserDict(zip(words, itertools.count())) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ {'a': 6, 'brown': 2, @@ -1409,7 +1451,7 @@ def test_user_dict(self): 'over': 5, 'quick': 1, 'the': 0}""") - self.assertEqual(_pformat(d.keys()), """\ + self.assertEqual(pprint.pformat(d.keys()), """\ KeysView({'a': 6, 'brown': 2, 'dog': 8, @@ -1419,7 +1461,7 @@ def test_user_dict(self): 'over': 5, 'quick': 1, 'the': 0})""") - self.assertEqual(_pformat(d.items()), """\ + self.assertEqual(pprint.pformat(d.items()), """\ ItemsView({'a': 6, 'brown': 2, 'dog': 8, @@ -1429,7 +1471,7 @@ def test_user_dict(self): 'over': 5, 'quick': 1, 'the': 0})""") - self.assertEqual(_pformat(d.values()), """\ + self.assertEqual(pprint.pformat(d.values()), """\ ValuesView({'a': 6, 'brown': 2, 'dog': 8, @@ -1442,24 +1484,31 @@ def test_user_dict(self): def test_user_list(self): d = collections.UserList() - self.assertEqual(_pformat(d, width=1), "[]") + self.assertEqual(pprint.pformat(d, width=1), "[]") words = 'the quick brown fox jumped over a lazy dog'.split() d = collections.UserList(zip(words, itertools.count())) - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ -[('the', 0), ('quick', 1), ('brown', 2), ('fox', 3), ('jumped', 4), ('over', 5), - ('a', 6), ('lazy', 7), ('dog', 8)]""") +[('the', 0), + ('quick', 1), + ('brown', 2), + ('fox', 3), + ('jumped', 4), + ('over', 5), + ('a', 6), + ('lazy', 7), + ('dog', 8)]""") def test_user_string(self): d = collections.UserString('') - self.assertEqual(_pformat(d, width=1), "''") + self.assertEqual(pprint.pformat(d, width=1), "''") d = collections.UserString('the quick brown fox jumped over a lazy dog') - self.assertEqual(_pformat(d, width=20), + self.assertEqual(pprint.pformat(d, width=20), """\ ('the quick brown ' 'fox jumped over ' 'a lazy dog')""") - self.assertEqual(_pformat({1: d}, width=20), + self.assertEqual(pprint.pformat({1: d}, width=20), """\ {1: 'the quick ' 'brown fox ' @@ -1468,22 +1517,22 @@ def test_user_string(self): def test_template(self): d = t"" - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), "Template(strings=('',), interpolations=())") - self.assertEqual(_pformat(d), repr(d)) - self.assertEqual(_pformat(d, width=1), + self.assertEqual(pprint.pformat(d), repr(d)) + self.assertEqual(pprint.pformat(d, width=1), """\ Template(strings=('',), interpolations=())""") name = "World" d = t"Hello {name}" - self.assertEqual(_pformat(d), + self.assertEqual(pprint.pformat(d), """\ Template(strings=('Hello ', ''), interpolations=(Interpolation('World', 'name', None, ''),))""") ver = {3.13: False, 3.14: True} d = t"Hello { {"name": "Python", "version": ver}!s:z}!" - self.assertEqual(_pformat(d, width=1), + self.assertEqual(pprint.pformat(d, width=1), """\ Template(strings=('Hello ', '!'), @@ -1501,13 +1550,13 @@ def test_template(self): def test_expand_template(self): d = t"" self.assertEqual( - pprint.pformat(d), + pprint.pformat(d, expand=True), "Template(strings=('',), interpolations=())", ) name = "World" d = t"Hello {name}" self.assertEqual( - pprint.pformat(d, width=40, indent=4), + pprint.pformat(d, width=40, indent=4, expand=True), """\ Template( strings=('Hello ', ''), @@ -1524,7 +1573,7 @@ def test_expand_template(self): ver = {3.13: False, 3.14: True} d = t"Hello { {"name": "Python", "version": ver}!s:z}!" self.assertEqual( - pprint.pformat(d, width=40, indent=4), + pprint.pformat(d, width=40, indent=4, expand=True), """\ Template( strings=('Hello ', '!'), @@ -1565,7 +1614,8 @@ class DummyDataclass: corge=7, garply=(1, 2, 3, 4), ) - self.assertEqual(pprint.pformat(dummy_dataclass, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_dataclass, width=40, indent=4, + expand=True), """\ DummyDataclass( foo='foo', @@ -1585,7 +1635,8 @@ def test_expand_dict(self): "quux": ["foo", "bar", "baz"], "corge": 7, } - self.assertEqual(pprint.pformat(dummy_dict, width=40, indent=4, sort_dicts=False), + self.assertEqual(pprint.pformat(dummy_dict, width=40, indent=4, + expand=True, sort_dicts=False), """\ { 'foo': 'bar', @@ -1603,7 +1654,8 @@ def test_expand_ordered_dict(self): ("baz", 123), ] ) - self.assertEqual(pprint.pformat(dummy_ordered_dict, width=20, indent=4), + self.assertEqual(pprint.pformat(dummy_ordered_dict, width=20, indent=4, + expand=True), """\ OrderedDict([ ('foo', 1), @@ -1618,7 +1670,8 @@ def test_expand_list(self): "baz", "qux", ] - self.assertEqual(pprint.pformat(dummy_list, width=20, indent=4), + self.assertEqual(pprint.pformat(dummy_list, width=20, indent=4, + expand=True), """\ [ 'foo', @@ -1636,7 +1689,8 @@ def test_expand_tuple(self): 5, 6, ) - self.assertEqual(pprint.pformat(dummy_tuple, width=20, indent=4), + self.assertEqual(pprint.pformat(dummy_tuple, width=20, indent=4, + expand=True), """\ ( 'foo', @@ -1649,7 +1703,7 @@ def test_expand_tuple(self): def test_expand_single_element_tuple(self): self.assertEqual( - pprint.pformat((1,), width=1, indent=4), + pprint.pformat((1,), width=1, indent=4, expand=True), """\ ( 1, @@ -1663,7 +1717,8 @@ def test_expand_set(self): "qux", (1, 2, 3), } - self.assertEqual(pprint.pformat(dummy_set, width=20, indent=4), + self.assertEqual(pprint.pformat(dummy_set, width=20, indent=4, + expand=True), """\ { 'bar', @@ -1686,7 +1741,8 @@ def test_expand_frozenset(self): frozenset(dummy_set), } ) - self.assertEqual(pprint.pformat(dummy_frozenset, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_frozenset, width=40, indent=4, + expand=True), """\ frozenset({ frozenset({(1, 2, 3)}), @@ -1701,7 +1757,7 @@ def test_expand_frozendict(self): {"foo": "bar", "baz": 123, "qux": [1, 2]} ) self.assertEqual( - pprint.pformat(dummy_frozendict, width=20, indent=4), + pprint.pformat(dummy_frozendict, width=20, indent=4, expand=True), """\ frozendict({ 'baz': 123, @@ -1712,7 +1768,8 @@ def test_expand_frozendict(self): def test_expand_bytes(self): dummy_bytes = b"Hello world! foo bar baz 123 456 789" - self.assertEqual(pprint.pformat(dummy_bytes, width=20, indent=4), + self.assertEqual(pprint.pformat(dummy_bytes, width=20, indent=4, + expand=True), """\ ( b'Hello world!' @@ -1723,7 +1780,8 @@ def test_expand_bytes(self): def test_expand_bytearray(self): dummy_bytes = b"Hello world! foo bar baz 123 456 789" dummy_byte_array = bytearray(dummy_bytes) - self.assertEqual(pprint.pformat(dummy_byte_array, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_byte_array, width=40, indent=4, + expand=True), """\ bytearray( b'Hello world! foo bar baz 123 456' @@ -1739,7 +1797,8 @@ def test_expand_mappingproxy(self): "corge": 7, } dummy_mappingproxy = types.MappingProxyType(dummy_dict) - self.assertEqual(pprint.pformat(dummy_mappingproxy, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_mappingproxy, width=40, indent=4, + expand=True), """\ mappingproxy({ 'baz': 123, @@ -1760,7 +1819,8 @@ def test_expand_namespace(self): ), ) - self.assertEqual(pprint.pformat(dummy_namespace, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_namespace, width=40, indent=4, + expand=True), """\ namespace( foo='bar', @@ -1778,7 +1838,8 @@ def test_expand_defaultdict(self): dummy_defaultdict["foo"].append("baz") dummy_defaultdict["foo"].append("qux") dummy_defaultdict["bar"] = {"foo": "bar", "baz": None} - self.assertEqual(pprint.pformat(dummy_defaultdict, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_defaultdict, width=40, indent=4, + expand=True), """\ defaultdict(, { 'bar': {'baz': None, 'foo': 'bar'}, @@ -1795,7 +1856,8 @@ def test_expand_counter(self): 'd': 2, 'e': 1, })""" - self.assertEqual(pprint.pformat(dummy_counter, width=40, indent=4), expected) + self.assertEqual(pprint.pformat(dummy_counter, width=40, indent=4, + expand=True), expected) expected2 = """\ Counter({ @@ -1805,7 +1867,8 @@ def test_expand_counter(self): 'd': 2, 'e': 1, })""" - self.assertEqual(pprint.pformat(dummy_counter, width=20, indent=2), expected2) + self.assertEqual(pprint.pformat(dummy_counter, width=20, indent=2, + expand=True), expected2) def test_expand_chainmap(self): dummy_dict = { @@ -1821,7 +1884,8 @@ def test_expand_chainmap(self): {"corge": dummy_dict}, ) dummy_chainmap.maps.append({"garply": "waldo"}) - self.assertEqual(pprint.pformat(dummy_chainmap, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_chainmap, width=40, indent=4, + expand=True), """\ ChainMap( {'foo': 'bar'}, @@ -1863,7 +1927,8 @@ def test_expand_deque(self): dummy_deque.append(dummy_dict) dummy_deque.extend(dummy_list) dummy_deque.appendleft(dummy_set) - self.assertEqual(pprint.pformat(dummy_deque, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_deque, width=40, indent=4, + expand=True), """\ deque([ {(1, 2, 3)}, @@ -1894,7 +1959,8 @@ def __init__(self, *args, **kwargs): "corge": 7 }) dummy_userdict.access_count = 5 - self.assertEqual(pprint.pformat(dummy_userdict, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_userdict, width=40, indent=4, + expand=True), """\ { 'baz': 123, @@ -1914,7 +1980,8 @@ def __init__(self, *args, **kwargs): dummy_userlist = DummyUserList(["first", 2, {"key": "value"}, [4, 5, 6]]) - self.assertEqual(pprint.pformat(dummy_userlist, width=40, indent=4), + self.assertEqual(pprint.pformat(dummy_userlist, width=40, indent=4, + expand=True), """\ [ 'first', @@ -1926,7 +1993,7 @@ def __init__(self, *args, **kwargs): def test_expand_dict_keys(self): d = {"foo": 1, "bar": 2, "baz": 3, "qux": 4, "quux": 5} self.assertEqual( - pprint.pformat(d.keys(), width=20, indent=4), + pprint.pformat(d.keys(), width=20, indent=4, expand=True), """\ dict_keys([ 'bar', @@ -1940,7 +2007,7 @@ def test_expand_dict_keys(self): def test_expand_dict_values(self): d = {"foo": 1, "bar": 2, "baz": 3, "qux": 4, "quux": 5} self.assertEqual( - pprint.pformat(d.values(), width=20, indent=4), + pprint.pformat(d.values(), width=20, indent=4, expand=True), """\ dict_values([ 1, @@ -1954,7 +2021,7 @@ def test_expand_dict_values(self): def test_expand_dict_items(self): d = {"foo": 1, "bar": 2, "baz": 3, "qux": 4, "quux": 5} self.assertEqual( - pprint.pformat(d.items(), width=20, indent=4), + pprint.pformat(d.items(), width=20, indent=4, expand=True), """\ dict_items([ ('bar', 2), @@ -1968,7 +2035,7 @@ def test_expand_dict_items(self): def test_expand_str(self): s = "The quick brown fox jumped over the lazy dog " * 3 self.assertEqual( - pprint.pformat(s, width=40, indent=4), + pprint.pformat(s, width=40, indent=4, expand=True), """\ ( 'The quick brown fox jumped over ' diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b2acb1cf577341d..ee27fdd3fa3053c 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -345,6 +345,21 @@ def test_process_frames_tracks_edge_samples(self): # Check that edge count is tracked self.assertGreater(len(collector.edge_samples), 0) + def test_process_frames_weight_applies_to_identical_samples(self): + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', (5, 5, -1, -1), 'callee', None), + ('caller.py', (10, 10, -1, -1), 'caller', None), + ] + + collector.process_frames(frames, thread_id=1, weight=5) + + edge_key = (('caller.py', 10), ('callee.py', 5)) + self.assertEqual(collector.edge_samples[edge_key], 5) + self.assertEqual(collector.line_samples[('callee.py', 5)], 5) + self.assertEqual(collector.line_samples[('caller.py', 10)], 5) + def test_process_frames_handles_empty_frames(self): """Test that process_frames handles empty frame list.""" collector = HeatmapCollector(sample_interval_usec=100) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 9cf706aa2dafeee..5efc60a92111754 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -2,6 +2,7 @@ import json import os +import pathlib import random import struct import tempfile @@ -814,6 +815,35 @@ def test_invalid_file_path(self): with BinaryReader("/nonexistent/path/file.bin") as reader: reader.replay_samples(RawCollector()) + def test_path_arguments_round_trip(self): + """Reader and writer accept str, bytes or os.PathLike.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + for path_arg in (filename, os.fsencode(filename), pathlib.Path(filename)): + with self.subTest(path_type=type(path_arg).__name__): + writer = _remote_debugging.BinaryWriter(path_arg, 1000, 0) + writer.finalize() + reader = _remote_debugging.BinaryReader(path_arg) + info = reader.get_info() + reader.close() + self.assertEqual(info["sample_count"], 0) + + def test_rejects_non_pathlike(self): + """Reader and writer raise TypeError on non-path-like filenames.""" + with self.assertRaises(TypeError): + _remote_debugging.BinaryWriter(123, 1000, 0) + with self.assertRaises(TypeError): + _remote_debugging.BinaryReader(123) + + def test_invalid_path_error_preserves_pathlib(self): + """Missing path: OSError carries the original path object, not a string.""" + missing = pathlib.Path("/i/do/not/exist") + with self.assertRaises(FileNotFoundError) as cm: + _remote_debugging.BinaryReader(missing) + self.assertEqual(os.fspath(cm.exception.filename), os.fspath(missing)) + def test_writer_handles_empty_stack_first_sample(self): """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. @@ -945,7 +975,11 @@ def test_writer_total_samples_after_close_returns_zero(self): class TestBinaryFormatValidation(BinaryFormatTestBase): """Tests for malformed binary files.""" + HDR_OFF_SAMPLES = 28 HDR_OFF_THREADS = 32 + HDR_OFF_STR_TABLE = 36 + HDR_OFF_FRAME_TABLE = 44 + FILE_HEADER_PLACEHOLDER_SIZE = 64 def test_replay_rejects_more_threads_than_declared(self): """Replay rejects files with more unique threads than the header declares.""" @@ -970,6 +1004,43 @@ def test_replay_rejects_more_threads_than_declared(self): "threads than declared in header (declared 1, found at least 2)", ) + def test_replay_rejects_sample_count_mismatch(self): + """Replay rejects files whose decoded samples disagree with the header.""" + samples = [[make_interpreter(0, [ + make_thread(1, [make_frame("sample.py", 10, "sample")]) + ])]] + filename = self.create_binary_file(samples, compression="none") + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_SAMPLES) + raw.write(struct.pack("=I", 2)) + + with BinaryReader(filename) as reader: + self.assertEqual(reader.get_info()["sample_count"], 2) + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual( + str(cm.exception), + "Sample count mismatch: header declares 2 samples " + "but replay decoded 1", + ) + + def test_replay_rejects_trailing_partial_sample_header(self): + """Replay rejects partial sample bytes instead of silently stopping.""" + filename = self.create_binary_file([], compression="none") + sample_data_end = self.FILE_HEADER_PLACEHOLDER_SIZE + 1 + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_STR_TABLE) + raw.write(struct.pack("=Q", sample_data_end)) + raw.seek(self.HDR_OFF_FRAME_TABLE) + raw.write(struct.pack("=Q", sample_data_end)) + + with BinaryReader(filename) as reader: + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual(str(cm.exception), "Truncated sample data: 1 trailing bytes") + class TestBinaryEncodings(BinaryFormatTestBase): """Tests specifically targeting different stack encodings.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py index 102eb51b556cc77..1f4b6da32810561 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py @@ -39,8 +39,9 @@ class TestBlockingModeStackAccuracy(unittest.TestCase): @classmethod def setUpClass(cls): # Test script that uses a generator consumed in a loop. - # When consume_generator is on the arithmetic lines (temp1, temp2, etc.), - # fibonacci_generator should NOT be in the stack at all. + # When consume_generator is the executing leaf frame on the arithmetic + # lines (temp1, temp2, etc.), fibonacci_generator should NOT be in the + # stack at all. # Line numbers are important here - see ARITHMETIC_LINES below. cls.generator_script = textwrap.dedent(''' def fibonacci_generator(n): @@ -65,29 +66,32 @@ def main(): main() ''') # Line numbers of the arithmetic operations in consume_generator. - # These are the lines where fibonacci_generator should NOT be in the stack. - # The socket injection code adds 7 lines before our script. - # temp1 = value + 1 -> line 17 - # temp2 = value * 2 -> line 18 - # temp3 = value - 1 -> line 19 - # result = ... -> line 20 - cls.ARITHMETIC_LINES = {17, 18, 19, 20} + # These are the lines where fibonacci_generator should NOT be in the + # stack when consume_generator is the executing leaf frame. They account + # for the socket prelude added by test_subprocess(). + # temp1 = value + 1 -> line 16 + # temp2 = value * 2 -> line 17 + # temp3 = value - 1 -> line 18 + # result = ... -> line 19 + cls.ARITHMETIC_LINES = {16, 17, 18, 19} def test_generator_not_under_consumer_arithmetic(self): """Test that fibonacci_generator doesn't appear when consume_generator does arithmetic. - When consume_generator is executing arithmetic lines (temp1, temp2, etc.), - fibonacci_generator should NOT be anywhere in the stack - it's not being - called at that point. + When consume_generator is the leaf frame on arithmetic lines (temp1, + temp2, etc.), fibonacci_generator should NOT be anywhere in the stack - + it's not being called at that point. Non-leaf frame line numbers are + caller/resume metadata, not proof that the frame is executing. Valid stacks: - - consume_generator at 'for value in gen:' line WITH fibonacci_generator - at the top (generator is yielding) + - fibonacci_generator at the top (generator is executing), with + consume_generator below it - consume_generator at arithmetic lines WITHOUT fibonacci_generator (we're just doing math, not calling the generator) Invalid stacks (indicate torn/inconsistent reads): - - consume_generator at arithmetic lines WITH fibonacci_generator + - consume_generator leaf frame at arithmetic lines WITH + fibonacci_generator anywhere in the stack Note: call_tree is ordered from bottom (index 0) to top (index -1). @@ -110,6 +114,8 @@ def test_generator_not_under_consumer_arithmetic(self): total_samples = 0 invalid_stacks = 0 arithmetic_samples = 0 + generator_samples = 0 + generator_not_leaf_samples = 0 for (call_tree, _thread_id), count in collector.stack_counter.items(): total_samples += count @@ -117,15 +123,21 @@ def test_generator_not_under_consumer_arithmetic(self): if not call_tree: continue - # Find consume_generator in the stack and check its line number - for i, (filename, lineno, funcname) in enumerate(call_tree): - if funcname == "consume_generator" and lineno in self.ARITHMETIC_LINES: - arithmetic_samples += count - # Check if fibonacci_generator appears anywhere in this stack - func_names = [frame[2] for frame in call_tree] - if "fibonacci_generator" in func_names: - invalid_stacks += count - break + # Non-leaf frame line numbers can point at resume locations while + # a callee is the executing leaf frame. + _, lineno, funcname = call_tree[-1] + func_names = [frame[2] for frame in call_tree] + + if "fibonacci_generator" in func_names: + generator_samples += count + if funcname != "fibonacci_generator": + generator_not_leaf_samples += count + + if funcname == "consume_generator" and lineno in self.ARITHMETIC_LINES: + arithmetic_samples += count + # Check if fibonacci_generator appears anywhere in this stack. + if "fibonacci_generator" in func_names: + invalid_stacks += count self.assertGreater(total_samples, 10, f"Expected at least 10 samples, got {total_samples}") @@ -134,8 +146,15 @@ def test_generator_not_under_consumer_arithmetic(self): self.assertGreater(arithmetic_samples, 0, f"Expected some samples on arithmetic lines, got {arithmetic_samples}") + self.assertGreater(generator_samples, 0, + f"Expected some samples in fibonacci_generator, got {generator_samples}") + + self.assertEqual(generator_not_leaf_samples, 0, + f"Found {generator_not_leaf_samples}/{generator_samples} stacks where " + f"fibonacci_generator appears but is not the leaf frame.") + self.assertEqual(invalid_stacks, 0, f"Found {invalid_stacks}/{arithmetic_samples} invalid stacks where " f"fibonacci_generator appears in the stack when consume_generator " - f"is on an arithmetic line. This indicates torn/inconsistent stack " - f"traces are being captured.") + f"is the leaf frame on an arithmetic line. This indicates " + f"torn/inconsistent stack traces are being captured.") diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index bb49faa890f3481..e64d917eedde56b 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -109,6 +109,39 @@ def _wait_for_process_ready(proc, timeout): return proc.poll() is None +@unittest.skipIf( + _build_child_profiler_args is None, + "profiling.sampling.cli unavailable", +) +class TestChildProfilerArgBuilder(unittest.TestCase): + """Tests for child profiler CLI argument construction.""" + + def test_build_child_profiler_args_diff_flamegraph(self): + """Test child args use the real --diff-flamegraph flag.""" + args = argparse.Namespace( + sample_interval_usec=1000, + duration=None, + all_threads=False, + realtime_stats=False, + native=False, + gc=True, + opcodes=False, + async_aware=False, + mode="wall", + format="diff_flamegraph", + diff_baseline="baseline.bin", + ) + + child_args = _build_child_profiler_args(args) + + self.assertIn("--diff-flamegraph", child_args) + self.assertNotIn("--diff_flamegraph", child_args) + + flag_index = child_args.index("--diff-flamegraph") + self.assertGreater(len(child_args), flag_index + 1) + self.assertEqual(child_args[flag_index + 1], "baseline.bin") + + @requires_remote_subprocess_debugging() class TestGetChildPids(unittest.TestCase): """Tests for the get_child_pids function.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index b42e7aa579f40ca..1ab31af67fec522 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -11,6 +11,7 @@ try: import _remote_debugging # noqa: F401 + from profiling.sampling import gecko_collector from profiling.sampling.pstats_collector import PstatsCollector from profiling.sampling.stack_collector import ( CollapsedStackCollector, @@ -18,6 +19,7 @@ ) from profiling.sampling.jsonl_collector import JsonlCollector from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.heatmap_collector import _TemplateLoader from profiling.sampling.collector import extract_lineno, normalize_location from profiling.sampling.opcode_utils import get_opcode_info, format_opcode from profiling.sampling.constants import ( @@ -58,6 +60,42 @@ def find_child_by_name(children, strings, substr): return None +def export_gecko_profile(testcase, collector): + gecko_out = tempfile.NamedTemporaryFile(suffix=".json", delete=False) + testcase.addCleanup(close_and_unlink, gecko_out) + # We cannot overwrite an open file on Windows. + gecko_out.close() + + with captured_stdout(), captured_stderr(): + collector.export(gecko_out.name) + + testcase.assertGreater(os.path.getsize(gecko_out.name), 0) + with open(gecko_out.name, encoding="utf-8") as file: + return json.load(file) + + +def assert_gecko_column_lengths(testcase, table, columns): + expected = table["length"] + for column in columns: + testcase.assertEqual( + len(table[column]), expected, + f"{column!r} has wrong length", + ) + + +def gecko_marker_names(profile, markers): + string_array = profile["shared"]["stringArray"] + return [string_array[idx] for idx in markers["name"]] + + +def gecko_opcode_marker_data(profile): + markers = profile["threads"][0]["markers"] + return [ + data for data in markers["data"] + if data.get("type") == "Opcode" + ] + + class TestSampleProfilerComponents(unittest.TestCase): """Unit tests for individual profiler components.""" @@ -82,6 +120,18 @@ def test_mock_frame_info_with_empty_and_unicode_values(self): self.assertEqual(frame.location.lineno, 999999) self.assertEqual(frame.funcname, long_funcname) + def test_heatmap_navigation_restarts_line_highlight(self): + """Test heatmap navigation can replay target line highlights.""" + loader = _TemplateLoader() + + self.assertIn(".code-line:target", loader.file_css) + self.assertIn("function restartLineHighlight(target)", loader.file_js) + self.assertIn("target.style.animation = 'none'", loader.file_js) + self.assertIn("void target.offsetWidth", loader.file_js) + self.assertIn("url.href === window.location.href", loader.file_js) + self.assertIn("navigateToLine(JSON.parse(navData).link)", loader.file_js) + self.assertIn("navigateToLine(linkData.link)", loader.file_js) + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" # Test with zero interval @@ -570,9 +620,10 @@ def test_gecko_collector_basic(self): # Verify samples samples = thread_data["samples"] - self.assertEqual(len(samples["stack"]), 1) - self.assertEqual(len(samples["time"]), 1) self.assertEqual(samples["length"], 1) + assert_gecko_column_lengths( + self, samples, ("stack", "time", "eventDelay") + ) # Verify function table structure and content func_table = thread_data["funcTable"] @@ -609,9 +660,6 @@ def test_gecko_collector_basic(self): @unittest.skipIf(is_emscripten, "threads not available") def test_gecko_collector_export(self): """Test Gecko profile export functionality.""" - gecko_out = tempfile.NamedTemporaryFile(suffix=".json", delete=False) - self.addCleanup(close_and_unlink, gecko_out) - collector = GeckoCollector(1000) test_frames1 = [ @@ -644,17 +692,7 @@ def test_gecko_collector_export(self): collector.collect(test_frames2) collector.collect(test_frames3) - # Export gecko profile - with captured_stdout(), captured_stderr(): - collector.export(gecko_out.name) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(gecko_out.name)) - self.assertGreater(os.path.getsize(gecko_out.name), 0) - - # Check file contains valid JSON - with open(gecko_out.name, "r") as f: - profile_data = json.load(f) + profile_data = export_gecko_profile(self, collector) # Should be valid Gecko profile format self.assertIn("meta", profile_data) @@ -675,6 +713,100 @@ def test_gecko_collector_export(self): self.assertIn("func2", string_array) self.assertIn("other_func", string_array) + thread_data = profile_data["threads"][0] + assert_gecko_column_lengths( + self, thread_data["samples"], ("stack", "time", "eventDelay") + ) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_export_after_spill_flush(self): + """Test Gecko profile export after spill buffers flush to disk.""" + old_buffer_bytes = gecko_collector.DEFAULT_SPILL_BUFFER_BYTES + gecko_collector.DEFAULT_SPILL_BUFFER_BYTES = 1 + self.addCleanup( + setattr, gecko_collector, "DEFAULT_SPILL_BUFFER_BYTES", + old_buffer_bytes + ) + + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames, timestamps_us=[1000, 2000, 3000]) + + profile_data = export_gecko_profile(self, collector) + samples = profile_data["threads"][0]["samples"] + self.assertEqual(samples["length"], 3) + assert_gecko_column_lengths( + self, samples, ("stack", "time", "eventDelay") + ) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_rejects_collect_after_export(self): + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames) + export_gecko_profile(self, collector) + + with self.assertRaisesRegex(RuntimeError, "after export"): + collector.collect(test_frames) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_export_failure_keeps_existing_file(self): + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames) + + with tempfile.TemporaryDirectory() as temp_dir: + filename = os.path.join(temp_dir, "profile.json") + with open(filename, "w", encoding="utf-8") as file: + file.write("existing") + + before = set(os.listdir(temp_dir)) + + def fail(file): + raise OSError("boom") + + collector._stream_profile = fail + with captured_stdout(), captured_stderr(): + with self.assertRaisesRegex(OSError, "boom"): + collector.export(filename) + + with open(filename, encoding="utf-8") as file: + self.assertEqual(file.read(), "existing") + self.assertEqual(set(os.listdir(temp_dir)), before) + def test_gecko_collector_markers(self): """Test Gecko profile markers for GIL and CPU state tracking.""" collector = GeckoCollector(1000) @@ -758,21 +890,16 @@ def test_gecko_collector_markers(self): self.assertIn("markers", thread_data) markers = thread_data["markers"] - # Should have marker arrays - self.assertIn("name", markers) - self.assertIn("startTime", markers) - self.assertIn("endTime", markers) - self.assertIn("category", markers) self.assertGreater( markers["length"], 0, "Should have generated markers" ) - - # Get marker names from string table - string_array = profile_data["shared"]["stringArray"] - marker_names = [string_array[idx] for idx in markers["name"]] + assert_gecko_column_lengths( + self, markers, + ("data", "name", "startTime", "endTime", "phase", "category"), + ) # Verify we have different marker types - marker_name_set = set(marker_names) + marker_name_set = set(gecko_marker_names(profile_data, markers)) # Should have "Has GIL" markers (when thread had GIL) self.assertIn( @@ -1403,6 +1530,39 @@ def test_diff_flamegraph_elided_stacks(self): self.assertGreater(child["baseline"], 0) self.assertAlmostEqual(child["diff"], -child["baseline"]) + def test_diff_flamegraph_elided_top_level_root(self): + """Elided top-level roots do not crash metadata generation.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "kept_leaf"), + MockFrameInfo("file.py", 20, "kept_root"), + ]) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_leaf"), + MockFrameInfo("file.py", 40, "old_root"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([ + baseline_frames_1, + baseline_frames_2, + ]) + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + + self.assertEqual(len(children), 1) + self.assertIn("old_root", resolve_name(children[0], elided_strings)) + def test_diff_flamegraph_function_matched_despite_line_change(self): """Functions match by (filename, funcname), ignoring lineno.""" baseline_frames = [ @@ -2613,6 +2773,7 @@ def test_gecko_collector_opcodes_enabled(self): def test_gecko_opcode_state_tracking(self): """Test that GeckoCollector tracks opcode state changes.""" collector = GeckoCollector(sample_interval_usec=1000, opcodes=True) + self.addCleanup(collector._cleanup_spills) # First sample with opcode 90 (RAISE_VARARGS) frame1 = MockFrameInfo("test.py", 10, "func", opcode=90) @@ -2656,10 +2817,28 @@ def test_gecko_opcode_state_change_emits_marker(self): collector.collect(frames2) # Should have emitted a marker for the first opcode - thread_data = collector.threads[1] - markers = thread_data["markers"] - # At least one marker should have been added - self.assertGreater(len(markers["name"]), 0) + profile = collector._build_profile() + markers = profile["threads"][0]["markers"] + assert_gecko_column_lengths( + self, markers, + ("data", "name", "startTime", "endTime", "phase", "category"), + ) + opcode_markers = gecko_opcode_marker_data(profile) + self.assertIn( + { + "opcode": 90, + "line": 10, + "function": "func", + }, + [ + { + "opcode": marker["opcode"], + "line": marker["line"], + "function": marker["function"], + } + for marker in opcode_markers + ], + ) def test_gecko_opcode_markers_not_emitted_when_disabled(self): """Test that no opcode markers when opcodes=False.""" @@ -2683,8 +2862,9 @@ def test_gecko_opcode_markers_not_emitted_when_disabled(self): ] collector.collect(frames2) - # opcode_state should not be tracked - self.assertEqual(len(collector.opcode_state), 0) + profile = collector._build_profile() + self.assertEqual(gecko_opcode_marker_data(profile), []) + self.assertEqual(profile["meta"]["markerSchema"], []) def test_gecko_opcode_with_none_opcode(self): """Test that None opcode doesn't cause issues.""" @@ -2700,9 +2880,8 @@ def test_gecko_opcode_with_none_opcode(self): ] collector.collect(frames) - # Should track the state but opcode is None - self.assertIn(1, collector.opcode_state) - self.assertIsNone(collector.opcode_state[1][0]) + profile = collector._build_profile() + self.assertEqual(gecko_opcode_marker_data(profile), []) class TestCollectorFrameFormat(unittest.TestCase): diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index c6731e956391a91..3487647b76683ee 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -686,8 +686,6 @@ def test_script_error_treatment(self): "-m", "profiling.sampling.cli", "run", - "-d", - "1", script_file.name, ], capture_output=True, diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index c0d39f487c8cbdf..59373a8d00c03cf 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -835,8 +835,7 @@ def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): # still failing for _ in range(n_times): mock_self.display.simulate_input(-1) - if n_times >= 500: - mock_self.display.simulate_input(ord('q')) + mock_self.display.simulate_input(ord('q')) def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05c..2f5a5e273286590 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -198,8 +198,83 @@ def test_sample_profiler_sample_method_timing(self): self.assertIn("samples", result) # Verify collector was called multiple times - self.assertGreaterEqual(mock_collector.collect.call_count, 5) - self.assertLessEqual(mock_collector.collect.call_count, 11) + total_weight = sum( + len(c.kwargs.get("timestamps_us") or [None]) + for c in mock_collector.collect.call_args_list + ) + self.assertGreaterEqual(total_weight, 5) + self.assertLessEqual(total_weight, 11) + + def test_sample_profiler_does_not_buffer_non_aggregating_collectors(self): + """Test that non-aggregating collectors get each sample immediately.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = False + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + manager = mock.Mock() + manager.attach_mock(u.instance.get_stack_trace, "unwind") + manager.attach_mock(mock_collector.collect, "collect") + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [0.0, 0.01, 0.011, 0.02, 0.03] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.025) + + self.assertEqual( + manager.mock_calls, + [ + mock.call.unwind(), + mock.call.collect(stack_frames), + mock.call.unwind(), + mock.call.collect(stack_frames), + ], + ) + + def test_sample_profiler_flushes_aggregated_batches_at_limit(self): + """Test that aggregating collectors flush after MAX_PENDING_SAMPLES samples.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = True + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [ + 0.0, + 0.01, 0.011, + 0.02, 0.021, + 0.03, 0.031, + 0.04, 0.041, + 0.05, 0.051, + ] + with mock.patch("profiling.sampling.sample.MAX_PENDING_SAMPLES", 2): + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.045) + + batches = [ + (c.args[0], len(c.kwargs["timestamps_us"])) + for c in mock_collector.collect.call_args_list + ] + self.assertEqual( + batches, + [(stack_frames, 2), (stack_frames, 2), (stack_frames, 1)], + ) def test_sample_profiler_error_handling(self): """Test that the sample method handles errors gracefully.""" diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index aa090b464a72222..bdf57776c82be13 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -1,3 +1,5 @@ +import random +import string import sys import unittest from test.support import import_helper @@ -56,59 +58,57 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid) self.assertRaises(TypeError, pwd.getpwuid, 3.14) + self.assertRaises(TypeError, pwd.getpwuid, 0.0) + self.assertRaises(TypeError, pwd.getpwuid, 0, 0) + # should be out of uid_t range + self.assertRaises(KeyError, pwd.getpwuid, 2**128) + self.assertRaises(KeyError, pwd.getpwuid, -2**128) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwnam, b'root') + self.assertRaises(TypeError, pwd.getpwnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, pwd.getpwnam, 'roo\udc74') + self.assertRaises(KeyError, pwd.getpwnam, '') + self.assertRaises(TypeError, pwd.getpwall, 42) - # try to get some errors - bynames = {} - byuids = {} - for (n, p, u, g, gecos, d, s) in pwd.getpwall(): - bynames[n] = u - byuids[u] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] if allnames else "invaliduser" - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue - else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, pwd.getpwnam, fakename) - - # In some cases, byuids isn't a complete list of all users in the - # system, so if we try to pick a value not in byuids (via a perturbing - # loop, say), pwd.getpwuid() might still be able to find data for that - # uid. Using sys.maxint may provoke the same problems, but hopefully - # it will be a more repeatable failure. - fakeuid = sys.maxsize - self.assertNotIn(fakeuid, byuids) - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + # Find a non-existent user name. + # getpwall() will not necessarily report all existing users + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + pwd.getpwnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent user name') + + # Find a non-existent uid. + maxuid = max(e.pw_uid for e in pwd.getpwall()) + if maxuid < 2**15: + maxuid = 2**15 + elif maxuid < 2**16: + maxuid = 2**16-1 + else: + maxuid = 2**31 + for _ in range(30): + fakeuid = random.randrange(maxuid) + try: + pwd.getpwuid(fakeuid) + except KeyError: + break + else: + self.fail('Cannot find non-existent uid') + + # On Cygwin, getpwuid(-1) returns 'Unknown+User' user + if sys.platform != 'cygwin': + # -1 shouldn't be a valid uid because it has a special meaning in many + # uid-related functions + self.assertRaises(KeyError, pwd.getpwuid, -1) - # -1 shouldn't be a valid uid because it has a special meaning in many - # uid-related functions - self.assertRaises(KeyError, pwd.getpwuid, -1) - # should be out of uid_t range - self.assertRaises(KeyError, pwd.getpwuid, 2**128) - self.assertRaises(KeyError, pwd.getpwuid, -2**128) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2e190d1b81be8ec..5cd26923f75c311 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -2172,7 +2172,7 @@ def mock_getline(prompt): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['lazy'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index aaa91aca36e3c4c..10dca684accee3c 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -227,8 +227,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -276,6 +275,119 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + ]) + def test_supported_encodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_encodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'\n' + '' + '<корінь атрибут="значення">зміст').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'UTF-7', + "Big5-HKSCS", "Big5", + "cp932", "cp949", "cp950", + "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR", + "GB18030", "GB2312", "GBK", + "ISO-2022-KR", + "johab", + "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupported_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -712,6 +824,20 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) + @support.requires_resource('cpu') + @support.requires_resource('walltime') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_character_data_does_not_crash(self): + # See https://github.com/python/cpython/issues/148441 + parser = expat.ParserCreate() + parser.buffer_text = True + parser.buffer_size = 2**31 - 1 # INT_MAX + N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX + self.assertGreater(N, parser.buffer_size) + parser.CharacterDataHandler = lambda text: None + xml_data = b"" + b"A" * N + b"" + self.assertEqual(parser.Parse(xml_data, True), 1) + class ElementDeclHandlerTest(unittest.TestCase): def test_trigger_leak(self): # Unfixed, this test would leak the memory of the so-called diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 02f6e0c74b5ce84..874c6bb76b1afe5 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2283,7 +2283,8 @@ def test_python_opts(self): proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) + text=True, + env=support.make_clean_env()) self.assertEqual(proc.returncode, 0, proc) def test_add_python_opts(self): diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index a8914953ce9eb48..e6d727d417b2985 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -135,6 +136,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 3ea0ec66fbfbe9e..cd1477037e94b74 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -646,26 +646,23 @@ def test_group_without_user_agent(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -687,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -702,31 +698,79 @@ def testRead(self): self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) - self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -738,6 +782,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py index ec0ed37ffd047b7..f9ab9207c3c23d8 100644 --- a/Lib/test/test_samply_profiler.py +++ b/Lib/test/test_samply_profiler.py @@ -240,5 +240,29 @@ def compile_trampolines_for_all_functions(): self.assertIn(line, child_perf_file_contents) +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerWithJitDump(unittest.TestCase, TestSamplyProfilerMixin): + # Regression test for gh-150723: exercises the binary jitdump backend + # (-Xperf_jit) end to end through samply, unlike TestSamplyProfiler which + # uses the textual perf-map backend (-Xperf). + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf_jit", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.jit_files = set(pathlib.Path("/tmp/").glob("jit-*.dump")) + self.jit_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit-*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete -= self.jit_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 13a3487382dfcfb..c2485e20a199039 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2914,6 +2914,23 @@ def test_destinsrc_false_positive(self): finally: os_helper.rmtree(TESTFN) + @os_helper.skip_unless_symlink + def test_destinsrc_symlink_bypass(self): + tmp = self.mkdtemp() + src = os.path.join(tmp, 'src') + os.makedirs(src) + # tmp/link -> tmp (one level up) + link = os.path.join(tmp, 'link') + os.symlink(tmp, link) + # raw path: tmp/link/src/sub - no src prefix in string space + # real path: tmp/src/sub - physically inside src + dst = os.path.join(link, 'src', 'sub') + self.assertTrue( + shutil._destinsrc(src, dst), + msg='_destinsrc failed to detect dst inside src via symlink ' + '(dst=%s, src=%s)' % (dst, src), + ) + @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink(self): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index ac69e2cbdbbe547..5fd65ad999210e4 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -187,19 +187,17 @@ def test_addsitedir(self): self.pth_file_tests(pth_file) def test_addsitedir_explicit_flush(self): - # addsitedir() reads .pth files and, with - # defer_processing_start_files=True, accumulates pending state - # without flushing. A subsequent process_startup_files() call - # then applies the paths and runs the import lines. + # StartupState.addsitedir() reads .pth files and accumulates pending + # state without flushing. A subsequent state.process() call then + # applies the paths and runs the import lines. pth_file = PthFile() # Ensure we have a clean slate. pth_file.cleanup(prep=True) with pth_file.create(): - # Pass defer_processing_start_files=True to prevent flushing. - site.addsitedir(pth_file.base_dir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(pth_file.base_dir) self.assertNotIn(pth_file.imported, sys.modules) - site.process_startup_files() + state.process() self.pth_file_tests(pth_file) def test_addsitedir_dotfile(self): @@ -423,15 +421,14 @@ def create(self): Used as a context manager: self.cleanup() is called on exit. """ - FILE = open(self.file_path, 'w') - try: - print("#import @bad module name", file=FILE) - print("\n", file=FILE) - print("import %s" % self.imported, file=FILE) - print(self.good_dirname, file=FILE) - print(self.bad_dirname, file=FILE) - finally: - FILE.close() + with open(self.file_path, 'w') as fp: + print(f"""\ +#import @bad module name +import {self.imported} +{self.good_dirname} +{self.bad_dirname} +""", file=fp) + os.mkdir(self.good_dir_path) try: yield self @@ -456,6 +453,7 @@ def cleanup(self, prep=False): if os.path.exists(self.bad_dir_path): os.rmdir(self.bad_dir_path) + class ImportSideEffectTests(unittest.TestCase): """Test side-effects from importing 'site'.""" @@ -545,7 +543,6 @@ def test_customization_modules_on_startup(self): output = subprocess.check_output([sys.executable, '-s', '-c', '""']) self.assertNotIn(eyecatcher, output.decode('utf-8')) - @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -915,68 +912,121 @@ class StartFileTests(unittest.TestCase): def setUp(self): self.enterContext(import_helper.DirsOnSysPath()) self.tmpdir = self.sitedir = self.enterContext(os_helper.temp_dir()) - # Save and clear all pending dicts. - self.saved_entrypoints = site._pending_entrypoints.copy() - self.saved_syspaths = site._pending_syspaths.copy() - self.saved_importexecs = site._pending_importexecs.copy() - site._pending_entrypoints.clear() - site._pending_syspaths.clear() - site._pending_importexecs.clear() + # Each test gets its own StartupState to batch the parsing and + # explicitly invoke the processing. Seed with an empty known_paths + # so dedup is not influenced by the current sys.path. + self.state = site.StartupState(known_paths=set()) - def tearDown(self): - site._pending_entrypoints = self.saved_entrypoints.copy() - site._pending_syspaths = self.saved_syspaths.copy() - site._pending_importexecs = self.saved_importexecs.copy() + def _make_start(self, content, name='testpkg', basedir=None): + """Write a .start file and return its basename. - def _make_start(self, content, name='testpkg'): - """Write a .start file and return its basename.""" + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .start file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.start" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _make_pth(self, content, name='testpkg'): - """Write a .pth file and return its basename.""" + def _make_pth(self, content, name='testpkg', basedir=None): + """Write a .pth file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .pth file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.pth" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _all_entrypoints(self): - """Flatten _pending_entrypoints dict into a list of (filename, entry) tuples.""" + def _make_mod(self, contents, name='mod', *, package=False, on_path=False): + """Write an importable module (or package), returning its parent dir.""" + extdir = os.path.join(self.sitedir, 'extdir') + os.makedirs(extdir, exist_ok=True) + + # Put the code in a package's dunder-init or flat module. + if package: + pkgdir = os.path.join(extdir, name) + os.mkdir(pkgdir) + modpath = os.path.join(pkgdir, '__init__.py') + else: + modpath = os.path.join(extdir, f'{name}.py') + + with open(modpath, 'w') as fp: + fp.write(contents) + + self.addCleanup(sys.modules.pop, name, None) + if on_path: + # Don't worry, DirsOnSysPath() in setUp() will clean this up. + sys.path.insert(0, extdir) + return extdir + + def _all_entrypoints(self, state=None): + """Flatten state._entrypoints into a list of (filename, entry) tuples.""" result = [] - for filename, entries in site._pending_entrypoints.items(): + state = self.state if state is None else state + for filename, entries in state._entrypoints.items(): for entry in entries: result.append((filename, entry)) return result - def _just_entrypoints(self): - return [entry for filename, entry in self._all_entrypoints()] + def _just_entrypoints(self, state=None): + return [entry for filename, entry in self._all_entrypoints(state)] - # --- _read_start_file tests --- + # There are two classes of tests here. Tests that start with `test_impl_` + # know details about the implementation and they access non-public methods + # and data structures to perform focused functional tests. + # + # Tests that start with `test_addsitedir_` are end-to-end tests that ensure + # integration semantics and functionality as a caller of the public + # surfaces would see. - def test_read_start_file_basic(self): + # --- Basic StartupState implementation tests --- + + def test_impl_startupstate_defaults_to_sys_path(self): + sys.path.insert(0, self.sitedir) + state = site.StartupState() + self.assertIn(site.makepath(self.sitedir)[1], state._known_paths) + + def test_impl_startupstate_uses_supplied_known_paths(self): + known_paths = set() + state = site.StartupState(known_paths) + state.addsitedir(self.sitedir) + self.assertIs(state._known_paths, known_paths) + self.assertIn(site.makepath(self.sitedir)[1], known_paths) + + # --- StartupState._read_start_file tests --- + + def test_impl_read_start_file_basic(self): self._make_start("os.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_multiple_entries(self): + def test_impl_read_start_file_multiple_entries(self): self._make_start("os.path:join\nos.path:exists\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:exists']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:exists'], + ) - def test_read_start_file_comments_and_blanks(self): + def test_impl_read_start_file_comments_and_blanks(self): self._make_start("# a comment\n\nos.path:join\n \n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_accepts_all_non_blank_lines(self): + def test_impl_read_start_file_accepts_all_non_blank_lines(self): # Syntax validation is deferred to entry-point execution time # (where pkgutil.resolve_name(strict=True) enforces the strict # pkg.mod:callable form), so parsing accepts every non-blank, @@ -989,9 +1039,9 @@ def test_read_start_file_accepts_all_non_blank_lines(self): "os.path:join\n" # valid ) self._make_start(content, name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], [ + self.assertEqual(self.state._entrypoints[fullname], [ 'os.path', 'pkg.mod:', ':callable', @@ -999,155 +1049,167 @@ def test_read_start_file_accepts_all_non_blank_lines(self): 'os.path:join', ]) - def test_read_start_file_empty(self): + def test_impl_read_start_file_empty(self): # PEP 829: an empty .start file is still registered as present - # (with an empty entry-point list) so that it suppresses `import` + # (with an empty entry point list) so that it suppresses `import` # lines in any matching .pth file. self._make_start("", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_comments_only(self): + def test_impl_read_start_file_comments_only(self): # As with an empty file, a comments-only .start file is registered # as present so it can suppress matching .pth `import` lines. self._make_start("# just a comment\n# another\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_nonexistent(self): + def test_impl_read_start_file_nonexistent(self): with captured_stderr(): - site._read_start_file(self.tmpdir, 'nonexistent.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.tmpdir, 'nonexistent.start') + self.assertEqual(self.state._entrypoints, {}) @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') - def test_read_start_file_hidden_flags(self): + def test_impl_read_start_file_hidden_flags(self): self._make_start("os.path:join\n", name='foo') filepath = os.path.join(self.tmpdir, 'foo.start') st = os.stat(filepath) os.chflags(filepath, st.st_flags | stat.UF_HIDDEN) - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) - def test_read_start_file_duplicates_not_deduplicated(self): + def test_impl_one_start_file_with_duplicates_not_deduplicated(self): # PEP 829: duplicate entry points are NOT deduplicated. self._make_start("os.path:join\nos.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:join'], + ) - def test_read_start_file_accepts_utf8_bom(self): + def test_impl_two_start_files_with_duplicates_not_deduplicated(self): + self._make_start("os.path:join", name="foo") + self._make_start("os.path:join", name="bar") + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'bar.start') + self.assertEqual( + self._just_entrypoints(), + ['os.path:join', 'os.path:join'], + ) + + def test_impl_read_start_file_accepts_utf8_bom(self): # PEP 829: .start files MUST be utf-8-sig (UTF-8 with optional BOM). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: f.write(b'\xef\xbb\xbf' + b'os.path:join\n') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') self.assertEqual( - site._pending_entrypoints[fullname], ['os.path:join']) + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_invalid_utf8_silently_skipped(self): - # PEP 829: .start files MUST be utf-8-sig. Unlike .pth, there is - # no locale-encoding fallback -- a .start file that is not valid + def test_impl_read_start_file_invalid_utf8_silently_skipped(self): + # PEP 829: .start files MUST be utf-8-sig. Unlike .pth files, there + # is no locale-encoding fallback. A .start file that is not valid # UTF-8 is silently skipped, with no key registered in - # _pending_entrypoints and no output to stderr (parsing errors - # are reported only under -v). + # state._entrypoints and no output to stderr (parsing errors are + # reported only under -v). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: # Bare continuation byte -- invalid as a UTF-8 start byte. f.write(b'\x80\x80\x80\n') with captured_stderr() as err: - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) self.assertEqual(err.getvalue(), "") - def test_two_start_files_with_duplicates_not_deduplicated(self): - self._make_start("os.path:join", name="foo") - self._make_start("os.path:join", name="bar") - site._read_start_file(self.sitedir, 'foo.start') - site._read_start_file(self.sitedir, 'bar.start') - self.assertEqual(self._just_entrypoints(), - ['os.path:join', 'os.path:join']) + # --- StartupState._read_pth_file tests --- - # --- _read_pth_file tests --- - - def test_read_pth_file_paths(self): + def test_impl_read_pth_file_paths(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths[fullname]) + self.assertIn((fullname, subdir), self.state._path_entries) - def test_read_pth_file_imports_collected(self): + def test_impl_read_pth_file_imports_collected(self): self._make_pth("import sys\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertEqual(site._pending_importexecs[fullname], ['import sys']) + self.assertEqual( + self.state._importexecs[fullname], ['import sys'] + ) - def test_read_pth_file_comments_and_blanks(self): + def test_impl_read_pth_file_comments_and_blanks(self): self._make_pth("# comment\n\n \n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(site._pending_syspaths, {}) - self.assertEqual(site._pending_importexecs, {}) + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.assertEqual(self.state._path_entries, []) + self.assertEqual(self.state._importexecs, {}) - def test_read_pth_file_deduplication(self): + def test_impl_read_pth_file_deduplication(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) - known_paths = set() + # self.state._known_paths acts as the deduplication ledger across + # both reads. self._make_pth("mylib\n", name='a') self._make_pth("mylib\n", name='b') - site._read_pth_file(self.sitedir, 'a.pth', known_paths) - site._read_pth_file(self.sitedir, 'b.pth', known_paths) - # Only one entry across both files. - all_dirs = [] - for dirs in site._pending_syspaths.values(): - all_dirs.extend(dirs) + self.state._read_pth_file(self.sitedir, 'a.pth') + self.state._read_pth_file(self.sitedir, 'b.pth') + # There is only one entry across both files. + all_dirs = [dir_ for filename, dir_ in self.state._path_entries] self.assertEqual(all_dirs, [subdir]) - def test_read_pth_file_bad_line_continues(self): - # PEP 829: errors on individual lines don't abort the file. + def test_impl_read_pth_file_bad_line_continues(self): + # PEP 829: errors on individual lines don't abort processing the file. subdir = os.path.join(self.sitedir, 'goodpath') os.mkdir(subdir) self._make_pth("abc\x00def\ngoodpath\n", name='foo') with captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), self.state._path_entries) def _flags_with_verbose(self, verbose): # Build a sys.flags clone with verbose overridden but every # other field preserved, so unrelated reads like # sys.flags.optimize during io.open_code() continue to work. - attrs = {name: getattr(sys.flags, name) - for name in sys.flags.__match_args__} + attrs = { + name: getattr(sys.flags, name) + for name in sys.flags.__match_args__ + } attrs['verbose'] = verbose return SimpleNamespace(**attrs) - def test_read_pth_file_parse_error_silent_by_default(self): + def test_impl_read_pth_file_parse_error_silent_by_default(self): # PEP 829: parse-time errors are silent unless -v is given. - # Force the error path by making makepath() raise. + # Force the error path by making makepath() raise an exception. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(False)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(False)), + captured_stderr() as err, + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') self.assertEqual(err.getvalue(), "") - def test_read_pth_file_parse_error_reported_under_verbose(self): + def test_impl_read_pth_file_parse_error_reported_under_verbose(self): # PEP 829: parse-time errors are reported when -v is given. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(True)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(True)), + captured_stderr() as err, + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') out = err.getvalue() self.assertIn('Error in', out) self.assertIn('foo.pth', out) - def test_read_pth_file_locale_fallback(self): + def test_impl_read_pth_file_locale_fallback(self): # PEP 829: .pth files that fail UTF-8 decoding fall back to the # locale encoding for backward compatibility (deprecated in # 3.15, to be removed in 3.20). Mock locale.getencoding() so @@ -1158,186 +1220,215 @@ def test_read_pth_file_locale_fallback(self): # \xe9 is invalid UTF-8 but valid in latin-1. with open(filepath, 'wb') as f: f.write(b'# caf\xe9 comment\nmylib\n') - with mock.patch('locale.getencoding', return_value='latin-1'), \ - captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('locale.getencoding', return_value='latin-1'), + captured_stderr(), + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), self.state._path_entries) - # --- _execute_start_entrypoints tests --- + # --- StartupState._execute_start_entrypoints tests --- - def test_execute_entrypoints_with_callable(self): - # Entrypoint with callable is invoked. - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_with_callable(self): + # An entry point with a callable. + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) +""", name='epmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'epmod.start') - site._pending_entrypoints[fullname] = ['epmod:startup'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = ['epmod:startup'] + self.state._execute_start_entrypoints() import epmod self.assertTrue(epmod.called) - def test_execute_entrypoints_import_error(self): - # Import error prints traceback but continues. + def test_impl_execute_entrypoints_import_error(self): + # Import errors print a traceback and continue. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'nosuchmodule_xyz:func', 'os.path:join'] + self.state._entrypoints[fullname] = [ + 'nosuchmodule_xyz:func', 'os.path:join', + ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('nosuchmodule_xyz', err.getvalue()) # os.path:join should still have been called (no exception for it) - def test_execute_entrypoints_strict_syntax_rejection(self): - # PEP 829: only the strict pkg.mod:callable form is valid. - # At entry-point execution, pkgutil.resolve_name(strict=True) - # raises ValueError for invalid syntax; the invalid entry is - # reported and execution continues with the next one. + def test_impl_execute_entrypoints_strict_syntax_rejection(self): + # PEP 829: only the strict pkg.mod:callable form is valid. At entry + # point execution time, pkgutil.resolve_name(strict=True) raises a + # ValueError for the invalid syntax. The invalid entry is reported + # and execution continues with the next one. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ + self.state._entrypoints[fullname] = [ 'os.path', # no colon 'pkg.mod:', # empty callable ':callable', # empty module 'pkg.mod:callable:extra', # multiple colons ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() out = err.getvalue() self.assertIn('Invalid entry point syntax', out) - for bad in ('os.path', 'pkg.mod:', ':callable', - 'pkg.mod:callable:extra'): + for bad in ( + 'os.path', + 'pkg.mod:', + ':callable', + 'pkg.mod:callable:extra', + ): self.assertIn(bad, out) - def test_execute_entrypoints_callable_error(self): - # Callable that raises prints traceback but continues. - mod_dir = os.path.join(self.sitedir, 'badmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_callable_error(self): + # A callable that errors prints a traceback but continues. + self._make_mod("""\ def fail(): raise RuntimeError("boom") -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'badmod', None) +""", name='badmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'badmod.start') - site._pending_entrypoints[fullname] = ['badmod:fail'] + self.state._entrypoints[fullname] = ['badmod:fail'] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('RuntimeError', err.getvalue()) self.assertIn('boom', err.getvalue()) - def test_execute_entrypoints_duplicates_called_twice(self): + def test_impl_execute_entrypoints_duplicates_called_twice(self): # PEP 829: duplicate entry points execute multiple times. - mod_dir = os.path.join(self.sitedir, 'countmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ call_count = 0 def bump(): global call_count call_count += 1 -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'countmod', None) +""", name='countmod', package=False, on_path=True) fullname = os.path.join(self.sitedir, 'countmod.start') - site._pending_entrypoints[fullname] = [ - 'countmod:bump', 'countmod:bump'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = [ + 'countmod:bump', 'countmod:bump', + ] + self.state._execute_start_entrypoints() import countmod self.assertEqual(countmod.call_count, 2) - # --- _exec_imports tests --- + # --- StartupState._exec_imports tests --- - def test_exec_imports_suppressed_by_matching_start(self): + def test_impl_exec_imports_suppressed_by_matching_start(self): # Import lines from foo.pth are suppressed when foo.start exists. - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should not exec the import line; no error expected. - site._exec_imports() + self._make_mod("""\ +call_count = 0 +def bump(incr=2): + global call_count + call_count += incr +""", name='countmod', package=False, on_path=True) + self._make_start("countmod:bump\n", name='foo') + self._make_pth("import countmod; countmod.bump(1)\n", name='foo') + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() + self.state._execute_start_entrypoints() + import countmod + # This will be 2 because the entry point is called with no + # arguments, and the .pth import line is never exec'd. + self.assertEqual(countmod.call_count, 2) - def test_exec_imports_not_suppressed_by_different_start(self): + def test_impl_exec_imports_not_suppressed_by_different_start(self): # Import lines from foo.pth are NOT suppressed by bar.start. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'bar.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should execute the import line without error. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 1) - def test_exec_imports_suppressed_by_empty_matching_start(self): + def test_impl_exec_imports_suppressed_by_empty_matching_start(self): self._make_start("", name='foo') self._make_pth("import epmod; epmod.startup()", name='foo') - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - site._read_pth_file(self.sitedir, 'foo.pth', set()) - site._read_start_file(self.sitedir, 'foo.start') - site._exec_imports() +""", name='epmod', package=True, on_path=True) + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() import epmod self.assertFalse(epmod.called) - # --- _extend_syspath tests --- + # --- StartupState._extend_syspath tests --- - def test_extend_syspath_existing_dir(self): + def test_impl_extend_syspath_existing_dir(self): subdir = os.path.join(self.sitedir, 'extlib') os.mkdir(subdir) - site._pending_syspaths['test.pth'] = [subdir] - site._extend_syspath() + self.state._path_entries.append(('test.pth', subdir)) + self.state._extend_syspath() self.assertIn(subdir, sys.path) - def test_extend_syspath_nonexistent_dir(self): - nosuch = os.path.join(self.sitedir, 'nosuchdir') - site._pending_syspaths['test.pth'] = [nosuch] + def test_impl_extend_syspath_nonexistent_dir(self): + nonesuch = os.path.join(self.sitedir, 'nosuchdir') + self.state._path_entries.append(('test.pth', nonesuch)) with captured_stderr() as err: - site._extend_syspath() - self.assertNotIn(nosuch, sys.path) + self.state._extend_syspath() + self.assertNotIn(nonesuch, sys.path) self.assertIn('does not exist', err.getvalue()) # --- addsitedir integration tests --- + def test_addsitedir_pth_import_skipped_when_matching_start_exists(self): + # PEP 829: an empty .start file disables the matching .pth's import + # lines, even when the .start has no entry points of its own. + self._make_mod("flag = False\n", name='suppressed', on_path=True) + self._make_start("", name='foo') + self._make_pth( + "import suppressed; suppressed.flag = True\n", + name='foo') + site.addsitedir(self.sitedir, set()) + import suppressed + self.assertFalse( + suppressed.flag, + "import line in foo.pth should be suppressed by foo.start") + + def test_addsitedir_dotfile_start_entrypoint_not_executed(self): + # .start files starting with '.' are skipped, so their entry + # points must not run. + self._make_mod("""\ +called = False +def hook(): + global called + called = True +""", + name='dotted', on_path=True) + self._make_start("dotted:hook\n", name='.hidden') + site.addsitedir(self.sitedir, set()) + import dotted + self.assertFalse(dotted.called) + + def test_addsitedir_dedups_paths_across_pth_files(self): + # PEP 829: when multiple .pth files reference the same path within + # a single addsitedir() invocation, the path is appended to + # sys.path exactly once. + subdir = os.path.join(self.sitedir, 'shared') + os.mkdir(subdir) + self._make_pth("shared\n", name='a') + self._make_pth("shared\n", name='b') + before = sys.path.count(subdir) + site.addsitedir(self.sitedir, set()) + self.assertEqual(sys.path.count(subdir), before + 1) + def test_addsitedir_discovers_start_files(self): # addsitedir() should discover .start files and accumulate entries. self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn('os.path:join', site._pending_entrypoints[fullname]) - - def test_addsitedir_start_suppresses_pth_imports(self): - # When foo.start exists, import lines in foo.pth are skipped - # at flush time by _exec_imports(). - self._make_start("os.path:join\n", name='foo') - self._make_pth("import sys\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - # Import line was collected... - self.assertIn('import sys', - site._pending_importexecs.get(pth_fullname, [])) - # ...but _exec_imports() will skip it because foo.start exists. - site._exec_imports() + self.assertIn( + 'os.path:join', state._entrypoints[fullname] + ) def test_addsitedir_pth_paths_still_work_with_start(self): # Path lines in .pth files still work even when a .start file exists. @@ -1345,74 +1436,129 @@ def test_addsitedir_pth_paths_still_work_with_start(self): os.mkdir(subdir) self._make_start("os.path:join\n", name='foo') self._make_pth("mylib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), state._path_entries) def test_addsitedir_start_alphabetical_order(self): # Multiple .start files are discovered alphabetically. self._make_start("os.path:join\n", name='zzz') self._make_start("os.path:exists\n", name='aaa') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - all_entries = self._all_entrypoints() - entries = [entry for _, entry in all_entries] + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + entries = self._just_entrypoints(state) idx_a = entries.index('os.path:exists') idx_z = entries.index('os.path:join') self.assertLess(idx_a, idx_z) - def test_addsitedir_pth_before_start(self): - # PEP 829: .pth files are scanned before .start files. - # Create a .pth and .start with the same basename; verify - # the .pth data is collected before .start data. + def test_addsitedir_pth_and_start(self): + # Create a .pth and .start with the same basename; verify both the + # .pth data and .start data is collected. subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) # Both should be collected. pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn(subdir, site._pending_syspaths.get(pth_fullname, [])) - self.assertIn('os.path:join', - site._pending_entrypoints.get(start_fullname, [])) - - def test_addsitedir_dotfile_start_ignored(self): - # .start files starting with '.' are skipped. Defer flushing so - # the assertion against _pending_entrypoints is meaningful; - # otherwise process_startup_files() would clear the dict - # regardless of whether the dotfile was picked up. + self.assertIn((pth_fullname, subdir), state._path_entries) + self.assertIn( + 'os.path:join', + state._entrypoints.get(start_fullname, []), + ) + + def test_impl_addsitedir_skips_dotfile_start(self): + # .start files starting with '.' are skipped. + # This will create `.hidden.start`. self._make_start("os.path:join\n", name='.hidden') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - self.assertEqual(site._pending_entrypoints, {}) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + self.assertEqual(state._entrypoints, {}) def test_addsitedir_standalone_flushes(self): - # When called with known_paths=None (standalone), addsitedir - # flushes immediately so the caller sees the effect. + # Standalone addsitedir creates a per-call StartupState and processes + # it before returning, so the caller sees the effect immediately. subdir = os.path.join(self.sitedir, 'flushlib') os.mkdir(subdir) self._make_pth("flushlib\n", name='foo') - site.addsitedir(self.sitedir) # known_paths=None + # No arguments means state is implied and processing is eager. + site.addsitedir(self.sitedir) self.assertIn(subdir, sys.path) - # Pending dicts should be cleared after flush. - self.assertEqual(site._pending_syspaths, {}) - def test_addsitedir_defer_does_not_flush(self): - # With defer_processing_start_files=True, addsitedir accumulates - # pending state but does not flush; sys.path is updated only when - # process_startup_files() is called explicitly. + def test_addsitedir_explicit_startup_state_does_not_flush(self): + # With an explicit StartupState, addsitedir accumulates pending state + # but does not flush it; sys.path is updated only when process() is + # called explicitly. subdir = os.path.join(self.sitedir, 'acclib') os.mkdir(subdir) self._make_pth("acclib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) # Path is pending, not yet on sys.path. self.assertNotIn(subdir, sys.path) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), state._path_entries) + + def test_addsitedir_startup_state_preserves_site_relative_order(self): + # As pointed out by @ncoghlan in + # https://github.com/python/cpython/issues/150228#issuecomment-4528614952 + # a subtle ordering change was inadvertently introduced where the + # interspersing of the sitedirs with the sys.path extensions they defined + # was lost during batch mode. You'd see all the sitedirs, then all path + # extensions. This test ensures that the old interspersing behavior + # has been restored. + # + # Let's start by creating two sitedirs, each with an extension directory + # which will be added to sys.path by .pth files in the respective sitedirs. + sitedir2 = self.enterContext(os_helper.temp_dir()) + extdir1 = os.path.join(self.sitedir, 'ext1') + extdir2 = os.path.join(sitedir2, 'ext2') + os.mkdir(extdir1) + os.mkdir(extdir2) + self._make_pth(extdir1 + "\n", name='one') + self._make_pth(extdir2 + "\n", name='two', basedir=sitedir2) + # Now create an explicit batch, add each sitedir, then process the + # entire batch. + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + state.addsitedir(sitedir2) + state.process() + # Ensure that on sys.path we see this interspersed order: + # [sitedir1, extdir1, sitedir2, extdir2] + indexes = [ + sys.path.index(path) + for path in (self.sitedir, extdir1, sitedir2, extdir2) + ] + # If the index ordering is the same, we preserved the intersperse. + self.assertEqual(indexes, sorted(indexes)) + + def test_addsitedir_startup_state_paths_before_entrypoints(self): + # Ensure that sys.path extensions are available by the time + # .start file entry points are called. + extdir = self._make_mod("""\ +called = False +def hook(): + global called + called = True +""") + self.assertNotIn(extdir, sys.path) + self._make_pth("extdir\n", name='extlib') + self._make_start("mod:hook\n", name='extlib') + # Before the startup state is explicitly processed, neither + # the path extension is added, nor the entry point called. + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + self.assertNotIn(extdir, sys.path) + self.assertNotIn('mod', sys.modules) + # After processing the batch, sys.path is extended and + # the entry point was called. + state.process() + self.assertIn(extdir, sys.path) + import mod + self.assertTrue(mod.called) def test_pth_path_is_available_to_start_entrypoint(self): # Core PEP 829 invariant: all .pth path extensions are applied to @@ -1420,18 +1566,12 @@ def test_pth_path_is_available_to_start_entrypoint(self): # point may live in a module reachable only via a .pth-extended # path. If the flush phases were inverted, resolving the entry # point would fail with ModuleNotFoundError. - extdir = os.path.join(self.sitedir, 'extdir') - os.mkdir(extdir) - modpath = os.path.join(extdir, 'mod.py') - with open(modpath, 'w') as f: - f.write("""\ + extdir = self._make_mod("""\ called = False def hook(): global called called = True """) - self.addCleanup(sys.modules.pop, 'mod', None) - # extdir is not on sys.path; only the .pth file makes it so. self.assertNotIn(extdir, sys.path) self._make_pth("extdir\n", name='extlib') @@ -1447,6 +1587,156 @@ def hook(): "entry point did not run; .pth path was likely not applied " "before .start entry-point execution") + # --- bugs --- + + # gh-75723 + def test_addsitdir_idempotent_pth(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +""") + self._make_pth(f"""\ +{extdir} +import mod; mod._pth_count += 1 +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + def test_addsitdir_idempotent_start(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +def increment(): + global _pth_count + _pth_count += 1 +""") + self._make_pth(f"""\ +{extdir} +""") + self._make_start("""\ +mod:increment +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + # gh-149504 + def test_reentrant_addsitedir_pth(self): + # An import line in a .pth file that calls site.addsitedir() + # must not crash or re-execute outer entries while the outer + # call is still processing its pending startup state. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_pth(f"import site; site.addsitedir({overlay!r})\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149504 + def test_reentrant_addsitedir_start(self): + # As above, but the re-entry happens from a .start entry point + # instead of a .pth import line. The entry point execution + # phase is vulnerable to the same class of bug. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_mod(f"""\ +import site +def bootstrap(): + site.addsitedir({overlay!r}) +""", + name='reenter_helper', on_path=True) + self._make_start("reenter_helper:bootstrap\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149819 + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_pth_processed_when_sitedir_already_on_path(self): + # A .pth file in a site-packages directory must still be processed by + # site.main() when that directory is already on sys.path at + # interpreter start up, for example in a subprocess that inherits + # PYTHONPATH from its parent. Before the fix, main() seeded + # known_paths with all entries derived from removeduppaths(), and + # addsitedir() then skipped .pth processing for any directory already + # in known_paths. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_PTH_RAN" + # Writing some text to stderr is the simplest observable side effect. + self._make_pth(f"""\ +import sys; sys.stderr.write({sentinel!r}); sys.stderr.flush() +""", + name='gh149819', + basedir=user_site) + with EnvironmentVarGuard() as env: + # PYTHONUSERBASE points USER_SITE at our temp directory so + # site.main() will call addsitedir() on it, rather than on the + # host interpreter's real user-site. + env['PYTHONUSERBASE'] = user_base + # PYTHONPATH puts that same directory on sys.path before + # site.main() runs in the subprocess. This is what triggers the + # bug: removeduppaths() records it in known_paths, and the unfixed + # addsitedir() then skips .pth processing. + env['PYTHONPATH'] = user_site + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_start_processed_when_sitedir_already_on_path(self): + # Companion to test_pth_processed_when_sitedir_already_on_path: + # the same dedup-guard skip in addsitedir() suppressed both .pth + # and .start file processing, so verify .start entry points also + # run for a site-packages directory inherited via PYTHONPATH. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_START_RAN" + # The .start entry point resolves to a callable, so we write a + # tiny importable module that outputs the sentinel text. It lands in + # /extdir. That path is added to PYTHONPATH below so + # the subprocess can import it. + extdir = self._make_mod(f"""\ +import sys +def run(): + sys.stderr.write({sentinel!r}) + sys.stderr.flush() +""", name='gh149819mod') + self._make_start( + 'gh149819mod:run\n', name='gh149819', basedir=user_site + ) + with EnvironmentVarGuard() as env: + # See above for details. + env['PYTHONUSERBASE'] = user_base + env['PYTHONPATH'] = os.pathsep.join([user_site, extdir]) + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 9e03069494345b3..47830d0e9645efc 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -205,6 +205,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -247,10 +266,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 73b40e82a96811f..5f6cb527955ca17 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1400,6 +1400,18 @@ def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) + def test_blob_set_empty_slice_wrong_type(self): + with self.assertRaises(TypeError): + self.blob[5:5] = None + + def test_blob_set_empty_slice_wrong_size(self): + with self.assertRaisesRegex(IndexError, "wrong size"): + self.blob[5:5] = b"123" + + def test_blob_set_empty_slice_correct(self): + self.blob[5:5] = b"" + self.assertEqual(self.blob[:], self.data) + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 776659e3b161089..a9abeab31936880 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -146,6 +146,16 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string + def test_delete_connection_row_factory(self): + # gh-149738: deleting row_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.row_factory + + def test_delete_connection_text_factory(self): + # gh-149738: deleting text_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.text_factory + def test_sqlite_row_index_unicode(self): row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f1f7a07701de165..f41262d81a82904 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1606,6 +1606,59 @@ def dummycallback(sock, servername, ctx, cycle=ctx): gc.collect() self.assertIs(wr(), None) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "test is only useful if the GIL is disabled") + @threading_helper.requires_working_threading() + def test_sni_callback_race(self): + # Replacing sni_callback while handshakes are in-flight must not + # crash (use-after-free on the callback in free-threaded builds). + client_ctx, server_ctx, hostname = testing_context() + + server_ctx.sni_callback = lambda *a: None + done = threading.Event() + + def do_handshakes(): + while not done.is_set(): + c_in = ssl.MemoryBIO() + c_out = ssl.MemoryBIO() + s_in = ssl.MemoryBIO() + s_out = ssl.MemoryBIO() + client = client_ctx.wrap_bio( + c_in, c_out, server_hostname=hostname) + server = server_ctx.wrap_bio(s_in, s_out, server_side=True) + for _ in range(50): + try: + client.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if c_out.pending: + s_in.write(c_out.read()) + try: + server.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if s_out.pending: + c_in.write(s_out.read()) + + def toggle_callback(): + while not done.is_set(): + server_ctx.sni_callback = lambda *a: None + server_ctx.sni_callback = None + + workers = max(4, (os.cpu_count() or 4) * 2) + threads = [threading.Thread(target=do_handshakes) + for _ in range(workers)] + threads.append(threading.Thread(target=toggle_callback)) + + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + done.set() + self.assertIsNone(cm.exc_value) + def test_cert_store_stats(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.cert_store_stats(), @@ -5642,17 +5695,24 @@ def non_linux_skip_if_other_okay_error(self, err): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I) or - re.search('record.layer.failure', str(getattr(err, "reason", "")), re.I) + re.search( + # Matches the following error messages: + # '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)' + # '[SSL: RECORD_LAYER_FAILURE] record layer failure (_ssl.c:1109)' + # '[SSL: HTTP_REQUEST] http request (_ssl.c:1143)' + r'wrong.version.number|record.layer.failure|http.request', + str(getattr(err, "reason", "")), + re.IGNORECASE, + ) ): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing # the actual wrapping... we get an SSLError from OpenSSL. This is # typically WRONG_VERSION_NUMBER. The same happens on iOS, but - # RECORD_LAYER_FAILURE is the error. + # RECORD_LAYER_FAILURE or HTTP_REQUEST is the error. # - # While appropriate, neither is the scenario we're specifically + # While appropriate, these scenarios aren't what we're specifically # trying to test. The way this test is written is known to work on # Linux. We'll skip it anywhere else that it does not present as # doing so. diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index ac5c4296c663d07..09ee2d53f98f585 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -1032,19 +1032,15 @@ def test_windows_feature_macros(self): 'PyOS_CheckStack', ) -EXPECTED_FEATURE_MACROS = set([ - 'HAVE_FORK', - 'MS_WINDOWS', - 'PY_HAVE_THREAD_NATIVE_ID', - 'Py_REF_DEBUG', - 'Py_TRACE_REFS', - 'USE_STACKCHECK', -]) -WINDOWS_FEATURE_MACROS = { - 'HAVE_FORK': False, - 'MS_WINDOWS': True, - 'PY_HAVE_THREAD_NATIVE_ID': True, - 'Py_REF_DEBUG': 'maybe', - 'Py_TRACE_REFS': 'maybe', - 'USE_STACKCHECK': 'maybe', -} +EXPECTED_FEATURE_MACROS = set(['HAVE_FORK', + 'MS_WINDOWS', + 'PY_HAVE_THREAD_NATIVE_ID', + 'Py_REF_DEBUG', + 'Py_TRACE_REFS', + 'USE_STACKCHECK']) +WINDOWS_FEATURE_MACROS = {'HAVE_FORK': False, + 'MS_WINDOWS': True, + 'PY_HAVE_THREAD_NATIVE_ID': True, + 'Py_REF_DEBUG': 'maybe', + 'Py_TRACE_REFS': 'maybe', + 'USE_STACKCHECK': 'maybe'} diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 1a3db527d3d5b83..f944084aaa6d6aa 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -3765,13 +3765,17 @@ def test_startupinfo_copy(self): self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE) self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []}) + # CREATE_NEW_CONSOLE creates a "popup" window. + @support.requires_resource('gui') def test_creationflags(self): # creationflags argument CREATE_NEW_CONSOLE = 16 sys.stderr.write(" a DOS box should flash briefly ...\n") - subprocess.call(sys.executable + - ' -c "import time; time.sleep(0.25)"', - creationflags=CREATE_NEW_CONSOLE) + rc = subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + support.skip_on_low_desktop_heap_memory_subprocess(rc) + self.assertEqual(rc, 0) def test_invalid_args(self): # invalid arguments should raise ValueError diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e270cbb22e2d1a9..4be207e8cbf4e60 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -893,10 +893,39 @@ def test_extractall_hardlink_on_symlink(self): self._assert_on_file_content(hardlink_filepath, sha256_regtype) +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -970,7 +999,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): @@ -3911,10 +3940,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -4174,6 +4212,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 47450d3fd5976fa..70731d3222ced94 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -54,7 +54,11 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + result = tcl.eval(r'set a "<\ud83d\udcbb>"') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalException(self): tcl = self.interp @@ -289,7 +293,11 @@ def test_evalfile_surrogates_in_result(self): set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') + result = tcl.eval('set b') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalFileException(self): tcl = self.interp diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b2b5390af33b005..638140b96d4517f 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -332,7 +332,9 @@ def test_read_only_directory(self): with _inside_empty_temp_dir(): probe = os.path.join(tempfile.tempdir, 'probe') if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/deny', 'Everyone:(W)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/deny', '*S-1-1-0:(W)'] stdout = None if support.verbose > 1 else subprocess.DEVNULL subprocess.run(cmd, check=True, stdout=stdout) else: @@ -355,7 +357,9 @@ def test_read_only_directory(self): self.make_temp() finally: if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/grant:r', 'Everyone:(M)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/grant:r', '*S-1-1-0:(M)'] subprocess.run(cmd, check=True, stdout=stdout) else: os.chmod(tempfile.tempdir, oldmode) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index d459f497e333e64..b751f825bb97d59 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -485,6 +485,13 @@ def test_comprehension_in_annotation(self): ns = run_code("x: [y for y in range(10)]") self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + def test_class_annotation_dunder_classdict(self): + ns = run_code(""" + class C: + __classdict__: int + """) + self.assertEqual(ns["C"].__annotations__, {"__classdict__": int}) + def test_future_annotations(self): code = """ from __future__ import annotations diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 22ad9f6243eda91..849a2afd8ed7986 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,4 +1,5 @@ """ Tests for the internal type cache in CPython. """ +import collections.abc import dis import unittest import warnings @@ -114,6 +115,25 @@ class HolderSub(Holder): Holder.set_value() HolderSub.value + def test_abc_register_invalidates_subclass_versions(self): + class Parent: + pass + + class Child(Parent): + pass + + type_assign_version(Parent) + type_assign_version(Child) + parent_version = type_get_version(Parent) + child_version = type_get_version(Child) + if parent_version == 0 or child_version == 0: + self.skipTest("Could not assign valid type versions") + + collections.abc.Mapping.register(Parent) + + self.assertEqual(type_get_version(Parent), 0) + self.assertEqual(type_get_version(Child), 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index ad644bb31288098..042604ed7c1a423 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5848,6 +5848,27 @@ def foo(x: T): foo(42) + def test_genericalias_instance_isclass(self): + # test against user-defined generic classes + T = TypeVar('T') + + class Node(Generic[T]): + def __init__(self, label: T, + left: 'Node[T] | None' = None, + right: 'Node[T] | None' = None): + self.label = label + self.left = left + self.right = right + + self.assertTrue(inspect.isclass(Node)) + self.assertFalse(inspect.isclass(Node[int])) + self.assertFalse(inspect.isclass(Node[str])) + + # test against standard generic classes + self.assertFalse(inspect.isclass(set[int])) + self.assertFalse(inspect.isclass(list[bytes])) + self.assertFalse(inspect.isclass(dict[str, str])) + def test_implicit_any(self): T = TypeVar('T') diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 8ecb0df2f8e5ddc..ad25be3da8cb347 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -616,6 +616,34 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079: avoid quadratic canonical ordering. + payload = "a" + ("\u0300\u0327" * 32) + nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) + nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" + short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) + short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) + long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) + long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -1106,6 +1134,22 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) + def test_unicodedata_unload_reload(self): + # gh-149449: dropping unicodedata and running gc must not leave the + # cached _ucnhash_CAPI pointer dangling. + code = ( + "import gc, sys\n" + "assert '\\N{GRINNING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '', 'exec')\n" + "del sys.modules['unicodedata']\n" + "gc.collect()\n" + "assert '\\N{WINKING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '', 'exec')\n" + ) + script_helper.assert_python_ok("-c", code) + def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index f8643552011f4e8..0e82c723ec3eaa2 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1162,7 +1162,9 @@ def test_call_list_str(self): mock.foo.bar().baz('fish', cat='dog') expected = ( - "[call(1, 2), call.foo(a=3), call.foo.bar()," + "[call(1, 2),\n" + " call.foo(a=3),\n" + " call.foo.bar(),\n" " call.foo.bar().baz('fish', cat='dog')]" ) self.assertEqual(str(mock.mock_calls), expected) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 78461abcd69f337..9d2960664abfad5 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -301,9 +301,9 @@ def test_sysconfig(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - # Usually compare to sys.executable, but if we're running in our own - # venv then we really need to compare to our base executable - ('_base_executable', sys._base_executable), + # Usually compare to sys.prefix, but if we're running in our own + # venv then we really need to compare to our base prefix + ('base_prefix', sys.base_prefix), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' @@ -656,6 +656,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): @@ -896,10 +916,10 @@ def test_venvwlauncher(self): exename = exename.replace("python", "pythonw") envpyw = os.path.join(self.env_dir, self.bindir, exename) try: - subprocess.check_call([envpyw, "-c", "import sys; " - "assert sys._base_executable.endswith('%s')" % exename]) + subprocess.check_call([envpyw, "-c", "import fnmatch, sys; " + "assert fnmatch.fnmatch(sys._base_executable, '**/pythonw*.exe')"]) except subprocess.CalledProcessError: - self.fail("venvwlauncher.exe did not run %s" % exename) + self.fail("venvwlauncher.exe did not run pythonw.exe") @requireVenvCreate diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 51d627d24c5a8a3..82f14ca968f266b 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -340,6 +340,10 @@ def close(self): @requires_subprocess() class MacOSTest(unittest.TestCase): + def setUp(self): + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset("BROWSER") + def test_default(self): browser = webbrowser.get() self.assertIsInstance(browser, webbrowser.MacOS) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 8f3efe9fc90794b..27ea3c8c32fd8a5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1009,12 +1009,12 @@ def check(encoding, body=''): check("cp437", '\u221a') check("mac-roman", '\u02da') - def xml(encoding): - return "" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "%s" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -1025,13 +1025,14 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('&#%d;' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ 'big5', 'big5hkscs', @@ -1043,14 +1044,16 @@ def bxml(encoding): 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) @@ -1278,7 +1281,15 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - def test_processinginstruction(self): + def test_comment_serialization(self): + comm = ET.Comment(' & ham') + # comments are not escaped + self.assertEqual(ET.tostring(comm), b'') + self.assertEqual(ET.tostring(comm, method='html'), b'') + # no comments in text serialization + self.assertEqual(ET.tostring(comm, method='text'), b'') + + def test_processinginstruction_serialization(self): # Test ProcessingInstruction directly self.assertEqual(ET.tostring(ET.ProcessingInstruction('test', 'instruction')), @@ -1287,12 +1298,32 @@ def test_processinginstruction(self): b'') # Issue #2746 - + # processing instructions are not escaped self.assertEqual(ET.tostring(ET.PI('test', '')), b'?>') self.assertEqual(ET.tostring(ET.PI('test', '\xe3'), 'latin-1'), b"\n" b"\xe3?>") + pi = ET.PI('test', 'ham & eggs < spam') + self.assertEqual(ET.tostring(pi), b'') + self.assertEqual(ET.tostring(pi, method='html'), b'') + # no processing instructions in text serialization + self.assertEqual(ET.tostring(pi, method='text'), b'') + + def test_empty_attribute_serialization(self): + # empty attrs only work in html + elem = ET.Element('tag', attrib={'attr': None}) + self.assertRaises(TypeError, ET.tostring, elem) + self.assertEqual(ET.tostring(elem, method='html'), b'') + + @support.subTests('tag', ("script", "style", "xmp", "iframe", "noembed", "noframes")) + def test_html_cdata_elems_serialization(self, tag): + # content of raw text elements is not escaped in html + tag = tag.title() + elem = ET.Element(tag) + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + ('<%s>&ham' % (tag, tag)).encode()) def test_html_empty_elems_serialization(self): # issue 15970 @@ -1308,6 +1339,14 @@ def test_html_empty_elems_serialization(self): method='html') self.assertEqual(serialized, expected) + def test_html_plaintext_serialization(self): + # content of plaintext is not escaped in html + # no end tag for plaintext + elem = ET.Element('PlainText') + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + b'<spam>&ham') + def test_dump_attribute_order(self): # See BPO 34160 e = ET.Element('cirriculum', status='public', company='example') @@ -3654,6 +3693,32 @@ def test_basic(self): doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>") self.assertEqual(''.join(doc.itertext()), 'a&b&c&') + def test_comment(self): + e = ET.Element('root') + e.text = 'before' + comment = ET.Comment('comment') + self.assertEqual(comment.text, 'comment') + comment.tail = 'after' + e.append(comment) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, comment]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(comment.itertext()), '') + self.assertEqual(list(comment.iter()), [comment]) + + def test_processinginstruction(self): + e = ET.Element('root') + e.text = 'before' + pi = ET.PI('test', 'instruction') + self.assertEqual(pi.text, 'test instruction') + pi.tail = 'after' + e.append(pi) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, pi]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(pi.itertext()), '') + self.assertEqual(list(pi.iter()), [pi]) + def test_corners(self): # single root, no subelements a = ET.Element('a') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 2803c6d45c27bfa..ee0e24f6e86ae33 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -208,6 +208,17 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) + def test_dump_escape_methodname(self): + payload = 'foo</methodName><injected attr="evil"/><methodName>bar' + s = xmlrpclib.dumps((), methodname=payload) + self.assertIn( + '<methodName>foo&lt;/methodName&gt;&lt;injected attr="evil"/&gt;' + '&lt;methodName&gt;bar</methodName>', s + ) + self.assertNotIn('<injected attr="evil"/>', s) + load, m = xmlrpclib.loads(s) + self.assertEqual(m, payload) + def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index b52e78bc4fb7e05..c6e9dc375d9a676 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -1,19 +1,39 @@ import unittest from test.support import import_helper -import types xxlimited = import_helper.import_module('xxlimited') -xxlimited_35 = import_helper.import_module('xxlimited_35') - -class CommonTests: - module: types.ModuleType - - def test_xxo_new(self): - xxo = self.module.Xxo() - - def test_xxo_attributes(self): - xxo = self.module.Xxo() +# if import of xxlimited succeeded, the other ones should be importable. +import xxlimited_3_13 +import xxlimited_35 + +MODULES = { + (3, 15): xxlimited, + (3, 13): xxlimited_3_13, + (3, 5): xxlimited_35, +} + +def test_with_xxlimited_modules(since=None, until=None): + def _decorator(func): + def _wrapper(self, *args, **kwargs): + for version, module in MODULES.items(): + if since and version < since: + continue + if until and version >= until: + continue + with self.subTest(version=version): + func(self, module, *args, **kwargs) + return _wrapper + return _decorator + +class XXLimitedTests(unittest.TestCase): + @test_with_xxlimited_modules() + def test_xxo_new(self, module): + xxo = module.Xxo() + + @test_with_xxlimited_modules() + def test_xxo_attributes(self, module): + xxo = module.Xxo() with self.assertRaises(AttributeError): xxo.foo with self.assertRaises(AttributeError): @@ -26,40 +46,61 @@ def test_xxo_attributes(self): with self.assertRaises(AttributeError): xxo.foo - def test_foo(self): + @test_with_xxlimited_modules() + def test_foo(self, module): # the foo function adds 2 numbers - self.assertEqual(self.module.foo(1, 2), 3) + self.assertEqual(module.foo(1, 2), 3) - def test_str(self): - self.assertIsSubclass(self.module.Str, str) - self.assertIsNot(self.module.Str, str) + @test_with_xxlimited_modules() + def test_str(self, module): + self.assertIsSubclass(module.Str, str) + self.assertIsNot(module.Str, str) - custom_string = self.module.Str("abcd") + custom_string = module.Str("abcd") self.assertEqual(custom_string, "abcd") self.assertEqual(custom_string.upper(), "ABCD") - def test_new(self): - xxo = self.module.new() + @test_with_xxlimited_modules() + def test_new(self, module): + xxo = module.new() self.assertEqual(xxo.demo("abc"), "abc") - -class TestXXLimited(CommonTests, unittest.TestCase): - module = xxlimited - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() + @test_with_xxlimited_modules() + def test_xxo_demo(self, module): + xxo = module.Xxo() self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + self.assertEqual(xxo.__module__, module.__name__) + with self.assertRaises(TypeError): + module.Xxo('arg') + with self.assertRaises(TypeError): + module.Xxo(kwarg='arg') + + @test_with_xxlimited_modules(since=(3, 13)) + def test_xxo_demo_extra(self, module): + xxo = module.Xxo() + other = module.Xxo() self.assertEqual(xxo.demo(xxo), xxo) self.assertEqual(xxo.demo(other), other) - self.assertEqual(xxo.demo(0), None) - def test_error(self): - with self.assertRaises(self.module.Error): - raise self.module.Error - - def test_buffer(self): - xxo = self.module.Xxo() + @test_with_xxlimited_modules(since=(3, 15)) + def test_xxo_subclass(self, module): + class Sub(module.Xxo): + pass + sub = Sub() + sub.a = 123 + self.assertEqual(sub.a, 123) + with self.assertRaisesRegex(AttributeError, "cannot set 'reserved'"): + sub.reserved = 123 + + @test_with_xxlimited_modules(since=(3, 13)) + def test_error(self, module): + with self.assertRaises(module.Error): + raise module.Error + + @test_with_xxlimited_modules(since=(3, 13)) + def test_buffer(self, module): + xxo = module.Xxo() self.assertEqual(xxo.x_exports, 0) b1 = memoryview(xxo) self.assertEqual(xxo.x_exports, 1) @@ -69,21 +110,13 @@ def test_buffer(self): self.assertEqual(b1[0], 1) self.assertEqual(b2[0], 1) - -class TestXXLimited35(CommonTests, unittest.TestCase): - module = xxlimited_35 - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() - self.assertEqual(xxo.demo("abc"), "abc") - self.assertEqual(xxo.demo(0), None) - + @test_with_xxlimited_modules(until=(3, 5)) def test_roj(self): # the roj function always fails with self.assertRaises(SystemError): self.module.roj(0) + @test_with_xxlimited_modules(until=(3, 5)) def test_null(self): null1 = self.module.Null() null2 = self.module.Null() diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 0d407371f40a0f7..ffed328b171fda2 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1886,11 +1886,8 @@ def test_write_with_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_source_date_epoch.txt") - get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] - # Compare each element of the date_time tuple - # Allow for a 1-second difference - for z_time, g_time in zip(zip_info.date_time, get_time): - self.assertAlmostEqual(z_time, g_time, delta=1) + expected_utc = (2025, 1, 1, 7, 19, 58) + self.assertEqual(zip_info.date_time, expected_utc) def test_write_without_source_date_epoch(self): with os_helper.EnvironmentVarGuard() as env: @@ -1901,9 +1898,13 @@ def test_write_without_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=2) + self.assertTimestampAlmostEqual(time.localtime(), zip_info.date_time, tolerance=2) + + def assertTimestampAlmostEqual(self, time1, time2, tolerance): + import datetime + dt1 = datetime.datetime(*time1[:6]) + dt2 = datetime.datetime(*time2[:6]) + self.assertLessEqual((dt1 - dt2).total_seconds(), tolerance) def test_close(self): """Check that the zipfile is closed after the 'with' block.""" @@ -3639,29 +3640,23 @@ def test_read_with_unsuitable_metadata_encoding(self): def test_read_after_append(self): newname = '\u56db' # Han 'four' - expected_names = [name.encode('shift_jis').decode('cp437') - for name in self.file_names[:2]] + self.file_names[2:] - expected_names.append(newname) - expected_content = (*self.file_content, b"newcontent") + newname2 = 'fünf' # representable in cp437, but still stored as UTF-8 + expected_names = [*self.file_names, newname, newname2] + mojibake_expected_names = [name.encode('shift_jis').decode('cp437') + if i < 2 else name + for i, name in enumerate(expected_names)] + expected_content = (*self.file_content, b"newcontent", b"newcontent2") with zipfile.ZipFile(TESTFN, "a") as zipfp: zipfp.writestr(newname, "newcontent") - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + zipfp.writestr(newname2, "newcontent2") + self.assertEqual(sorted(zipfp.namelist()), sorted(mojibake_expected_names)) with zipfile.ZipFile(TESTFN, "r") as zipfp: - self._test_read(zipfp, expected_names, expected_content) + self._test_read(zipfp, mojibake_expected_names, expected_content) with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) - for i, (name, content) in enumerate(zip(expected_names, expected_content)): - info = zipfp.getinfo(name) - self.assertEqual(info.filename, name) - self.assertEqual(info.file_size, len(content)) - if i < 2: - with self.assertRaises(zipfile.BadZipFile): - zipfp.read(name) - else: - self.assertEqual(zipfp.read(name), content) + self._test_read(zipfp, expected_names, expected_content) def test_write_with_metadata_encoding(self): ZF = zipfile.ZipFile @@ -3670,6 +3665,20 @@ def test_write_with_metadata_encoding(self): "^metadata_encoding is only"): ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + def test_add_comment(self): + with zipfile.ZipFile(TESTFN, "r") as zipfp: + mojibake_expected_names = zipfp.namelist() + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.comment = b'comment' + self.assertEqual(zipfp.namelist(), mojibake_expected_names) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, mojibake_expected_names, self.file_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + def test_cli_with_metadata_encoding(self): errmsg = "Non-conforming encodings not supported with -c." args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce45562a60..f7eeffd575c1c76 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Lib/traceback.py b/Lib/traceback.py index 88529e1c259a29f..d16ab468db99624 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -115,10 +115,10 @@ def extract_tb(tb, limit=None): This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a FrameSummary object - containing attributes filename, lineno, name, and line - representing the information that is usually printed for a stack - trace. The line is a string with leading and trailing - whitespace stripped; if the source is not available it is None. + representing the information that is usually printed for a + stack trace. The line attribute is a string with + leading and trailing whitespace stripped; if the source is not + available the corresponding attribute is None. """ return StackSummary._extract_from_extended_frame_gen( _walk_tb_with_full_positions(tb), limit=limit) @@ -295,9 +295,8 @@ def extract_stack(f=None, limit=None): The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for - print_stack(). Each item in the list is a quadruple (filename, - line number, function name, text), and the entries are in order - from oldest to newest stack frame. + print_stack(). Each item in the list is a FrameSummary object, + and the entries are in order from oldest to newest stack frame. """ if f is None: f = sys._getframe().f_back @@ -325,7 +324,7 @@ class FrameSummary: active when the frame was captured. - :attr:`name` The name of the function or method that was executing when the frame was captured. - - :attr:`line` The text from the linecache module for the + - :attr:`line` The text from the linecache module for the line of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. @@ -1053,7 +1052,7 @@ def _wlen(s: str) -> int: def _display_width(line, offset=None): - """Calculate the extra amount of width space the given source + """Calculate the amount of width space the given source code segment might take if it were to be displayed on a fixed width output device. Supports wide unicode characters and emojis.""" @@ -1134,7 +1133,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): - # NB: we need to accept exc_traceback, exc_value, exc_traceback to + # NB: we need to accept exc_type, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. diff --git a/Lib/types.py b/Lib/types.py index b4f9a5c5140860b..6c069591ab26ef0 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -195,18 +195,19 @@ class Baz(list[str]): ... class DynamicClassAttribute: """Route attribute access on a class to __getattr__. - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name. (Enum used this between Python - versions 3.4 - 3.9 .) - - Subclass from this to use a different method of accessing virtual attributes - and still be treated properly by the inspect module. (Enum uses this since - Python 3.10 .) + This is a descriptor, used to define attributes that act differently + when accessed through an instance and through a class. Instance access + remains normal, but access to an attribute through a class will be + routed to the class's __getattr__ method; this is done by raising + AttributeError. + + This allows one to have properties active on an instance, and have + virtual attributes on the class with the same name. (Enum used this + between Python versions 3.4 - 3.9 .) + + Subclass from this to use a different method of accessing virtual + attributes and still be treated properly by the inspect module. (Enum + uses this since Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): diff --git a/Lib/typing.py b/Lib/typing.py index 5b1e223d59641e1..6011b62cd26944f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -604,12 +604,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -728,7 +728,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -758,7 +758,7 @@ class FastConnector(Connection): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -801,7 +801,7 @@ def open_helper(file: str, mode: MODE) -> str: def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -1356,32 +1356,35 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - # The type of parameterized generics. - # - # That is, for example, `type(List[int])` is `_GenericAlias`. - # - # Objects which are instances of this class include: - # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. - # * Note that native container types, e.g. `tuple`, `list`, use - # `types.GenericAlias` instead. - # * Parameterized classes: - # class C[T]: pass - # # C[int] is a _GenericAlias - # * `Callable` aliases, generic `Callable` aliases, and - # parameterized `Callable` aliases: - # T = TypeVar('T') - # # _CallableGenericAlias inherits from _GenericAlias. - # A = Callable[[], None] # _CallableGenericAlias - # B = Callable[[T], None] # _CallableGenericAlias - # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: - # # All _GenericAlias - # Final[int] - # ClassVar[float] - # TypeForm[bytes] - # TypeGuard[bool] - # TypeIs[range] - + """The type of parameterized generics. + + That is, for example, `type(List[int])` is `_GenericAlias`. + + Objects which are instances of this class include: + * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + * Note that native container types, e.g. `tuple`, `list`, use + `types.GenericAlias` instead. + * Parameterized classes: + class C[T]: pass + # C[int] is a _GenericAlias + * `Callable` aliases, generic `Callable` aliases, and + parameterized `Callable` aliases: + T = TypeVar('T') + # _CallableGenericAlias inherits from _GenericAlias. + A = Callable[[], None] # _CallableGenericAlias + B = Callable[[T], None] # _CallableGenericAlias + C = B[int] # _CallableGenericAlias + * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: + # All _GenericAlias + Final[int] + ClassVar[float] + TypeForm[bytearray] + TypeGuard[bool] + TypeIs[range] + + Note that instances of this class are not classes (e.g by `inspect.isclass`), + even though they behave like them. + """ def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): @@ -1413,20 +1416,21 @@ def __ror__(self, left): @_tp_cache def __getitem__(self, args): - # Parameterizes an already-parameterized object. - # - # For example, we arrive here doing something like: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # T3 = TypeVar('T3') - # class A(Generic[T1]): pass - # B = A[T2] # B is a _GenericAlias - # C = B[T3] # Invokes _GenericAlias.__getitem__ - # - # We also arrive here when parameterizing a generic `Callable` alias: - # T = TypeVar('T') - # C = Callable[[T], None] - # C[int] # Invokes _GenericAlias.__getitem__ + """Parameterizes an already-parameterized object. + + For example, we arrive here doing something like: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + class A(Generic[T1]): pass + B = A[T2] # B is a _GenericAlias + C = B[T3] # Invokes _GenericAlias.__getitem__ + + We also arrive here when parameterizing a generic `Callable` alias: + T = TypeVar('T') + C = Callable[[T], None] + C[int] # Invokes _GenericAlias.__getitem__ + """ if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. @@ -1443,20 +1447,20 @@ def __getitem__(self, args): return r def _determine_new_args(self, args): - # Determines new __args__ for __getitem__. - # - # For example, suppose we had: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # class A(Generic[T1, T2]): pass - # T3 = TypeVar('T3') - # B = A[int, T3] - # C = B[str] - # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. - # Unfortunately, this is harder than it looks, because if `T3` is - # anything more exotic than a plain `TypeVar`, we need to consider - # edge cases. - + """Determines new __args__ for __getitem__. + + For example, suppose we had: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + class A(Generic[T1, T2]): pass + T3 = TypeVar('T3') + B = A[int, T3] + C = B[str] + `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + Unfortunately, this is harder than it looks, because if `T3` is + anything more exotic than a plain `TypeVar`, we need to consider + edge cases. + """ params = self.__parameters__ # In the example above, this would be {T3: str} for param in params: @@ -1809,7 +1813,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -2320,7 +2324,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2390,8 +2394,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=None): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as annotationlib.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2603,7 +2607,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2700,10 +2704,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2711,10 +2715,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2722,8 +2727,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2759,7 +2762,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2824,7 +2827,7 @@ class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2912,7 +2915,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2925,7 +2928,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2936,7 +2939,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2947,7 +2950,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2958,7 +2961,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2969,7 +2972,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2980,7 +2983,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2991,7 +2994,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -3108,7 +3111,7 @@ def annotate(format): def NamedTuple(typename, fields, /): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -3120,8 +3123,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + annotationlib.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3174,7 +3177,7 @@ def __new__(cls, name, bases, ns, total=True, closed=None, This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3328,14 +3331,22 @@ def TypedDict(typename, fields, /, *, total=True, closed=None, >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling annotationlib.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3344,16 +3355,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3387,7 +3390,7 @@ class Point3D(Point2D): by default, and it may not be used with the closed argument at the same time. - See PEP 728 for more information about closed and extra_items. + See PEPs 589, 655, 705, and 728 for more information. """ ns = {'__annotations__': dict(fields)} module = _caller() @@ -3417,7 +3420,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3439,7 +3442,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3459,7 +3462,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3546,8 +3549,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3637,7 +3640,7 @@ def __exit__(self, type, value, traceback, /) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3651,7 +3654,7 @@ def __enter__(self) -> BinaryIO: class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3718,7 +3721,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 1cee67fa5d70946..2f6f03c7a11ae64 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -3121,6 +3121,10 @@ def _mock_call(self, *args, **kwargs): return ret_value + def _increment_mock_call(self, /, *args, **kwargs): + with self._mock_calls_events_lock: + super()._increment_mock_call(*args, **kwargs) + def wait_until_called(self, *, timeout=_timeout_unset): """Wait until the mock object is called. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index d64f678d235b6f9..82b95adbdc283ef 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -277,7 +277,7 @@ def _hostinfo(self): return hostname, port -_UNSPECIFIED = ['not specified'] +_UNSPECIFIED = sentinel("_UNSPECIFIED", repr="<not specified>") _MISSING_AS_NONE_DEFAULT = False class _ResultBase: diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d41f80..241a8650bda33aa 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,7 +17,7 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -73,4 +73,4 @@ fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Lib/xml/__init__.py b/Lib/xml/__init__.py index 002d6d3e0e8267c..ecfce1c6ae52cf2 100644 --- a/Lib/xml/__init__.py +++ b/Lib/xml/__init__.py @@ -18,4 +18,4 @@ from .utils import * -__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name"] +__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name", "is_valid_text"] diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 85766e02b531ce2..53727d7940b3f2a 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -8,8 +8,8 @@ 2. Element represents a single node in this tree. Interactions with the whole document (reading and writing to/from files) are - usually done on the ElementTree level. Interactions with a single XML element - and its sub-elements are done on the Element level. + usually done on the ElementTree level. Interactions with a single XML + element and its sub-elements are done on the Element level. Element is a flexible container object designed to store hierarchical data structures in memory. It can be described as a cross between a list and a @@ -277,7 +277,8 @@ def find(self, path, namespaces=None): """Find first matching element by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -289,7 +290,8 @@ def findtext(self, path, default=None, namespaces=None): *path* is a string having either an element tag or an XPath, *default* is the value to return if the element was not found, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return text content of first matching element, or default value if none was found. Note that if an element is found having no text @@ -302,7 +304,8 @@ def findall(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Returns list containing all matching elements in document order. @@ -313,7 +316,8 @@ def iterfind(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return an iterable yielding all matching elements in document order. @@ -553,8 +557,8 @@ def _setroot(self, element): def parse(self, source, parser=None): """Load external XML document into element tree. - *source* is a file name or file object, *parser* is an optional parser - instance that defaults to XMLParser. + *source* is a file name or file object, *parser* is an optional + parser instance that defaults to XMLParser. ParseError is raised if the parser fails to parse the document. @@ -587,7 +591,8 @@ def parse(self, source, parser=None): def iter(self, tag=None): """Create and return tree iterator for the root element. - The iterator loops over all elements in this tree, in document order. + The iterator loops over all elements in this tree, in document + order. *tag* is a string with the tag name to iterate over (default is to return all elements). @@ -602,7 +607,8 @@ def find(self, path, namespaces=None): Same as getroot().find(path), which is Element.find() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -624,7 +630,8 @@ def findtext(self, path, default=None, namespaces=None): Same as getroot().findtext(path), which is Element.findtext() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -646,7 +653,8 @@ def findall(self, path, namespaces=None): Same as getroot().findall(path), which is Element.findall(). *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return list containing all matching elements in document order. @@ -693,24 +701,26 @@ def write(self, file_or_filename, """Write element tree to a file as XML. Arguments: - *file_or_filename* -- file name or a file object opened for writing + *file_or_filename* -- file name or a file object opened for + writing *encoding* -- the output encoding (default: US-ASCII) - *xml_declaration* -- bool indicating if an XML declaration should be - added to the output. If None, an XML declaration - is added if encoding IS NOT either of: - US-ASCII, UTF-8, or Unicode + *xml_declaration* -- bool indicating if an XML declaration should + be added to the output. If None, an XML + declaration is added if encoding IS NOT + either of: US-ASCII, UTF-8, or Unicode - *default_namespace* -- sets the default XML namespace (for "xmlns") + *default_namespace* -- sets the default XML namespace (for + "xmlns") *method* -- either "xml" (default), "html, "text", or "c14n" *short_empty_elements* -- controls the formatting of elements - that contain no content. If True (default) - they are emitted as a single self-closed - tag, otherwise they are emitted as a pair - of start/end tags + that contain no content. If True + (default) they are emitted as a single + self-closed tag, otherwise they are + emitted as a pair of start/end tags """ if self._root is None: @@ -907,17 +917,20 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) +_CDATA_CONTENT_ELEMENTS = {"script", "style", "xmp", "iframe", "noembed", + "noframes", "plaintext"} + HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", "img", "input", "isindex", "link", "meta", "param", "source", - "track", "wbr"} + "track", "wbr", "plaintext"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag text = elem.text if tag is Comment: - write("<!--%s-->" % _escape_cdata(text)) + write("<!--%s-->" % text) elif tag is ProcessingInstruction: - write("<?%s?>" % _escape_cdata(text)) + write("<?%s?>" % text) else: tag = qnames[tag] if tag is None: @@ -941,16 +954,19 @@ def _serialize_html(write, elem, qnames, namespaces, **kwargs): for k, v in items: if isinstance(k, QName): k = k.text - if isinstance(v, QName): - v = qnames[v.text] + k = qnames[k] + if v is None: + write(" %s" % k) # empty attr else: - v = _escape_attrib_html(v) - # FIXME: handle boolean attributes - write(" %s=\"%s\"" % (qnames[k], v)) + if isinstance(v, QName): + v = qnames[v.text] + else: + v = _escape_attrib_html(v) + write(" %s=\"%s\"" % (k, v)) write(">") ltag = tag.lower() if text: - if ltag == "script" or ltag == "style": + if ltag in _CDATA_CONTENT_ELEMENTS: write(text) else: write(_escape_cdata(text)) @@ -1083,9 +1099,9 @@ def tostring(element, encoding=None, method=None, *, is returned. Otherwise a bytestring is returned. *element* is an Element instance, *encoding* is an optional output - encoding defaulting to US-ASCII, *method* is an optional output which can - be one of "xml" (default), "html", "text" or "c14n", *default_namespace* - sets the default XML namespace (for "xmlns"). + encoding defaulting to US-ASCII, *method* is an optional output which + can be one of "xml" (default), "html", "text" or "c14n", + *default_namespace* sets the default XML namespace (for "xmlns"). Returns an (optionally) encoded string containing the XML data. @@ -1225,7 +1241,8 @@ def iterparse(source, events=None, parser=None): "end" events are reported. *source* is a filename or file object containing XML data, *events* is - a list of events to report back, *parser* is an optional parser instance. + a list of events to report back, *parser* is an optional parser + instance. Returns an iterator providing (event, elem) pairs. @@ -1761,10 +1778,11 @@ def flush(self): def canonicalize(xml_data=None, *, out=None, from_file=None, **options): """Convert XML to its C14N 2.0 serialised form. - If *out* is provided, it must be a file or file-like object that receives - the serialised canonical XML output (text, not bytes) through its ``.write()`` - method. To write to a file, open it in text mode with encoding "utf-8". - If *out* is not provided, this function returns the output as text string. + If *out* is provided, it must be a file or file-like object that + receives the serialised canonical XML output (text, not bytes) through + its ``.write()`` method. To write to a file, open it in text mode with + encoding "utf-8". If *out* is not provided, this function returns the + output as text string. Either *xml_data* (an XML string) or *from_file* (a file path or file-like object) must be provided as input. @@ -1798,19 +1816,22 @@ class C14NWriterTarget: Serialises parse events to XML C14N 2.0. The *write* function is used for writing out the resulting data stream - as text (not bytes). To write to a file, open it in text mode with encoding - "utf-8" and pass its ``.write`` method. + as text (not bytes). To write to a file, open it in text mode with + encoding "utf-8" and pass its ``.write`` method. Configuration options: - *with_comments*: set to true to include comments - - *strip_text*: set to true to strip whitespace before and after text content - - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" + - *strip_text*: set to true to strip whitespace before and after text + content + - *rewrite_prefixes*: set to true to replace namespace prefixes by + "n{number}" - *qname_aware_tags*: a set of qname aware tag names in which prefixes should be replaced in text content - - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes - should be replaced in text content - - *exclude_attrs*: a set of attribute names that should not be serialised + - *qname_aware_attrs*: a set of qname aware attribute names in which + prefixes should be replaced in text content + - *exclude_attrs*: a set of attribute names that should not be + serialised - *exclude_tags*: a set of tag names that should not be serialised """ def __init__(self, write, *, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index f441376d09c4aa2..84e4e4d11a7319e 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -965,7 +965,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, data = ( xmlheader, "<methodCall>\n" - "<methodName>", methodname, "</methodName>\n", + "<methodName>", escape(methodname), "</methodName>\n", data, "</methodCall>\n" ) diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 86c3bc36b695c79..71e4dd4f6f625ce 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -566,8 +566,12 @@ def FileHeader(self, zip64=None): return header + filename + extra def _encodeFilenameFlags(self): + if self.flag_bits & _MASK_UTF_FILENAME: + encoding = 'ascii' + else: + encoding = 'cp437' try: - return self.filename.encode('ascii'), self.flag_bits + return self.filename.encode(encoding), self.flag_bits & ~_MASK_UTF_FILENAME except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME @@ -620,11 +624,12 @@ def _decodeExtra(self, filename_crc): def from_file(cls, filename, arcname=None, *, strict_timestamps=True): """Construct an appropriate ZipInfo for a file on the filesystem. - filename should be the path to a file or directory on the filesystem. + filename should be the path to a file or directory on the + filesystem. - arcname is the name which it will have within the archive (by default, - this will be the same as filename, but without a drive letter and with - leading path separators removed). + arcname is the name which it will have within the archive (by + default, this will be the same as filename, but without a drive + letter and with leading path separators removed). """ if isinstance(filename, os.PathLike): filename = os.fspath(filename) @@ -663,9 +668,12 @@ def _for_archive(self, archive): Return self. """ # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp - epoch = os.environ.get('SOURCE_DATE_EPOCH') - get_time = int(epoch) if epoch else time.time() - self.date_time = time.localtime(get_time)[:6] + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + + if source_date_epoch: + self.date_time = time.gmtime(int(source_date_epoch))[:6] + else: + self.date_time = time.localtime(time.time())[:6] self.compress_type = archive.compression self.compress_level = archive.compresslevel @@ -1392,19 +1400,19 @@ class ZipFile: mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or - ZIP_ZSTANDARD (requires compression.zstd). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - compresslevel: None (default for the given compression type) or an integer - specifying the level to pass to the compressor. - When using ZIP_STORED or ZIP_LZMA this keyword has no effect. - When using ZIP_DEFLATED integers 0 through 9 are accepted. - When using ZIP_BZIP2 integers 1 through 9 are accepted. - When using ZIP_ZSTANDARD integers -7 though 22 are common, - see the CompressionParameter enum in compression.zstd for - details. + ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or + ZIP_ZSTANDARD (requires compression.zstd). + allowZip64: if True ZipFile will create files with ZIP64 extensions + when needed, otherwise it will raise an exception when this + would be necessary. + compresslevel: None (default for the given compression type) or + an integer specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. + When using ZIP_ZSTANDARD integers -7 though 22 are common, + see the CompressionParameter enum in compression.zstd for + details. """ @@ -1414,8 +1422,8 @@ class ZipFile: def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" + """Open the ZIP file with mode read 'r', write 'w', exclusive create + 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") @@ -1693,10 +1701,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): pwd is the password to decrypt files (only used for reading). - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. + When writing, if the file size is not known in advance but may + exceed 2 GiB, pass force_zip64 to use the ZIP64 format, which can + handle large files. If the size is known in advance, it is best to + pass a ZipInfo instance for name, with zinfo.file_size set. """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') @@ -1808,7 +1816,7 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.compress_size = 0 zinfo.CRC = 0 - zinfo.flag_bits = 0x00 + zinfo.flag_bits = _MASK_UTF_FILENAME if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index c5f92a99a1e0766..e0e7076d681887b 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.50.4", - url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", - checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", + name="SQLite 3.53.1", + url="https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", + checksum="83e6b2020a034e9a7ad4a72feea59e1ad52f162e09cbd26735a3ffb98359fc4f", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' diff --git a/Makefile.pre.in b/Makefile.pre.in index dce0139d8d6e35a..9c358bc6fbc6818 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -835,7 +835,7 @@ check-clean-src: echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ echo "Try to run:" ; \ - echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ + echo " (cd \"$(srcdir)\" && make distclean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -3211,7 +3211,7 @@ Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_inner.c # emcc has a path that ends with emsdk/upstream/emscripten/emcc, we're looking for emsdk/upstream/bin/clang. - $$(dirname $$(dirname $(CC)))/bin/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib + $$(em-config LLVM_ROOT)/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule diff --git a/Misc/ACKS b/Misc/ACKS index 234d0d2d0a2a164..14f0db7549534be 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -144,6 +144,7 @@ Bas van Beek Ian Beer Stefan Behnel Reimer Behrends +Tomi Belan Maxime Bélanger Ben Bell Thomas Bellman diff --git a/Misc/Brewfile b/Misc/Brewfile new file mode 100644 index 000000000000000..c799f099957f757 --- /dev/null +++ b/Misc/Brewfile @@ -0,0 +1,14 @@ +brew "gdbm" +brew "mpdecimal" +brew "openssl@3.5" +brew "pkg-config" +brew "tcl-tk@9" +brew "xz" +brew "zstd" + +brew "bzip2" if OS.linux? +brew "libedit" if OS.linux? +brew "libffi" if OS.linux? +brew "ncurses" if OS.linux? +brew "unzip" if OS.linux? +brew "zlib-ng-compat" if OS.linux? diff --git a/Misc/NEWS.d/3.15.0a8.rst b/Misc/NEWS.d/3.15.0a8.rst index ff7930aeb292d67..28e9ca85166620f 100644 --- a/Misc/NEWS.d/3.15.0a8.rst +++ b/Misc/NEWS.d/3.15.0a8.rst @@ -180,16 +180,6 @@ dealing with contradictions in ``make_bottom``. .. -.. date: 2026-03-24-13-06-52 -.. gh-issue: 146369 -.. nonce: 6wDI6S -.. section: Core and Builtins - -Ensure ``-X lazy_imports=none`` and ``PYTHON_LAZY_IMPORTS=none`` override -:attr:`~module.__lazy_modules__`. Patch by Hugo van Kemenade. - -.. - .. date: 2026-03-22-19-30-00 .. gh-issue: 146308 .. nonce: AxnRVA @@ -350,7 +340,7 @@ Fix :func:`repr` for lists and tuples containing ``NULL``\ s. .. nonce: aB3xKm .. section: Core and Builtins -Fixed ``sys.lazy_modules`` to include lazy modules without submodules. Patch +Fixed :py:attr:`sys.lazy_modules` to include lazy modules without submodules. Patch by Bartosz Sławecki. .. diff --git a/Misc/NEWS.d/3.15.0b2.rst b/Misc/NEWS.d/3.15.0b2.rst new file mode 100644 index 000000000000000..afbdd4e072ed7d4 --- /dev/null +++ b/Misc/NEWS.d/3.15.0b2.rst @@ -0,0 +1,795 @@ +.. date: 2026-05-11-21-15-07 +.. gh-issue: 149698 +.. nonce: OudOcW +.. release date: 2026-06-02 +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.1 +for the fix for :cve:`2026-45186`. + +.. + +.. date: 2026-05-10-18-05-32 +.. gh-issue: 87451 +.. nonce: XkKB6M +.. section: Security + +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source +:class:`ftplib.FTP` instance to ``True``. Thanks to Qi Deng at Aurascape AI +for the report. + +.. + +.. date: 2026-05-08-02-18-54 +.. gh-issue: 149474 +.. nonce: ujQ-mu +.. section: Security + +Fix the binary writer in :mod:`profiling.sampling` not firing the audit +(:pep:`578`) when creating the output file. The writer and the reader now +accept any path-like object. Patch by Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-05-03-21-00-00 +.. gh-issue: 149486 +.. nonce: tarflt +.. section: Security + +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. + +.. + +.. date: 2026-04-27-16-36-11 +.. gh-issue: 149079 +.. nonce: vKl-LM +.. section: Security + +Fix a potential denial of service in :func:`unicodedata.normalize`. The +canonical ordering step of Unicode normalization used a quadratic-time +insertion sort for reordering combining characters, which could be exploited +with crafted input containing many combining characters in non-canonical +order. Replaced with a linear-time counting sort for long runs. + +.. + +.. date: 2026-04-26-19-30-45 +.. gh-issue: 149018 +.. nonce: a9SqWb +.. section: Security + +Improved protection against XML hash-flooding attacks in +:mod:`xml.parsers.expat` and :mod:`xml.etree.ElementTree` when Python is +compiled with libExpat 2.8.0 or later. + +.. + +.. date: 2026-05-25-16-00-22 +.. gh-issue: 150374 +.. nonce: Emu6d8 +.. section: Core and Builtins + +Fix double release of the import lock on lazy import reification errors. + +.. + +.. date: 2026-05-24-14-45-00 +.. gh-issue: 149156 +.. nonce: NP73rB +.. section: Core and Builtins + +Fix an intermittent crash after :func:`os.fork` when perf trampoline +profiling is enabled and the child returns through trampoline frames +inherited from the parent process. + +.. + +.. date: 2026-05-23-22-08-01 +.. gh-issue: 149449 +.. nonce: 2lhQFF +.. section: Core and Builtins + +Fix a use-after-free crash when the :mod:`unicodedata` module was removed +from :data:`sys.modules` and garbage-collected between calls that decode +``\N{...}`` escapes or use the ``namereplace`` codec error handler. + +.. + +.. date: 2026-05-22-17-09-28 +.. gh-issue: 150107 +.. nonce: GD72-D +.. section: Core and Builtins + +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, even if +*offset* is ``0`` (default value). + +.. + +.. date: 2026-05-20-13-06-17 +.. gh-issue: 150146 +.. nonce: i5m_SL +.. section: Core and Builtins + +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. + +.. + +.. date: 2026-05-18-18-36-28 +.. gh-issue: 148587 +.. nonce: -RD3z5 +.. section: Core and Builtins + +:py:attr:`sys.lazy_modules` is now a set instead of a dict as initially spelled out +in PEP 810. + +.. + +.. date: 2026-05-18-16-54-54 +.. gh-issue: 150042 +.. nonce: LSr5W8 +.. section: Core and Builtins + +Fix refleak in queue.SimpleQueue.put if memory allocation fails. + +.. + +.. date: 2026-05-18-13-47-17 +.. gh-issue: 149590 +.. nonce: IPBeQx +.. section: Core and Builtins + +Fix crash when faulthandler is imported more than once. + +.. + +.. date: 2026-05-16-11-03-54 +.. gh-issue: 149816 +.. nonce: X_gqMT +.. section: Core and Builtins + +Fix a race condition in ``_PyBytes_FromList`` in free-threading mode. + +.. + +.. date: 2026-05-15-11-31-57 +.. gh-issue: 149816 +.. nonce: ugN2rx +.. section: Core and Builtins + +Fix a race condition in :class:`memoryview` with free-threading. + +.. + +.. date: 2026-05-14-19-41-03 +.. gh-issue: 149807 +.. nonce: IwGaCo +.. section: Core and Builtins + +Fix ``hash(frozendict)``: compute the hash of each ``(key, value)`` pair +correctly. Patch by Victor Stinner. + +.. + +.. date: 2026-05-13-06-54-41 +.. gh-issue: 149738 +.. nonce: 4BLFoH +.. section: Core and Builtins + +:mod:`sqlite3`: Disallow removing ``row_factory`` and ``text_factory`` +attributes of a connection to prevent a crash on a query. + +.. + +.. date: 2026-05-12-16-47-23 +.. gh-issue: 139808 +.. nonce: iIs7_E +.. section: Core and Builtins + +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). + +.. + +.. date: 2026-05-11-14-48-56 +.. gh-issue: 149676 +.. nonce: 6aTrw1 +.. section: Core and Builtins + +Fix ``frozendict | frozendict`` hash. + +.. + +.. date: 2026-05-10-16-43-50 +.. gh-issue: 148829 +.. nonce: gscS14 +.. section: Core and Builtins + +:class:`sentinel` objects now support a ``repr=`` argument and their +:attr:`~sentinel.__module__` attribute is writable. + +.. + +.. date: 2026-05-10-07-42-36 +.. gh-issue: 149642 +.. nonce: 6ZksML +.. section: Core and Builtins + +Allow imports inside ``exec()`` calls within functions under +``PYTHON_LAZY_IMPORTS=all``. + +.. + +.. date: 2026-05-09-15-22-32 +.. gh-issue: 144957 +.. nonce: u1F2aQ +.. section: Core and Builtins + +Fix lazy ``from`` imports of module attributes provided by module-level +``__getattr__``. + +.. + +.. date: 2026-05-07-03-18-59 +.. gh-issue: 149459 +.. nonce: 5fhAqP +.. section: Core and Builtins + +Fix a crash in the JIT optimizer when a specialized ``LOAD_SPECIAL`` guard +deoptimized after inserting the synthetic ``NULL`` stack entry. + +.. + +.. date: 2026-04-15-15-48-04 +.. gh-issue: 148450 +.. nonce: 2MEVqH +.. section: Core and Builtins + +Fix ``abc.register()`` so it invalidates type version tags for registered +classes. + +.. + +.. date: 2026-05-31-17-47-30 +.. gh-issue: 150685 +.. nonce: EBB2mU +.. section: Library + +Update bundled pip to 26.1.2 + +.. + +.. date: 2026-05-27-11-18-36 +.. gh-issue: 150228 +.. nonce: pNPiO- +.. section: Library + +The new :class:`site.StartupState` class lets callers batch-process +:pep:`829` startup configuration files across multiple site directories +before any startup code runs, with public +:meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, +:meth:`~site.StartupState.addsitepackages`, and +:meth:`~site.StartupState.process` methods. The signature of +:func:`site.addsitedir` is unchanged from Python 3.14. The +:data:`!defer_processing_start_files` argument and the +``process_startup_files()`` function added earlier in the 3.15 cycle have +been removed; use :class:`!site.StartupState` instead. + +.. + +.. date: 2026-05-25-17-00-00 +.. gh-issue: 150406 +.. nonce: jF3g63 +.. section: Library + +Fix a possible crash occurring during :mod:`socket` module initialization +when the system is out of memory on platforms without a reentrant +``gethostbyname``. + +.. + +.. date: 2026-05-25-07-22-05 +.. gh-issue: 150372 +.. nonce: 9hLqhe +.. section: Library + +:mod:`readline`: Fix a potential crash during tab completion caused by an +out-of-memory error during module initialization. + +.. + +.. date: 2026-05-21-20-47-45 +.. gh-issue: 150157 +.. nonce: ZvmO-bQZ +.. section: Library + +Fix a crash in free-threaded builds that occurs when pickling by name +objects without a ``__module__`` attribute while :data:`sys.modules` is +concurrently being modified. + +.. + +.. date: 2026-05-21-11-25-58 +.. gh-issue: 150175 +.. nonce: 8H4Caz +.. section: Library + +Fix race condition in :class:`unittest.mock.ThreadingMock` where concurrent +calls could lose increments to ``call_count`` and other attributes due to a +missing lock in ``_increment_mock_call``. + +.. + +.. date: 2026-05-19-19-00-49 +.. gh-issue: 84353 +.. nonce: ZU5zaQ +.. section: Library + +Preserve non-UTF-8 encoded filenames when appending to a +:class:`zipfile.ZipFile`. Previously, non-ASCII names stored in a legacy +encoding (without the UTF-8 flag bit set) could be corrupted when the +central directory was rewritten: they were decoded as cp437 and then +re-stored as UTF-8. + +.. + +.. date: 2026-05-18-17-17-20 +.. gh-issue: 149189 +.. nonce: a8IooK +.. section: Library + +Revert the changes to :mod:`pprint` defaults. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-05-18-07-44-46 +.. gh-issue: 149995 +.. nonce: vvtFHn +.. section: Library + +Update various docstrings in :mod:`typing`. + +.. + +.. date: 2026-05-17-22-37-02 +.. gh-issue: 88726 +.. nonce: BAoL6j +.. section: Library + +The :mod:`email` package now uses standard MIME charset names "gb2312" and +"big5" instead of non-standard names "eucgb2312_cn" and "big5_tw". + +.. + +.. date: 2026-05-17-02-25-56 +.. gh-issue: 149571 +.. nonce: LNyuWJ +.. section: Library + +Fix the C implementation of :meth:`xml.etree.ElementTree.Element.itertext`: +it no longer emits text for comments and processing instructions. + +.. + +.. date: 2026-05-16-21-08-33 +.. gh-issue: 149921 +.. nonce: I1yNML +.. section: Library + +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. + +.. + +.. date: 2026-05-15-18-44-20 +.. gh-issue: 142349 +.. nonce: fHK3v1 +.. section: Library + +Add :keyword:`lazy` to the list of support topic by :func:`help`. + +.. + +.. date: 2026-05-15-16-28-00 +.. gh-issue: 149819 +.. nonce: fixpth +.. section: Library + +Fix regression in :func:`site.addsitedir` where ``.pth`` files were no +longer processed in Python subprocesses. This happened because +:func:`site.main` seeded ``known_paths`` with entries inherited from the +parent process, causing ``addsitedir`` to skip ``.pth`` processing. + +.. + +.. date: 2026-05-14-15-55-28 +.. gh-issue: 149816 +.. nonce: ZaXQ0q +.. section: Library + +Fix a race condition in ``_random.Random.__init__`` method in free-threading +mode. + +.. + +.. date: 2026-05-13-23-18-39 +.. gh-issue: 149801 +.. nonce: S_FfGr +.. section: Library + +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. + +.. + +.. date: 2026-05-12-13-03-45 +.. gh-issue: 149718 +.. nonce: SaM1NJ +.. section: Library + +Coalesce consecutive identical stack frames in Tachyon, so aggregating +collectors (pstats, collapsed, flamegraph, gecko) receive one collect. +Improves sample rate 3x, error rate and missed rate drop by 70%. Patch by +Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-05-12-06-24-54 +.. gh-issue: 149701 +.. nonce: 8v9RTm +.. section: Library + +Fix bad return code from Lib/venv/bin/activate if hashing is disabled + +.. + +.. date: 2026-05-10-23-51-23 +.. gh-issue: 149504 +.. nonce: pDSCbn +.. section: Library + +Fix :func:`site.addsitedir` to allow re-entrant calls from within startup +files. Previously, a ``.pth`` file containing an ``import`` line that +called :func:`site.addsitedir` (or a ``.start`` entry point doing the same) +could crash with ``RuntimeError: dictionary changed size during iteration`` +during site initialization, breaking tools such as ``uv run --with``. + +.. + +.. date: 2026-05-10-19-26-50 +.. gh-issue: 149584 +.. nonce: x7Qm9A +.. section: Library + +Fix excessive overhead in the Tachyon profiler when inspecting a remote +process by avoiding repeated remote page-cache scans, batching predicted +remote reads, and reusing cached profiler result objects. Patch by Pablo +Galindo and Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-05-10-07-21-51 +.. gh-issue: 139489 +.. nonce: rS7LTA +.. section: Library + +Add :func:`xml.is_valid_text` to ``xml.__all__``. + +.. + +.. date: 2026-05-09-21-02-08 +.. gh-issue: 149614 +.. nonce: U4snj3 +.. section: Library + +Fix a regression that broke the ability to deepcopy +:class:`argparse.ArgumentParser` instances. + +.. + +.. date: 2026-05-08-15-08-35 +.. gh-issue: 112821 +.. nonce: t9T1YD +.. section: Library + +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. + +.. + +.. date: 2026-05-08-09-11-48 +.. gh-issue: 149534 +.. nonce: Tw7eeY +.. section: Library + +Fix merging of :class:`collections.defaultdict` and :class:`frozendict`. + +.. + +.. date: 2026-05-07-21-58-17 +.. gh-issue: 149388 +.. nonce: DDBPeA +.. section: Library + +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. + +.. + +.. date: 2026-05-07-14-18-47 +.. gh-issue: 149489 +.. nonce: bX9iHe +.. section: Library + +Fix :mod:`~xml.etree.ElementTree` serialization to HTML. The content of +comments, processing instructions and elements "xmp", "iframe", "noembed", +"noframes", and "plaintext" is no longer escaped. The "plaintext" element no +longer have the closing tag. Add support of empty attributes (with value +``None``). + +.. + +.. date: 2026-04-29-08-10-17 +.. gh-issue: 149056 +.. nonce: jnaD4W +.. section: Library + +Fix :func:`json.load` not forwarding the *array_hook* argument to +:func:`json.loads`. Patch by Thomas Kowalski. + +.. + +.. date: 2026-04-27-11-12-00 +.. gh-issue: 149046 +.. nonce: 74shDd +.. section: Library + +:mod:`io`: Fix :class:`io.StringIO` serialization: no longer call +``str(obj)`` on :class:`str` subclasses. Patch by Thomas Kowalski. + +.. + +.. date: 2026-04-23-12-50-15 +.. gh-issue: 148441 +.. nonce: zvpCkR +.. section: Library + +:mod:`xml.parsers.expat`: prevent a crash in +:meth:`~xml.parsers.expat.xmlparser.CharacterDataHandler` when the character +data size exceeds the parser's :attr:`buffer size +<xml.parsers.expat.xmlparser.buffer_size>`. + +.. + +.. date: 2026-03-26-09-30-00 +.. gh-issue: 146452 +.. nonce: Y2N6qZ8J +.. section: Library + +Fix segfault in :mod:`pickle` when pickling a dictionary concurrently +mutated by another thread in the free-threaded build. + +.. + +.. date: 2025-08-30-07-44-30 +.. gh-issue: 86533 +.. nonce: pathlib +.. section: Library + +The :func:`os.makedirs` function and :meth:`pathlib.Path.mkdir` method now +have a *parent_mode* parameter to specify the mode for intermediate +directories when creating parent directories. This allows one to match the +behavior from Python 3.6 and earlier for :func:`os.makedirs`. + +.. + +.. date: 2025-05-19-21-08-25 +.. gh-issue: 134261 +.. nonce: ravGYm +.. section: Library + +zip: On reproducible builds, ZipFile uses UTC instead of the local time when +writing file datetimes to avoid underflows. + +.. + +.. date: 2025-05-19-20-29-35 +.. gh-issue: 133998 +.. nonce: KmElUw +.. section: Library + +Fix :exc:`struct.error` exception when creating a file with +:class:`gzip.GzipFile` or compressing data with :func:`gzip.compress` if the +system time is outside the range 00:00:00 UTC, January 1, 1970 through +06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* argument is +outside the range ``0`` to ``2**32-1``. + +.. + +.. date: 2025-03-01-13-36-02 +.. gh-issue: 128110 +.. nonce: 9wx_G0 +.. section: Library + +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, +per section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). + +.. + +.. date: 2024-11-02-02-02-31 +.. gh-issue: 107398 +.. nonce: uUtA6Q +.. section: Library + +Fix :mod:`tarfile` stream mode exception when process the file with the gzip +extra field. + +.. + +.. date: 2024-07-02-20-57-43 +.. gh-issue: 121109 +.. nonce: Tp6R2s +.. section: Library + +Fix :mod:`tarfile` performance issue when reading archives in streaming mode +(e.g. ``r|*``). + +.. + +.. bpo: 45509 +.. date: 2021-10-18-13-46-55 +.. nonce: Upwb60 +.. section: Library + +Gzip headers are now checked for corrupted NAME, COMMENT and HCRC fields. + +.. + +.. date: 2026-05-25-15-39-53 +.. gh-issue: 150387 +.. nonce: yzZ7jq +.. section: Tests + +Fix hang in +``test.test_profiling.test_sampling_profiler.test_live_collector_ui.TestLiveModeErrors.test_run_failed_script_live`` +on slow buildbots. The test now always queues a final ``q`` keystroke so the +live TUI loop exits even when the profiler collects enough samples to enter +the post-finished input loop. + +.. + +.. date: 2026-05-13-14-53-23 +.. gh-issue: 149776 +.. nonce: orqgsn +.. section: Tests + +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. + +.. + +.. date: 2026-05-21-15-14-59 +.. gh-issue: 148294 +.. nonce: VtFaW4 +.. section: Build + +Corrected the use of ``AC_PATH_TOOL`` in ``configure.ac`` to allow a C++ +compiler to be found on :envvar:`!PATH`. + +.. + +.. date: 2026-05-18-16-00-41 +.. gh-issue: 148260 +.. nonce: UwFiIX +.. section: Build + +On Linux when Python is linked to the musl C library, use a thread stack +size of at least 1 MiB instead of musl default which is 128 kiB. Patch by +Victor Stinner. + +.. + +.. date: 2026-05-14-22-09-46 +.. gh-issue: 149786 +.. nonce: UI-HZM +.. section: Windows + +Fixes virtual environment launchers on Windows free-threaded builds. + +.. + +.. date: 2026-05-06-21-36-53 +.. gh-issue: 124111 +.. nonce: m4OBX8 +.. section: Windows + +Updated Windows builds to use Tcl/Tk 9.0.3. + +.. + +.. date: 2026-04-29-14-44-51 +.. gh-issue: 138489 +.. nonce: 234aj6 +.. section: Windows + +Windows distributions now include a :file:`build-details.json` file (see +:pep:`739`). The legacy installer does not install it, but all other +distributions from python.org and all preset configurations in the +``PC\layout`` script will include one. + +.. + +.. date: 2026-04-26-23-14-45 +.. gh-issue: 149029 +.. nonce: oPTXP4 +.. section: Windows + +Update Windows installer to ship with SQLite 3.53.1. + +.. + +.. date: 2026-05-31-10-40-00 +.. gh-issue: 150644 +.. nonce: zLWyjj +.. section: macOS + +When system logging is enabled (with ``config.use_system_logger``, messages +are now tagged as public. This allows the macOS 26 system logger to view +messages without special configuration. + +.. + +.. date: 2026-04-26-23-15-09 +.. gh-issue: 149029 +.. nonce: Lsx--T +.. section: macOS + +Update macOS installer to ship with SQLite version 3.53.1. + +.. + +.. date: 2026-05-22-18-51-09 +.. gh-issue: 150258 +.. nonce: dh8GVK +.. section: Tools/Demos + +Update the tooltip on the Tachyon flame graph to show both absolute and +relative percentages. + +.. + +.. date: 2026-05-12-16-47-21 +.. gh-issue: 149725 +.. nonce: HZLBTZ +.. section: C API + +Add :c:func:`PySentinel_CheckExact` for exact :class:`sentinel` type tests +to accompany the existing :c:func:`PySentinel_Check`. + +.. + +.. date: 2026-02-25-13-37-10 +.. gh-issue: 145235 +.. nonce: -1ySNR +.. section: C API + +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst b/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst new file mode 100644 index 000000000000000..f58b248f3a0b986 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst @@ -0,0 +1,2 @@ +Fix ``dynamic_annotations.h`` header file when built with C++ and Valgrind: +add ``extern "C++" scope`` for the C++ template. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst new file mode 100644 index 000000000000000..44e96ce7be0bfb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst @@ -0,0 +1 @@ +Do not support ``none`` as a lazy imports mode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst new file mode 100644 index 000000000000000..02d050840ee1f9b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst @@ -0,0 +1,2 @@ +Fix a :exc:`SystemError` when compiling a compiling ``__classdict__`` class +annotation. Found by OSS-Fuzz in :oss-fuzz:`512907042`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst new file mode 100644 index 000000000000000..12fbffcd170684c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst @@ -0,0 +1 @@ +Fix a crash when a memory allocation fails during tokenizer initialization. A proper :exc:`MemoryError` is now raised instead. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst new file mode 100644 index 000000000000000..71a701bf3eb3551 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst @@ -0,0 +1,2 @@ +Fix a data race in the free-threaded build between :func:`gc.set_threshold` +and garbage collection scheduling during object allocation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst new file mode 100644 index 000000000000000..c397ad61f086c1b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst @@ -0,0 +1,3 @@ +Fix the frozen importer accepting module names with embedded null bytes, which +caused it to bypass the :data:`sys.modules` cache and create duplicate module +objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst new file mode 100644 index 000000000000000..1920c8cdfce4f4c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst @@ -0,0 +1,4 @@ +Fix malformed perf jitdump thread ids on macOS. The ``thread_id`` field of the +``JR_CODE_LOAD`` record was written as a 64-bit value instead of the 32-bit +value required by the jitdump format, which shifted every following field and +prevented profilers from resolving Python frames. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst new file mode 100644 index 000000000000000..78c896b669c2393 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst @@ -0,0 +1,4 @@ +Fix perf jitdump timestamps on macOS. Events were stamped using +``CLOCK_MONOTONIC``, but macOS profilers timestamp their samples with +``mach_absolute_time()``. The mismatch prevented the JIT code mappings from +lining up with the samples, so no Python frame could be resolved. diff --git a/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst b/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst new file mode 100644 index 000000000000000..d56ccbce2fa325c --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst @@ -0,0 +1,2 @@ +Generic builtin and standard library types now document the meaning of their +type parameters. diff --git a/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst b/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst new file mode 100644 index 000000000000000..e7fb9bf1b3bdf6a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst @@ -0,0 +1 @@ +Warn the user if a file will be overwritten when saving. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst b/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst new file mode 100644 index 000000000000000..d886e8ac6032a4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst @@ -0,0 +1,2 @@ +:meth:`imaplib.IMAP4.login` now raises exceptions with :class:`str` instead of +:class:`bytes`. Patch by Florian Best. diff --git a/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst b/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst new file mode 100644 index 000000000000000..6245af7e362e920 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst @@ -0,0 +1 @@ +Fix XML injection vulnerability in :func:`xmlrpc.client.dumps` where the ``methodname`` was not being escaped before interpolation into the XML body. diff --git a/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst b/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst new file mode 100644 index 000000000000000..db624aba31a9de0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst @@ -0,0 +1,2 @@ +Calling ``os.environ.clear()`` now emits ``os._clearenv`` auditing event. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst b/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst new file mode 100644 index 000000000000000..f8bc28659533af8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst @@ -0,0 +1 @@ +Add support for more encoding aliases `officially registered in IANA <https://www.iana.org/assignments/character-sets/character-sets.xhtml>`__. diff --git a/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst b/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst new file mode 100644 index 000000000000000..283a5ba44d1f19f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst @@ -0,0 +1,2 @@ +The :mod:`email` package now supports all aliases of Python codecs and uses +MIME/IANA names for all IANA registered charsets. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst b/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst new file mode 100644 index 000000000000000..9996cc7ec0e8664 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst @@ -0,0 +1 @@ +Fix race condition in :attr:`ssl.SSLContext.sni_callback` diff --git a/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst b/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst new file mode 100644 index 000000000000000..a0b7a9740cd518d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst @@ -0,0 +1 @@ +Fix ``profiling.sampling`` on Windows virtual environments to resolve the actual Python PID from a virtual environment shim. diff --git a/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst b/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst new file mode 100644 index 000000000000000..bda500383e7cda3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst @@ -0,0 +1 @@ +Fix a race condition in :meth:`collections.deque.index` with free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst b/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst new file mode 100644 index 000000000000000..42ed6ad7cd3c65f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst @@ -0,0 +1,4 @@ +Fix the ``--gecko`` collector in :mod:`profiling.sampling` that kept every +sample in memory. It now writes sample and marker data to temporary files +and reads them back, ultimately building the output file at the end. Patch +by Pablo Galindo and Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst b/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst new file mode 100644 index 000000000000000..f95a6ee6ee15bf7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst @@ -0,0 +1,3 @@ +Fix :class:`sqlite3.Blob` slice assignment to raise +:exc:`TypeError` and :exc:`IndexError` for type and size mismatches +respectively, even when the target slice is empty. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst b/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst new file mode 100644 index 000000000000000..95a4fb1c61d4c30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst @@ -0,0 +1,2 @@ +Encodings "KS_C_5601-1987", "KS X 1001", etc are now aliases of "CP949" +instead of "EUC-KR". diff --git a/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst b/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst new file mode 100644 index 000000000000000..0617fa068c844d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst @@ -0,0 +1,3 @@ +Fix :meth:`mmap.mmap.find` and :meth:`~mmap.mmap.rfind` to return ``-1`` +when searching for an empty subsequence with a start position past the end +of the mapping. diff --git a/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst b/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst new file mode 100644 index 000000000000000..20cab736552486d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst @@ -0,0 +1,3 @@ +:func:`shutil.move` now resolves symlinks via :func:`os.path.realpath` +when checking whether the destination is inside the source directory, +preventing a symlink-based bypass of that guard. diff --git a/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst new file mode 100644 index 000000000000000..a37d86cf423f820 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst @@ -0,0 +1,3 @@ +Fix a possible stack buffer overflow in :mod:`bz2` when a +:class:`bz2.BZ2Decompressor` is reused after a decompression error. +The decompressor now becomes unusable after libbz2 reports an error. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 593fa01bf25ed1e..080330c1cb75a53 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -91,63 +91,63 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" + "checksumValue": "15e8fc7dc059f7b156e53629540951c2691acd71e027f6f8f66dacab5c66c884" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.53.1.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.53.1.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.50.4.0" + "versionInfo": "3.53.1.0" }, { - "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "SPDXID": "SPDXRef-PACKAGE-tcl", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" + "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", - "name": "tcl-core", + "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" + "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index aaeffd58e799ede..1eca892fb12acee 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5343adc95840915b022b1d4524d0acb66b369ba2" + "checksumValue": "58101ef0951568acadd3117033bef084fea24cc1" }, { "algorithm": "SHA256", - "checksumValue": "1ec3bad08b6864c2c479e1fd941038c2dcd24c6d9a16400f4da54912d95aa321" + "checksumValue": "52d756026bf09befdb211c453e2009a646d6c6b519e6885e971b2550396619fb" } ], "fileName": "Modules/expat/expat.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cb0af01558ec7b6474d2bd0c9386380c82618e8f" + "checksumValue": "1dad2ab196cdbe37572674c465bd9187fdbe4495" }, { "algorithm": "SHA256", - "checksumValue": "6745a6b8cdd7344d4bd8f27f605363ed746e57ff02d4ebce3eb1806579cd030f" + "checksumValue": "740137e670d2f3b7269364ffb6f60064e6560091850c5d6f2c3bb1b8ca6e3dd1" } ], "fileName": "Modules/expat/xmlparse.c" @@ -1730,14 +1730,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" + "checksumValue": "a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_0/expat-2.8.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,7 +1745,7 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.0" + "versionInfo": "2.8.1" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 8fd7aba09241e63..d59a7c788fa9e02 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1813,7 +1813,6 @@ [const.METH_COEXIST] added = '3.2' # METH_STACKLESS is undocumented -# METH_FASTCALL is not part of limited API. # The following are defined in private headers, but historically # they were exported as part of the stable ABI. @@ -2149,8 +2148,6 @@ # New method flags in 3.7 (PEP 590): -[const.METH_FASTCALL] - added = '3.7' [const.METH_METHOD] added = '3.7' @@ -2300,6 +2297,10 @@ [data.PyStructSequence_UnnamedField] added = '3.11' +# Added in 3.7 but in the Stable ABI from 3.10 +[const.METH_FASTCALL] + added = '3.10' + # Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459) [struct.Py_buffer] added = '3.11' diff --git a/Modules/Setup b/Modules/Setup index 33737c21cb4066e..e97a78e628693dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #xx xxmodule.c #xxlimited xxlimited.c #xxlimited_35 xxlimited_35.c +#xxlimited_3_13 xxlimited_3_13.c #xxsubtype xxsubtype.c # Testing diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 19765bc313555b7..5f8b0cf482472d2 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -190,6 +190,7 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c +@MODULE_XXLIMITED_3_13_TRUE@xxlimited_3_13 xxlimited_3_13.c # for performance diff --git a/Modules/_abc.c b/Modules/_abc.c index 3c4e0280525e1eb..5826efbfecb6901 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -915,14 +915,14 @@ _abc.get_cache_token Returns the current ABC cache token. -The token is an opaque object (supporting equality testing) identifying the -current version of the ABC cache for virtual subclasses. The token changes -with every call to register() on any ABC. +The token is an opaque object (supporting equality testing) identifying +the current version of the ABC cache for virtual subclasses. The token +changes with every call to register() on any ABC. [clinic start generated code]*/ static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=d87acc04492f6bf3]*/ { _abcmodule_state *state = get_abc_state(module); return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 9679a7dde31b0d0..6620ee26449b163 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -955,12 +955,13 @@ Return the result this future represents. If the future has been cancelled, raises CancelledError. If the future's result isn't yet available, raises InvalidStateError. If -the future is done and has an exception set, this exception is raised. +the future is done and has an exception set, this exception is +raised. [clinic start generated code]*/ static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=ee20e126776cbb04]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -1095,15 +1096,15 @@ _asyncio.Future.add_done_callback Add a callback to be run when the future becomes done. -The callback is called with a single argument - the future object. If -the future is already done when this is called, the callback is +The callback is called with a single argument - the future object. +If the future is already done when this is called, the callback is scheduled with call_soon. [clinic start generated code]*/ static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=f4f6adb074cd3e0f]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -1252,15 +1253,15 @@ _asyncio.Future.cancel Cancel the future and schedule callbacks. -If the future is already done or cancelled, return False. Otherwise, -change the future's state to cancelled, schedule the callbacks and -return True. +If the future is already done or cancelled, return False. +Otherwise, change the future's state to cancelled, schedule the +callbacks and return True. [clinic start generated code]*/ static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ +/*[clinic end generated code: output=074956f35904b034 input=0c9157547a964c4c]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1292,13 +1293,13 @@ _asyncio.Future.done Return True if the future is done. -Done means either that a result / exception are available, or that the -future was cancelled. +Done means either that a result / exception are available, or that +the future was cancelled. [clinic start generated code]*/ static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ +/*[clinic end generated code: output=244c5ac351145096 input=acf2c2347f3c01d8]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1741,7 +1742,8 @@ static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_DONE_METHODDEF _ASYNCIO_FUTURE_GET_LOOP_METHODDEF _ASYNCIO_FUTURE__MAKE_CANCELLED_ERROR_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Futures are generic over the type of their results")}, {NULL, NULL} /* Sentinel */ }; @@ -2926,7 +2928,8 @@ static PyMethodDef TaskType_methods[] = { _ASYNCIO_TASK_SET_NAME_METHODDEF _ASYNCIO_TASK_GET_CORO_METHODDEF _ASYNCIO_TASK_GET_CONTEXT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Tasks are generic over the return type of their wrapped coroutines")}, {NULL, NULL} /* Sentinel */ }; @@ -3844,6 +3847,7 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) /*[clinic input] +@permit_long_summary _asyncio._swap_current_task loop: object @@ -3858,7 +3862,7 @@ This is intended for use during eager coroutine execution. static PyObject * _asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, PyObject *task) -/*[clinic end generated code: output=9f88de958df74c7e input=c9c72208d3d38b6c]*/ +/*[clinic end generated code: output=9f88de958df74c7e input=ec14ed25855e3068]*/ { _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); return swap_current_task(ts, loop, task); diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 329aa8e117ec3cf..a953f8bfa11aeaa 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -157,8 +157,8 @@ _bisect.bisect_right -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in -a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will -insert just after the rightmost x already there. +a[i:] have e > x. So if x already appears in the list, a.insert(i, x) +will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -169,7 +169,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=3a4bc09cc7c8a73d input=b476bc45667273ac]*/ +/*[clinic end generated code: output=3a4bc09cc7c8a73d input=27717afe1a61bfaa]*/ { return internal_bisect_right(a, x, lo, hi, key); } @@ -338,8 +338,8 @@ _bisect.bisect_left -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in -a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will -insert just before the leftmost x already there. +a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) +will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -350,7 +350,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=70749d6e5cae9284 input=9b4d49b5ddecfad7]*/ +/*[clinic end generated code: output=70749d6e5cae9284 input=259fedbe35e882e1]*/ { return internal_bisect_left(a, x, lo, hi, key); } diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 4bff90e6fd2b2e0..9db3ac39da52099 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -108,6 +108,7 @@ typedef struct { typedef struct { PyObject_HEAD bz_stream bzs; + int bzerror; char eof; /* Py_T_BOOL expects a char */ PyObject *unused_data; char needs_input; @@ -435,8 +436,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) d->bzs_avail_in_real += bzs->avail_in; - if (catch_bz2_error(bzret)) + if (catch_bz2_error(bzret)) { + d->bzerror = bzret; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 0); goto error; + } if (bzret == BZ_STREAM_END) { FT_ATOMIC_STORE_CHAR_RELAXED(d->eof, 1); break; @@ -577,7 +581,6 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) } /*[clinic input] -@permit_long_docstring_body _bz2.BZ2Decompressor.decompress data: Py_buffer @@ -585,32 +588,40 @@ _bz2.BZ2Decompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=3703e78f91757655]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=7f68faa9ff7a1b51]*/ { PyObject *result = NULL; PyMutex_Lock(&self->mutex); - if (self->eof) + if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); - else + } + else if (self->bzerror) { + // Re-entering BZ2_bzDecompress() after an error can write out of bounds. + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + } + else { result = decompress(self, data->buf, data->len, max_length); + } PyMutex_Unlock(&self->mutex); return result; } @@ -638,6 +649,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) } self->mutex = (PyMutex){0}; + self->bzerror = 0; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index ff52bfd8291ac14..272182f7bf49acc 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -55,14 +55,15 @@ _codecs.register Register a codec search function. -Search functions are expected to take one argument, the encoding name in -all lower case letters, and either return None, or a tuple of functions -(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object). +Search functions are expected to take one argument, the encoding +name in all lower case letters, and either return None, or a tuple +of functions (encoder, decoder, stream_reader, stream_writer) (or +a CodecInfo object). [clinic start generated code]*/ static PyObject * _codecs_register(PyObject *module, PyObject *search_function) -/*[clinic end generated code: output=d1bf21e99db7d6d3 input=369578467955cae4]*/ +/*[clinic end generated code: output=d1bf21e99db7d6d3 input=2321d8c8c0420dfc]*/ { if (PyCodec_Register(search_function)) return NULL; @@ -116,16 +117,16 @@ _codecs.encode Encodes obj using the codec registered for encoding. The default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_encode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=385148eb9a067c86 input=cd5b685040ff61f0]*/ +/*[clinic end generated code: output=385148eb9a067c86 input=e5271d443e391d7f]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -143,16 +144,16 @@ _codecs.decode Decodes obj using the codec registered for encoding. Default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_decode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=679882417dc3a0bd input=7702c0cc2fa1add6]*/ +/*[clinic end generated code: output=679882417dc3a0bd input=3e6254628f9ca538]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -962,14 +963,15 @@ _codecs.register_error Register the specified error handler under the name errors. handler must be a callable object, that will be called with an exception -instance containing information about the location of the encoding/decoding -error and must return a (replacement, new position) tuple. +instance containing information about the location of the +encoding/decoding error and must return a (replacement, new position) +tuple. [clinic start generated code]*/ static PyObject * _codecs_register_error_impl(PyObject *module, const char *errors, PyObject *handler) -/*[clinic end generated code: output=fa2f7d1879b3067d input=5e6709203c2e33fe]*/ +/*[clinic end generated code: output=fa2f7d1879b3067d input=5bea01dfe835d9d8]*/ { if (PyCodec_RegisterError(errors, handler)) return NULL; @@ -1007,13 +1009,13 @@ _codecs.lookup_error lookup_error(errors) -> handler -Return the error handler for the specified error handling name or raise a -LookupError, if no handler exists under this name. +Return the error handler for the specified error handling name or raise +a LookupError, if no handler exists under this name. [clinic start generated code]*/ static PyObject * _codecs_lookup_error_impl(PyObject *module, const char *name) -/*[clinic end generated code: output=087f05dc0c9a98cc input=4775dd65e6235aba]*/ +/*[clinic end generated code: output=087f05dc0c9a98cc input=86cfb6a7a9c67113]*/ { return PyCodec_LookupError(name); } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4ff05727ebc8ce3..5ca6362406a78b9 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1077,6 +1077,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) } /*[clinic input] +@permit_long_summary @critical_section _collections.deque.rotate as deque_rotate @@ -1089,7 +1090,7 @@ Rotate the deque n steps to the right. If n is negative, rotates left. static PyObject * deque_rotate_impl(dequeobject *deque, Py_ssize_t n) -/*[clinic end generated code: output=96c2402a371eb15d input=5bf834296246e002]*/ +/*[clinic end generated code: output=96c2402a371eb15d input=3543c3b2297de8f1]*/ { if (!_deque_rotate(deque, n)) Py_RETURN_NONE; @@ -1250,7 +1251,7 @@ _collections.deque.index as deque_index deque: dequeobject value as v: object start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL - stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = NULL / Return first index of value. @@ -1261,7 +1262,7 @@ Raises ValueError if the value is not present. static PyObject * deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ +/*[clinic end generated code: output=df45132753175ef9 input=1c3b19632cf3484f]*/ { Py_ssize_t i, n; PyObject *item; @@ -1269,22 +1270,23 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; + Py_ssize_t size = Py_SIZE(deque); if (start < 0) { - start += Py_SIZE(deque); + start += size; if (start < 0) start = 0; } if (stop < 0) { - stop += Py_SIZE(deque); + stop += size; if (stop < 0) stop = 0; } - if (stop > Py_SIZE(deque)) - stop = Py_SIZE(deque); + if (stop > size) + stop = size; if (start > stop) start = stop; - assert(0 <= start && start <= stop && stop <= Py_SIZE(deque)); + assert(0 <= start && start <= stop && stop <= size); for (i=0 ; i < start - BLOCKLEN ; i += BLOCKLEN) { b = b->rightlink; @@ -1854,7 +1856,7 @@ static PyMethodDef deque_methods[] = { DEQUE_ROTATE_METHODDEF DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("deques are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -2330,6 +2332,12 @@ defdict_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) return result; } + +PyDoc_STRVAR(defdict_class_getitem_doc, +"defaultdicts are generic over two types, signifying (respectively) the types \ +of the dictionary's keys and values"); + + static PyMethodDef defdict_methods[] = { {"__missing__", defdict_missing, METH_O, defdict_missing_doc}, @@ -2340,7 +2348,7 @@ static PyMethodDef defdict_methods[] = { {"__reduce__", defdict_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + defdict_class_getitem_doc}, {NULL} }; @@ -2420,7 +2428,7 @@ defdict_or(PyObject* left, PyObject* right) self = right; other = left; } - if (!PyDict_Check(other)) { + if (!PyAnyDict_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } // Like copy(), this calls the object's class. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 98ac821c525a647..e891249668c20f5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -708,7 +708,7 @@ StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) } assert(stginfo); /* Cannot be NULL for structure/union instances */ - parg->tag = 'V'; + parg->tag = "V"; parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; @@ -1282,7 +1282,7 @@ PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -1703,7 +1703,7 @@ PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; - p->tag = 'P'; + p->tag = "P"; p->pffi_type = &ffi_type_pointer; p->value.p = (char *)self->b_ptr; p->obj = Py_NewRef(self); @@ -1909,7 +1909,7 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -1998,7 +1998,7 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2092,7 +2092,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2110,7 +2110,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2127,7 +2127,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2152,7 +2152,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; - if (a->tag == 'P') { + if (strcmp(a->tag, "P") == 0) { return Py_NewRef(value); } } @@ -2165,7 +2165,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; Py_INCREF(value); // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; @@ -2191,7 +2191,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ Py_BEGIN_CRITICAL_SECTION(value); @@ -2332,7 +2332,8 @@ PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); memcpy(&parg->value, self->b_ptr, self->b_size); @@ -2578,7 +2579,8 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) @@ -2832,7 +2834,7 @@ PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -4303,7 +4305,7 @@ _byref(ctypes_state *st, PyObject *obj) return NULL; } - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = obj; parg->value.p = ((CDataObject *)obj)->b_ptr; @@ -5303,7 +5305,7 @@ Array_length(PyObject *myself) static PyMethodDef Array_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Arrays are generic over the type of their elements")}, { NULL, NULL } }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a0c9d8b70fee469..991ff0d675c2f1c 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -446,7 +446,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index e208e27c5dbed42..e453cfeec9cc8ca 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -468,7 +468,7 @@ PyCArgObject_new(ctypes_state *st) if (p == NULL) return NULL; p->pffi_type = NULL; - p->tag = '\0'; + p->tag = ""; p->obj = NULL; memset(&p->value, 0, sizeof(p->value)); PyObject_GC_Track(p); @@ -512,45 +512,50 @@ static PyObject * PyCArg_repr(PyObject *op) { PyCArgObject *self = _PyCArgObject_CAST(op); - switch(self->tag) { + + if (strlen(self->tag) != 1) { + goto generic; + } + + switch(self->tag[0]) { case 'b': case 'B': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.b); case 'h': case 'H': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.h); case 'i': case 'I': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.i); case 'l': case 'L': - return PyUnicode_FromFormat("<cparam '%c' (%ld)>", + return PyUnicode_FromFormat("<cparam '%s' (%ld)>", self->tag, self->value.l); case 'q': case 'Q': - return PyUnicode_FromFormat("<cparam '%c' (%lld)>", + return PyUnicode_FromFormat("<cparam '%s' (%lld)>", self->tag, self->value.q); case 'd': case 'f': { - PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); + PyObject *f = PyFloat_FromDouble((strcmp(self->tag, "f") == 0) ? self->value.f : self->value.d); if (f == NULL) { return NULL; } - PyObject *result = PyUnicode_FromFormat("<cparam '%c' (%R)>", self->tag, f); + PyObject *result = PyUnicode_FromFormat("<cparam '%s' (%R)>", self->tag, f); Py_DECREF(f); return result; } case 'c': if (is_literal_char((unsigned char)self->value.c)) { - return PyUnicode_FromFormat("<cparam '%c' ('%c')>", + return PyUnicode_FromFormat("<cparam '%s' ('%c')>", self->tag, self->value.c); } else { - return PyUnicode_FromFormat("<cparam '%c' ('\\x%02x')>", + return PyUnicode_FromFormat("<cparam '%s' ('\\x%02x')>", self->tag, (unsigned char)self->value.c); } @@ -561,20 +566,16 @@ PyCArg_repr(PyObject *op) case 'z': case 'Z': case 'P': - return PyUnicode_FromFormat("<cparam '%c' (%p)>", + return PyUnicode_FromFormat("<cparam '%s' (%p)>", self->tag, self->value.p); - break; default: - if (is_literal_char((unsigned char)self->tag)) { - return PyUnicode_FromFormat("<cparam '%c' at %p>", - (unsigned char)self->tag, (void *)self); - } - else { - return PyUnicode_FromFormat("<cparam 0x%02x at %p>", - (unsigned char)self->tag, (void *)self); - } + break; } + +generic: + return PyUnicode_FromFormat("<cparam '%s' at %p>", + self->tag, (void *)self); } static PyMemberDef PyCArgType_members[] = { @@ -1807,7 +1808,7 @@ _ctypes_byref_impl(PyObject *module, PyObject *obj, Py_ssize_t offset) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(obj); parg->value.p = (char *)((CDataObject *)obj)->b_ptr + offset; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7b6b7f08582251b..248559aa364a198 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -494,7 +494,7 @@ PyObject *_ctypes_callproc(ctypes_state *st, struct tagPyCArgObject { PyObject_HEAD ffi_type *pffi_type; - char tag; + const char *tag; union { char c; char b; @@ -511,7 +511,7 @@ struct tagPyCArgObject { long double G[2]; } value; PyObject *obj; - Py_ssize_t size; /* for the 'V' tag */ + Py_ssize_t size; /* for the "V" tag */ }; #define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 83802605e1f4dc9..52411e413533ce2 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -360,17 +360,17 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self) } /*[clinic input] -@permit_long_docstring_body _curses_panel.panel.hide Hide the panel. -This does not delete the object, it just makes the window on screen invisible. +This does not delete the object, it just makes the window on screen +invisible. [clinic start generated code]*/ static PyObject * _curses_panel_panel_hide_impl(PyCursesPanelObject *self) -/*[clinic end generated code: output=a7bbbd523e1eab49 input=9071b463a39a1a6a]*/ +/*[clinic end generated code: output=a7bbbd523e1eab49 input=9456aca9b264dde1]*/ { int rtn = hide_panel(self->pan); return curses_panel_panel_check_err(self, rtn, "hide_panel", "hide"); @@ -772,12 +772,13 @@ _curses_panel.update_panels Updates the virtual screen after changes in the panel stack. -This does not call curses.doupdate(), so you'll have to do this yourself. +This does not call curses.doupdate(), so you'll have to do this +yourself. [clinic start generated code]*/ static PyObject * _curses_panel_update_panels_impl(PyObject *module) -/*[clinic end generated code: output=2f3b4c2e03d90ded input=5299624c9a708621]*/ +/*[clinic end generated code: output=2f3b4c2e03d90ded input=0d0db79f05ec3ef4]*/ { PyCursesInitialised; update_panels(); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 000d7318557a6e6..4438e384aab9b26 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1359,7 +1359,6 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, } /*[clinic input] -@permit_long_docstring_body _curses.window.border ls: object(c_default="NULL") = _curses.ACS_VLINE @@ -1382,10 +1381,10 @@ _curses.window.border Draw a border around the edges of the window. -Each parameter specifies the character to use for a specific part of the -border. The characters can be specified as integers or as one-character -strings. A 0 value for any parameter will cause the default character to be -used for that parameter. +Each parameter specifies the character to use for a specific part of +the border. The characters can be specified as integers or as +one-character strings. A 0 value for any parameter will cause the +default character to be used for that parameter. [clinic start generated code]*/ static PyObject * @@ -1393,7 +1392,7 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *rs, PyObject *ts, PyObject *bs, PyObject *tl, PyObject *tr, PyObject *bl, PyObject *br) -/*[clinic end generated code: output=670ef38d3d7c2aa3 input=adaafca87488ee35]*/ +/*[clinic end generated code: output=670ef38d3d7c2aa3 input=42568c1458221d24]*/ { chtype ch[8]; int i, rtn; @@ -1436,14 +1435,15 @@ _curses.window.box Draw a border around the edges of the window. -Similar to border(), but both ls and rs are verch and both ts and bs are -horch. The default corner characters are always used by this function. +Similar to border(), but both ls and rs are verch and both ts and bs +are horch. The default corner characters are always used by this +function. [clinic start generated code]*/ static PyObject * _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, PyObject *verch, PyObject *horch) -/*[clinic end generated code: output=f3fcb038bb287192 input=f00435f9c8c98f60]*/ +/*[clinic end generated code: output=f3fcb038bb287192 input=e11acb7dbf6790b6]*/ { chtype ch1 = 0, ch2 = 0; if (group_right_1) { @@ -1596,7 +1596,6 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.derwin [ @@ -1613,15 +1612,15 @@ _curses.window.derwin Create a sub-window (window-relative coordinates). -derwin() is the same as calling subwin(), except that begin_y and begin_x -are relative to the origin of the window, rather than relative to the entire -screen. +derwin() is the same as calling subwin(), except that begin_y and +begin_x are relative to the origin of the window, rather than +relative to the entire screen. [clinic start generated code]*/ static PyObject * _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=7924b112d9f70d6e input=ebe95ded1c284c8e]*/ +/*[clinic end generated code: output=7924b112d9f70d6e input=6efb50722be444ba]*/ { WINDOW *win; @@ -1731,7 +1730,6 @@ curses_check_signals_on_input_error(PyCursesWindowObject *self, } /*[clinic input] -@permit_long_docstring_body _curses.window.getch [ @@ -1744,15 +1742,16 @@ _curses.window.getch Get a character code from terminal keyboard. -The integer returned does not have to be in ASCII range: function keys, -keypad keys and so on return numbers higher than 256. In no-delay mode, -1 -is returned if there is no input, else getch() waits until a key is pressed. +The integer returned does not have to be in ASCII range: function +keys, keypad keys and so on return numbers higher than 256. In +no-delay mode, -1 is returned if there is no input, else getch() +waits until a key is pressed. [clinic start generated code]*/ static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=e1639e87d545e676 input=9a053077373e2a30]*/ +/*[clinic end generated code: output=e1639e87d545e676 input=0dc5ff40e079787a]*/ { int rtn; @@ -1779,7 +1778,6 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.getkey [ @@ -1792,15 +1790,16 @@ _curses.window.getkey Get a character (string) from terminal keyboard. -Returning a string instead of an integer, as getch() does. Function keys, -keypad keys and other special keys return a multibyte string containing the -key name. In no-delay mode, an exception is raised if there is no input. +Returning a string instead of an integer, as getch() does. Function +keys, keypad keys and other special keys return a multibyte string +containing the key name. In no-delay mode, an exception is raised +if there is no input. [clinic start generated code]*/ static PyObject * _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=8490a182db46b10f input=5177f03fb6c31ea6]*/ +/*[clinic end generated code: output=8490a182db46b10f input=bd24a7da1ed9c73b]*/ { int rtn; @@ -2021,7 +2020,6 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.insch [ @@ -2042,15 +2040,15 @@ _curses.window.insch Insert a character before the current or specified position. -All characters to the right of the cursor are shifted one position right, with -the rightmost characters on the line being lost. +All characters to the right of the cursor are shifted one position +right, with the rightmost characters on the line being lost. [clinic start generated code]*/ static PyObject * _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=3f2a230cb09fed5a]*/ +/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=d662a0f96f33e15a]*/ { int rtn; chtype ch_ = 0; @@ -2072,7 +2070,6 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.inch [ @@ -2085,13 +2082,14 @@ _curses.window.inch Return the character at the given position in the window. -The bottom 8 bits are the character proper, and upper bits are the attributes. +The bottom 8 bits are the character proper, and upper bits are the +attributes. [clinic start generated code]*/ static PyObject * _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=97ca8581baaafd06 input=a5846f315464dc86]*/ +/*[clinic end generated code: output=97ca8581baaafd06 input=7a03956d94dc9a69]*/ { chtype rtn; const char *funcname; @@ -2183,18 +2181,18 @@ _curses.window.insstr Insert the string before the current or specified position. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor. All characters to the right of -the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, -if specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor. All characters to the +right of the cursor are shifted right, with the rightmost characters +on the line being lost. The cursor position does not change (after +moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int group_right_1, long attr) -/*[clinic end generated code: output=c259a5265ad0b777 input=6827cddc6340a7f3]*/ +/*[clinic end generated code: output=c259a5265ad0b777 input=dbfbdd3892155ea6]*/ { int rtn; int strtype; @@ -2260,7 +2258,6 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.insnstr [ @@ -2284,19 +2281,19 @@ _curses.window.insnstr Insert at most n characters of the string. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor, up to n characters. If n is zero -or negative, the entire string is inserted. All characters to the right -of the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, if -specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor, up to n characters. If +n is zero or negative, the entire string is inserted. All +characters to the right of the cursor are shifted right, with the +rightmost characters on the line being lost. The cursor position +does not change (after moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int n, int group_right_1, long attr) -/*[clinic end generated code: output=971a32ea6328ec8b input=dcdc554102fbcd5d]*/ +/*[clinic end generated code: output=971a32ea6328ec8b input=fd0a9b65b84b385f]*/ { int rtn; int strtype; @@ -2361,7 +2358,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _curses.window.is_linetouched line: int @@ -2370,12 +2367,13 @@ _curses.window.is_linetouched Return True if the specified line was modified, otherwise return False. -Raise a curses.error exception if line is not valid for the given window. +Raise a curses.error exception if line is not valid for the given +window. [clinic start generated code]*/ static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) -/*[clinic end generated code: output=ad4a4edfee2db08c input=af71c040b951c467]*/ +/*[clinic end generated code: output=ad4a4edfee2db08c input=18924dfac25ab7f1]*/ { int erg; erg = is_linetouched(self->win, line); @@ -2388,7 +2386,6 @@ _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) #ifdef py_is_pad /*[clinic input] -@permit_long_docstring_body _curses.window.noutrefresh [ @@ -2403,9 +2400,9 @@ _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * @@ -2413,22 +2410,21 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=809a1f3c6a03e23e input=b39fe8fc79b9980b]*/ +/*[clinic end generated code: output=809a1f3c6a03e23e input=8b4c74bf55008803]*/ #else /*[clinic input] -@permit_long_docstring_body _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=6ef6dec666643fee input=6a9f59ae5e4c139e]*/ +/*[clinic end generated code: output=6ef6dec666643fee input=a7c6306f8af9d0dd]*/ #endif { int rtn; @@ -2461,7 +2457,6 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) } /*[clinic input] -@permit_long_docstring_body _curses.window.overlay destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2478,14 +2473,15 @@ _curses.window.overlay Overlay the window on top of destwin. -The windows need not be the same size, only the overlapping region is copied. -This copy is non-destructive, which means that the current background -character does not overwrite the old contents of destwin. +The windows need not be the same size, only the overlapping region +is copied. This copy is non-destructive, which means that the +current background character does not overwrite the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overlay() can be used. sminrow and smincol are the upper-left coordinates -of the source window, and the other variables mark a rectangle in the -destination window. +To get fine-grained control over the copied region, the second form +of overlay() can be used. sminrow and smincol are the upper-left +coordinates of the source window, and the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2493,7 +2489,7 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=dd6af34deb892a65]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=da0cec7f7bda1b3f]*/ { int rtn; @@ -2509,7 +2505,6 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, } /*[clinic input] -@permit_long_docstring_body _curses.window.overwrite destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2526,14 +2521,15 @@ _curses.window.overwrite Overwrite the window on top of destwin. -The windows need not be the same size, in which case only the overlapping -region is copied. This copy is destructive, which means that the current -background character overwrites the old contents of destwin. +The windows need not be the same size, in which case only the +overlapping region is copied. This copy is destructive, which means +that the current background character overwrites the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overwrite() can be used. sminrow and smincol are the upper-left coordinates -of the source window, the other variables mark a rectangle in the destination -window. +To get fine-grained control over the copied region, the second form +of overwrite() can be used. sminrow and smincol are the upper-left +coordinates of the source window, the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2542,7 +2538,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=e84d8ebdf1c09596]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=4244ab8a97087898]*/ { int rtn; @@ -2558,6 +2554,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, } /*[clinic input] +@permit_long_summary _curses.window.putwin file: object @@ -2570,7 +2567,7 @@ This information can be later retrieved using the getwin() function. static PyObject * _curses_window_putwin_impl(PyCursesWindowObject *self, PyObject *file) -/*[clinic end generated code: output=fdae68ac59b0281b input=0608648e09c8ea0a]*/ +/*[clinic end generated code: output=fdae68ac59b0281b input=959fc85a9e4a31c2]*/ { /* We have to simulate this by writing to a temporary FILE*, then reading back, then writing to the argument file. */ @@ -2626,7 +2623,6 @@ _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) } /*[clinic input] -@permit_long_docstring_body _curses.window.refresh [ @@ -2642,23 +2638,24 @@ _curses.window.refresh Update the display immediately. Synchronize actual screen with previous drawing/deleting methods. -The 6 optional arguments can only be specified when the window is a pad -created with newpad(). The additional parameters are needed to indicate -what part of the pad and screen are involved. pminrow and pmincol specify -the upper left-hand corner of the rectangle to be displayed in the pad. -sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to -be displayed on the screen. The lower right-hand corner of the rectangle to -be displayed in the pad is calculated from the screen coordinates, since the -rectangles must be the same size. Both rectangles must be entirely contained -within their respective structures. Negative values of pminrow, pmincol, -sminrow, or smincol are treated as if they were zero. +The 6 optional arguments can only be specified when the window is +a pad created with newpad(). The additional parameters are needed +to indicate what part of the pad and screen are involved. pminrow +and pmincol specify the upper left-hand corner of the rectangle to +be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol +specify the edges of the rectangle to be displayed on the screen. +The lower right-hand corner of the rectangle to be displayed in the +pad is calculated from the screen coordinates, since the rectangles +must be the same size. Both rectangles must be entirely contained +within their respective structures. Negative values of pminrow, +pmincol, sminrow, or smincol are treated as if they were zero. [clinic start generated code]*/ static PyObject * _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=42199543115e6e63 input=65405c03290496a6]*/ +/*[clinic end generated code: output=42199543115e6e63 input=ff2e900c6b2696b1]*/ { int rtn; @@ -2711,7 +2708,6 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, } /*[clinic input] -@permit_long_docstring_body _curses.window.subwin [ @@ -2728,14 +2724,14 @@ _curses.window.subwin Create a sub-window (screen-relative coordinates). -By default, the sub-window will extend from the specified position to the -lower right corner of the window. +By default, the sub-window will extend from the specified position +to the lower right corner of the window. [clinic start generated code]*/ static PyObject * _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=93e898afc348f59a input=5292cf610e2f3585]*/ +/*[clinic end generated code: output=93e898afc348f59a input=07b5058cb8820595]*/ { WINDOW *win; const char *funcname; @@ -2763,7 +2759,6 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.scroll [ @@ -2774,13 +2769,14 @@ _curses.window.scroll Scroll the screen or scrolling region. -Scroll upward if the argument is positive and downward if it is negative. +Scroll upward if the argument is positive and downward if it is +negative. [clinic start generated code]*/ static PyObject * _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines) -/*[clinic end generated code: output=4541a8a11852d360 input=386456524c550113]*/ +/*[clinic end generated code: output=4541a8a11852d360 input=d8d81a5b52b9b40f]*/ { int rtn; const char *funcname; @@ -2796,7 +2792,6 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.touchline start: int @@ -2808,14 +2803,15 @@ _curses.window.touchline Pretend count lines have been changed, starting with line start. -If changed is supplied, it specifies whether the affected lines are marked -as having been changed (changed=True) or unchanged (changed=False). +If changed is supplied, it specifies whether the affected lines are +marked as having been changed (changed=True) or unchanged +(changed=False). [clinic start generated code]*/ static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=36e13b6f5eb591f5]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=e0dc62f90d9dea55]*/ { int rtn; const char *funcname; @@ -3182,20 +3178,20 @@ _curses.cbreak Enter cbreak mode. -In cbreak mode (sometimes called "rare" mode) normal tty line buffering is -turned off and characters are available to be read one by one. However, -unlike raw mode, special characters (interrupt, quit, suspend, and flow -control) retain their effects on the tty driver and calling program. -Calling first raw() then cbreak() leaves the terminal in cbreak mode. +In cbreak mode (sometimes called "rare" mode) normal tty line buffering +is turned off and characters are available to be read one by one. +However, unlike raw mode, special characters (interrupt, quit, suspend, +and flow control) retain their effects on the tty driver and calling +program. Calling first raw() then cbreak() leaves the terminal in +cbreak mode. [clinic start generated code]*/ static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=42d81687f11ddbf3]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] -@permit_long_docstring_body _curses.color_content color_number: color @@ -3204,13 +3200,14 @@ _curses.color_content Return the red, green, and blue (RGB) components of the specified color. -A 3-tuple is returned, containing the R, G, B values for the given color, -which will be between 0 (no component) and 1000 (maximum amount of component). +A 3-tuple is returned, containing the R, G, B values for the given +color, which will be between 0 (no component) and 1000 (maximum amount +of component). [clinic start generated code]*/ static PyObject * _curses_color_content_impl(PyObject *module, int color_number) -/*[clinic end generated code: output=17b466df7054e0de input=baffe25b351eb916]*/ +/*[clinic end generated code: output=17b466df7054e0de input=c95fb50093fa0be0]*/ { _CURSES_COLOR_VAL_TYPE r,g,b; @@ -3236,12 +3233,13 @@ _curses.color_pair Return the attribute value for displaying text in the specified color. This attribute value can be combined with A_STANDOUT, A_REVERSE, and the -other A_* attributes. pair_number() is the counterpart to this function. +other A_* attributes. pair_number() is the counterpart to this +function. [clinic start generated code]*/ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ +/*[clinic end generated code: output=60718abb10ce9feb input=cf74bb81d3cc3370]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -3259,14 +3257,14 @@ _curses.curs_set Set the cursor state. If the terminal supports the visibility requested, the previous cursor -state is returned; otherwise, an exception is raised. On many terminals, -the "visible" mode is an underline cursor and the "very visible" mode is -a block cursor. +state is returned; otherwise, an exception is raised. On many +terminals, the "visible" mode is an underline cursor and the "very +visible" mode is a block cursor. [clinic start generated code]*/ static PyObject * _curses_curs_set_impl(PyObject *module, int visibility) -/*[clinic end generated code: output=ee8e62483b1d6cd4 input=81a7924a65d29504]*/ +/*[clinic end generated code: output=ee8e62483b1d6cd4 input=e010767a328f322b]*/ { int erg; @@ -3301,14 +3299,15 @@ _curses.def_shell_mode Save the current terminal mode as the "shell" mode. -The "shell" mode is the mode when the running program is not using curses. +The "shell" mode is the mode when the running program is not using +curses. Subsequent calls to reset_shell_mode() will restore this mode. [clinic start generated code]*/ static PyObject * _curses_def_shell_mode_impl(PyObject *module) -/*[clinic end generated code: output=d6e42f5c768f860f input=5ead21f6f0baa894]*/ +/*[clinic end generated code: output=d6e42f5c768f860f input=3809f85615c0b693]*/ NoArgNoReturnFunctionBody(def_shell_mode) /*[clinic input] @@ -3350,12 +3349,13 @@ _curses.echo Enter echo mode. -In echo mode, each character input is echoed to the screen as it is entered. +In echo mode, each character input is echoed to the screen as it is +entered. [clinic start generated code]*/ static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=b4e9064326da9da4]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -3389,17 +3389,17 @@ _curses_erasechar_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body _curses.flash Flash the screen. -That is, change it to reverse-video and then change it back in a short interval. +That is, change it to reverse-video and then change it back in a short +interval. [clinic start generated code]*/ static PyObject * _curses_flash_impl(PyObject *module) -/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=dd33d718e6edf436]*/ +/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=90878e305432add9]*/ NoArgNoReturnFunctionBody(flash) /*[clinic input] @@ -3407,13 +3407,13 @@ _curses.flushinp Flush all input buffers. -This throws away any typeahead that has been typed by the user and has not -yet been processed by the program. +This throws away any typeahead that has been typed by the user and has +not yet been processed by the program. [clinic start generated code]*/ static PyObject * _curses_flushinp_impl(PyObject *module) -/*[clinic end generated code: output=7e7a1fc1473960f5 input=59d042e705cef5ec]*/ +/*[clinic end generated code: output=7e7a1fc1473960f5 input=3a63c7213be8043c]*/ NoArgNoReturnVoidFunctionBody(flushinp) #ifdef getsyx @@ -3599,6 +3599,7 @@ _curses_has_colors_impl(PyObject *module) NoArgTrueFalseFunctionBody(has_colors) /*[clinic input] +@permit_long_summary _curses.has_ic Return True if the terminal has insert- and delete-character capabilities. @@ -3606,7 +3607,7 @@ Return True if the terminal has insert- and delete-character capabilities. static PyObject * _curses_has_ic_impl(PyObject *module) -/*[clinic end generated code: output=6be24da9cb1268fe input=9bc2d3a797cc7324]*/ +/*[clinic end generated code: output=6be24da9cb1268fe input=e37fa080d879f7a9]*/ NoArgTrueFalseFunctionBody(has_ic) /*[clinic input] @@ -3622,6 +3623,7 @@ NoArgTrueFalseFunctionBody(has_il) #ifdef HAVE_CURSES_HAS_KEY /*[clinic input] +@permit_long_summary _curses.has_key key: int @@ -3633,7 +3635,7 @@ Return True if the current terminal type recognizes a key with that value. static PyObject * _curses_has_key_impl(PyObject *module, int key) -/*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ +/*[clinic end generated code: output=19ad48319414d0b1 input=046ac6c72bbc9587]*/ { PyCursesStatefulInitialised(module); @@ -3688,13 +3690,14 @@ _curses.init_pair Change the definition of a color-pair. -If the color-pair was previously initialized, the screen is refreshed and -all occurrences of that color-pair are changed to the new definition. +If the color-pair was previously initialized, the screen is refreshed +and all occurrences of that color-pair are changed to the new +definition. [clinic start generated code]*/ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) -/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ +/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=5486c3a105130dae]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -3914,14 +3917,14 @@ _curses.get_escdelay Gets the curses ESCDELAY setting. -Gets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Gets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_get_escdelay_impl(PyObject *module) -/*[clinic end generated code: output=222fa1a822555d60 input=be2d5b3dd974d0a4]*/ +/*[clinic end generated code: output=222fa1a822555d60 input=b39eeae4b8f169ab]*/ { return PyLong_FromLong(ESCDELAY); } @@ -3933,14 +3936,14 @@ _curses.set_escdelay Sets the curses ESCDELAY setting. -Sets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Sets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_set_escdelay_impl(PyObject *module, int ms) -/*[clinic end generated code: output=43818efbf7980ac4 input=7796fe19f111e250]*/ +/*[clinic end generated code: output=43818efbf7980ac4 input=cc2529bcdda3b06c]*/ { if (ms <= 0) { PyErr_SetString(PyExc_ValueError, "ms must be > 0"); @@ -3955,13 +3958,13 @@ _curses.get_tabsize Gets the curses TABSIZE setting. -Gets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Gets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_get_tabsize_impl(PyObject *module) -/*[clinic end generated code: output=7e9e51fb6126fbdf input=74af86bf6c9f5d7e]*/ +/*[clinic end generated code: output=7e9e51fb6126fbdf input=58bdaacb337c103b]*/ { return PyLong_FromLong(TABSIZE); } @@ -3973,13 +3976,13 @@ _curses.set_tabsize Sets the curses TABSIZE setting. -Sets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Sets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_set_tabsize_impl(PyObject *module, int size) -/*[clinic end generated code: output=c1de5a76c0daab1e input=78cba6a3021ad061]*/ +/*[clinic end generated code: output=c1de5a76c0daab1e input=34c1be9a78cd28a2]*/ { if (size <= 0) { PyErr_SetString(PyExc_ValueError, "size must be > 0"); @@ -4087,18 +4090,17 @@ _curses_killchar_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body _curses.longname Return the terminfo long name field describing the current terminal. -The maximum length of a verbose description is 128 characters. It is defined -only after the call to initscr(). +The maximum length of a verbose description is 128 characters. It is +defined only after the call to initscr(). [clinic start generated code]*/ static PyObject * _curses_longname_impl(PyObject *module) -/*[clinic end generated code: output=fdf30433727ef568 input=5de06852f2230ddb]*/ +/*[clinic end generated code: output=fdf30433727ef568 input=a924fabba0de78a6]*/ NoArgReturnStringFunctionBody(longname) /*[clinic input] @@ -4133,13 +4135,13 @@ _curses.mouseinterval Set and retrieve the maximum time between press and release in a click. Set the maximum time that can elapse between press and release events in -order for them to be recognized as a click, and return the previous interval -value. +order for them to be recognized as a click, and return the previous +interval value. [clinic start generated code]*/ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) -/*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ +/*[clinic end generated code: output=c4f5ff04354634c5 input=b90249254389c080]*/ { PyCursesStatefulInitialised(module); int value = mouseinterval(interval); @@ -4160,14 +4162,15 @@ _curses.mousemask Set the mouse events to be reported, and return a tuple (availmask, oldmask). Return a tuple (availmask, oldmask). availmask indicates which of the -specified mouse events can be reported; on complete failure it returns 0. -oldmask is the previous value of the given window's mouse event mask. -If this function is never called, no mouse events are ever reported. +specified mouse events can be reported; on complete failure it returns +0. oldmask is the previous value of the given window's mouse event +mask. If this function is never called, no mouse events are ever +reported. [clinic start generated code]*/ static PyObject * _curses_mousemask_impl(PyObject *module, unsigned long newmask) -/*[clinic end generated code: output=9406cf1b8a36e485 input=b92ff4fbe5ce61b1]*/ +/*[clinic end generated code: output=9406cf1b8a36e485 input=78990ec6c52aa888]*/ { mmask_t oldmask, availmask; @@ -4249,14 +4252,14 @@ _curses.newwin Return a new window. -By default, the window will extend from the specified position to the lower -right corner of the screen. +By default, the window will extend from the specified position to the +lower right corner of the screen. [clinic start generated code]*/ static PyObject * _curses_newwin_impl(PyObject *module, int nlines, int ncols, int group_right_1, int begin_y, int begin_x) -/*[clinic end generated code: output=c1e0a8dc8ac2826c input=29312c15a72a003d]*/ +/*[clinic end generated code: output=c1e0a8dc8ac2826c input=a1517cbfea4ab24b]*/ { WINDOW *win; @@ -4281,13 +4284,14 @@ _curses.nl Enter newline mode. -This mode translates the return key into newline on input, and translates -newline into return and line-feed on output. Newline mode is initially on. +This mode translates the return key into newline on input, and +translates newline into return and line-feed on output. Newline mode +is initially on. [clinic start generated code]*/ static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=3fb21dcf55521ee4]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -4321,13 +4325,13 @@ _curses.nonl Leave newline mode. -Disable translation of return into newline on input, and disable low-level -translation of newline into newline/return on output. +Disable translation of return into newline on input, and disable +low-level translation of newline into newline/return on output. [clinic start generated code]*/ static PyObject * _curses_nonl_impl(PyObject *module) -/*[clinic end generated code: output=99e917e9715770c6 input=9d37dd122d3022fc]*/ +/*[clinic end generated code: output=99e917e9715770c6 input=75cce08e4b6b3ef1]*/ NoArgNoReturnFunctionBody(nonl) /*[clinic input] @@ -4358,6 +4362,7 @@ _curses_noraw_impl(PyObject *module) NoArgNoReturnFunctionBody(noraw) /*[clinic input] +@permit_long_summary _curses.pair_content pair_number: pair @@ -4369,7 +4374,7 @@ Return a tuple (fg, bg) containing the colors for the requested color pair. static PyObject * _curses_pair_content_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=4a726dd0e6885f3f input=03970f840fc7b739]*/ +/*[clinic end generated code: output=4a726dd0e6885f3f input=faede9e26f1f2ca4]*/ { _CURSES_COLOR_NUM_TYPE f, b; @@ -4393,6 +4398,7 @@ _curses_pair_content_impl(PyObject *module, int pair_number) } /*[clinic input] +@permit_long_summary _curses.pair_number attr: int @@ -4405,7 +4411,7 @@ color_pair() is the counterpart to this function. static PyObject * _curses_pair_number_impl(PyObject *module, int attr) -/*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ +/*[clinic end generated code: output=85bce7d65c0aa3f4 input=b11152a78c2f9abf]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -4414,6 +4420,7 @@ _curses_pair_number_impl(PyObject *module, int attr) } /*[clinic input] +@permit_long_summary _curses.putp string: str(accept={robuffer}) @@ -4426,7 +4433,7 @@ Note that the output of putp() always goes to standard output. static PyObject * _curses_putp_impl(PyObject *module, const char *string) -/*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ +/*[clinic end generated code: output=e98081d1b8eb5816 input=2f3b9e0f22829ee7]*/ { return curses_check_err(module, putp(string), "putp", NULL); } @@ -4539,13 +4546,13 @@ _curses.raw Enter raw mode. In raw mode, normal line buffering and processing of interrupt, quit, -suspend, and flow control keys are turned off; characters are presented to -curses input functions one by one. +suspend, and flow control keys are turned off; characters are presented +to curses input functions one by one. [clinic start generated code]*/ static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ +/*[clinic end generated code: output=a750e4b342be015b input=18a7de7eef16987a]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4595,13 +4602,13 @@ _curses.resizeterm Resize the standard and current windows to the specified dimensions. -Adjusts other bookkeeping data used by the curses library that record the -window dimensions (in particular the SIGWINCH handler). +Adjusts other bookkeeping data used by the curses library that record +the window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ +/*[clinic end generated code: output=4de3abab50c67f02 input=7f0f077df2da1cf5]*/ { PyObject *result; int code; @@ -4623,7 +4630,6 @@ _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) #ifdef HAVE_CURSES_RESIZE_TERM /*[clinic input] -@permit_long_docstring_body _curses.resize_term nlines: short @@ -4635,15 +4641,16 @@ _curses.resize_term Backend function used by resizeterm(), performing most of the work. When resizing the windows, resize_term() blank-fills the areas that are -extended. The calling application should fill in these areas with appropriate -data. The resize_term() function attempts to resize all windows. However, -due to the calling convention of pads, it is not possible to resize these -without additional interaction with the application. +extended. The calling application should fill in these areas with +appropriate data. The resize_term() function attempts to resize all +windows. However, due to the calling convention of pads, it is not +possible to resize these without additional interaction with the +application. [clinic start generated code]*/ static PyObject * _curses_resize_term_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=46c6d749fa291dbd input=ebfa840f6b5f03fa]*/ +/*[clinic end generated code: output=46c6d749fa291dbd input=ff4baaf2320c8ac9]*/ { PyObject *result; int code; @@ -4701,21 +4708,22 @@ _curses_setsyx_impl(PyObject *module, int y, int x) #endif /*[clinic input] +@permit_long_summary _curses.start_color Initializes eight basic colors and global variables COLORS and COLOR_PAIRS. -Must be called if the programmer wants to use colors, and before any other -color manipulation routine is called. It is good practice to call this -routine right after initscr(). +Must be called if the programmer wants to use colors, and before any +other color manipulation routine is called. It is good practice to call +this routine right after initscr(). -It also restores the colors on the terminal to the values they had when the -terminal was just turned on. +It also restores the colors on the terminal to the values they had when +the terminal was just turned on. [clinic start generated code]*/ static PyObject * _curses_start_color_impl(PyObject *module) -/*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ +/*[clinic end generated code: output=8b772b41d8090ede input=7daacc6b6baba643]*/ { PyCursesStatefulInitialised(module); @@ -4804,13 +4812,13 @@ _curses.tigetnum Return the value of the numeric capability. -The value -2 is returned if capname is not a numeric capability, or -1 if -it is canceled or absent from the terminal description. +The value -2 is returned if capname is not a numeric capability, or -1 +if it is canceled or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ +/*[clinic end generated code: output=46f8b0a1b5dff42f input=87a64beec16ae077]*/ { PyCursesStatefulSetupTermCalled(module); @@ -4826,13 +4834,13 @@ _curses.tigetstr Return the value of the string capability. -None is returned if capname is not a string capability, or is canceled or -absent from the terminal description. +None is returned if capname is not a string capability, or is canceled +or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ +/*[clinic end generated code: output=f22b576ad60248f3 input=00bf0feda2207724]*/ { PyCursesStatefulSetupTermCalled(module); @@ -5030,7 +5038,6 @@ _curses_unget_wch(PyObject *module, PyObject *ch) #ifdef HAVE_CURSES_USE_ENV /*[clinic input] -@permit_long_docstring_body _curses.use_env flag: bool @@ -5038,19 +5045,19 @@ _curses.use_env Use environment variables LINES and COLUMNS. -If used, this function should be called before initscr() or newterm() are -called. +If used, this function should be called before initscr() or newterm() +are called. -When flag is False, the values of lines and columns specified in the terminfo -database will be used, even if environment variables LINES and COLUMNS (used -by default) are set, or if curses is running in a window (in which case -default behavior would be to use the window size if LINES and COLUMNS are -not set). +When flag is False, the values of lines and columns specified in the +terminfo database will be used, even if environment variables LINES and +COLUMNS (used by default) are set, or if curses is running in a window +(in which case default behavior would be to use the window size if LINES +and COLUMNS are not set). [clinic start generated code]*/ static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=eaa9047ec73c27d3]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=8e8feed746cf7fc1]*/ { use_env(flag); Py_RETURN_NONE; @@ -5078,6 +5085,7 @@ _curses_use_default_colors_impl(PyObject *module) } /*[clinic input] +@permit_long_summary _curses.assume_default_colors fg: int bg: int @@ -5093,7 +5101,7 @@ Use this to support transparency in your application. static PyObject * _curses_assume_default_colors_impl(PyObject *module, int fg, int bg) -/*[clinic end generated code: output=54985397a7d2b3a5 input=7fe301712ef3e9fb]*/ +/*[clinic end generated code: output=54985397a7d2b3a5 input=8945333c09893cf2]*/ { int code; @@ -5162,6 +5170,7 @@ make_ncurses_version(PyTypeObject *type) #endif /* NCURSES_VERSION */ /*[clinic input] +@permit_long_summary _curses.has_extended_color_support Return True if the module supports extended colors; otherwise, return False. @@ -5172,7 +5181,7 @@ that support more than 16 colors (e.g. xterm-256color). static PyObject * _curses_has_extended_color_support_impl(PyObject *module) -/*[clinic end generated code: output=68f1be2b57d92e22 input=4b905f046e35ee9f]*/ +/*[clinic end generated code: output=68f1be2b57d92e22 input=40d673471c5056f0]*/ { return PyBool_FromLong(_NCURSES_EXTENDED_COLOR_FUNCS); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 163e499d957b2e2..59af7afcfcc644e 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3325,7 +3325,6 @@ datetime_date_today_impl(PyTypeObject *type) } /*[clinic input] -@permit_long_docstring_body @classmethod datetime.date.fromtimestamp @@ -3334,13 +3333,13 @@ datetime.date.fromtimestamp Create a date from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) -/*[clinic end generated code: output=59def4e32c028fb6 input=55ff6940f0a8339f]*/ +/*[clinic end generated code: output=59def4e32c028fb6 input=15720eef43b169a1]*/ { return date_fromtimestamp(type, timestamp); } @@ -3476,6 +3475,7 @@ datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, } /*[clinic input] +@permit_long_summary @classmethod datetime.date.strptime @@ -3492,7 +3492,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, PyObject *format) -/*[clinic end generated code: output=454d473bee2d5161 input=31d57bb789433e99]*/ +/*[clinic end generated code: output=454d473bee2d5161 input=2db8f0b2b5242deb]*/ { PyObject *result; @@ -4744,6 +4744,7 @@ datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, } /*[clinic input] +@permit_long_summary @classmethod datetime.time.strptime @@ -4760,7 +4761,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, PyObject *format) -/*[clinic end generated code: output=ae05a9bc0241d3bf input=82ba425ecacc54aa]*/ +/*[clinic end generated code: output=ae05a9bc0241d3bf input=f01d0b9eb5383da5]*/ { PyObject *result; @@ -4856,8 +4857,8 @@ datetime.time.isoformat Return the time formatted according to ISO. -The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional -part is omitted if self.microsecond == 0. +The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the +fractional part is omitted if self.microsecond == 0. The optional argument timespec specifies the number of additional terms of the time to include. Valid options are 'auto', 'hours', @@ -4866,7 +4867,7 @@ terms of the time to include. Valid options are 'auto', 'hours', static PyObject * datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) -/*[clinic end generated code: output=2bcc7cab65c35545 input=afbbbd953d10ad07]*/ +/*[clinic end generated code: output=2bcc7cab65c35545 input=0efae103081060f4]*/ { char buf[100]; @@ -4927,14 +4928,14 @@ datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) } /*[clinic input] -@permit_long_docstring_body datetime.time.strftime format: unicode Format using strftime(). -The date part of the timestamp passed to underlying strftime should not be used. +The date part of the timestamp passed to underlying strftime should +not be used. For a list of supported format codes, see the documentation: https://docs.python.org/3/library/datetime.html#format-codes @@ -4942,7 +4943,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format) -/*[clinic end generated code: output=10f65af20e2a78c7 input=c4a5bbecd798654b]*/ +/*[clinic end generated code: output=10f65af20e2a78c7 input=184e1c0d7d356c5d]*/ { PyObject *result; PyObject *tuple; @@ -5510,15 +5511,15 @@ datetime.datetime.__new__ A combination of a date and a time. -The year, month and day arguments are required. tzinfo may be None, or an -instance of a tzinfo subclass. The remaining arguments may be ints. +The year, month and day arguments are required. tzinfo may be None, or +an instance of a tzinfo subclass. The remaining arguments may be ints. [clinic start generated code]*/ static PyObject * datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, int hour, int minute, int second, int microsecond, PyObject *tzinfo, int fold) -/*[clinic end generated code: output=47983ddb47d36037 input=2af468d7a9c1e568]*/ +/*[clinic end generated code: output=47983ddb47d36037 input=c7fd85dcf6fe9691]*/ { return new_datetime_ex2(year, month, day, hour, minute, second, microsecond, @@ -5735,7 +5736,6 @@ datetime_datetime_utcnow_impl(PyTypeObject *type) } /*[clinic input] -@permit_long_docstring_body @classmethod datetime.datetime.fromtimestamp @@ -5744,14 +5744,14 @@ datetime.datetime.fromtimestamp Create a datetime from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, PyObject *tzinfo) -/*[clinic end generated code: output=9c47ea2b2ebdaded input=d6b5b2095c5a34b2]*/ +/*[clinic end generated code: output=9c47ea2b2ebdaded input=7a2bc81a049ea287]*/ { PyObject *self; if (check_tzinfo_subclass(tzinfo) < 0) diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 6b07ef74cfa51db..a9f4f27d9eb742e 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -431,13 +431,14 @@ _dbm.dbm.setdefault Return the value for key if present, otherwise default. -If key is not in the database, it is inserted with default as the value. +If key is not in the database, it is inserted with default as the +value. [clinic start generated code]*/ static PyObject * _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=9c2f6ea6d0fb576c input=c01510ef7571e13b]*/ +/*[clinic end generated code: output=9c2f6ea6d0fb576c input=81224965c110f830]*/ { datum dbm_key, val; Py_ssize_t tmp_size; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0a8308d9ebce7a7..2760792a3fe18ed 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -939,13 +939,13 @@ _decimal.Context.Etop Return a value equal to Emax - prec + 1. -This is the maximum exponent if the _clamp field of the context is set -to 1 (IEEE clamp mode). Etop() must not be negative. +This is the maximum exponent if the _clamp field of the context is +set to 1 (IEEE clamp mode). Etop() must not be negative. [clinic start generated code]*/ static PyObject * _decimal_Context_Etop_impl(PyObject *self) -/*[clinic end generated code: output=f0a3f6e1b829074e input=838a4409316ec728]*/ +/*[clinic end generated code: output=f0a3f6e1b829074e input=35b9defc69d5e5d1]*/ { return PyLong_FromSsize_t(mpd_etop(CTX(self))); } @@ -2997,6 +2997,7 @@ PyDecType_FromSequenceExact(PyTypeObject *type, PyObject *v, PyDecType_FromSequenceExact((st)->PyDec_Type, sequence, context) /*[clinic input] +@permit_long_docstring_body @classmethod _decimal.Decimal.from_float @@ -3022,7 +3023,7 @@ Decimal.from_float(0.1) is not the same as Decimal('0.1'). static PyObject * _decimal_Decimal_from_float_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *pyfloat) -/*[clinic end generated code: output=fcb7d55d2f9dc790 input=03bc8dbe963e52ca]*/ +/*[clinic end generated code: output=fcb7d55d2f9dc790 input=29abf05dd8fe79e4]*/ { PyObject *context; PyObject *result; @@ -3068,6 +3069,8 @@ PyDecType_FromNumberExact(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @classmethod _decimal.Decimal.from_number @@ -3088,7 +3091,7 @@ Class method that converts a real number to a decimal number, exactly. static PyObject * _decimal_Decimal_from_number_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *number) -/*[clinic end generated code: output=4d3ec722b7acfd8b input=271cb4feb3148804]*/ +/*[clinic end generated code: output=4d3ec722b7acfd8b input=34ff3696955d3def]*/ { PyObject *context; PyObject *result; @@ -3959,6 +3962,7 @@ dec_as_long(PyObject *dec, PyObject *context, int round) } /*[clinic input] +@permit_long_summary _decimal.Decimal.as_integer_ratio cls: defining_class @@ -3971,7 +3975,7 @@ Raise OverflowError on infinities and a ValueError on NaNs. static PyObject * _decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=eb49c512701f844b input=07e33d8852184761]*/ +/*[clinic end generated code: output=eb49c512701f844b input=136f1dc585ca8d80]*/ { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -4146,17 +4150,17 @@ _decimal.Decimal.to_integral_exact = _decimal.Decimal.to_integral_value Round to the nearest integer. -Decimal.to_integral_exact() signals Inexact or Rounded as appropriate -if rounding occurs. The rounding mode is determined by the rounding -parameter if given, else by the given context. If neither parameter is -given, then the rounding mode of the current default context is used. +This method signals Inexact or Rounded as appropriate if rounding +occurs. The rounding mode is determined by the rounding parameter +if given, else by the given context. If neither parameter is given, +then the rounding mode of the current default context is used. [clinic start generated code]*/ static PyObject * _decimal_Decimal_to_integral_exact_impl(PyObject *self, PyTypeObject *cls, PyObject *rounding, PyObject *context) -/*[clinic end generated code: output=543a39a02eea9917 input=fabce7a744b8087c]*/ +/*[clinic end generated code: output=543a39a02eea9917 input=d4d8abe543393de1]*/ { PyObject *result; uint32_t status = 0; @@ -4791,13 +4795,14 @@ _decimal.Decimal.sqrt = _decimal.Decimal.exp Return the square root of the argument to full precision. -The result is correctly rounded using the ROUND_HALF_EVEN rounding mode. +The result is correctly rounded using the ROUND_HALF_EVEN rounding +mode. [clinic start generated code]*/ static PyObject * _decimal_Decimal_sqrt_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=deb1280077b5e586 input=3a76afbd39dc20b9]*/ +/*[clinic end generated code: output=deb1280077b5e586 input=c565a7216e9605e7]*/ Dec_UnaryFuncVA(mpd_qsqrt) /* Binary arithmetic functions, optional context arg */ @@ -4853,6 +4858,7 @@ _decimal_Decimal_max_impl(PyObject *self, PyTypeObject *cls, PyObject *other, Dec_BinaryFuncVA(mpd_qmax) /*[clinic input] +@permit_long_summary _decimal.Decimal.max_mag = _decimal.Decimal.compare As the max() method, but compares the absolute values of the operands. @@ -4861,7 +4867,7 @@ As the max() method, but compares the absolute values of the operands. static PyObject * _decimal_Decimal_max_mag_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=f71f2c27d9bc7cac input=88b105e66cf138c5]*/ +/*[clinic end generated code: output=f71f2c27d9bc7cac input=5f81b9da49b45e5d]*/ Dec_BinaryFuncVA(mpd_qmax_mag) /*[clinic input] @@ -4880,6 +4886,7 @@ _decimal_Decimal_min_impl(PyObject *self, PyTypeObject *cls, PyObject *other, Dec_BinaryFuncVA(mpd_qmin) /*[clinic input] +@permit_long_summary _decimal.Decimal.min_mag = _decimal.Decimal.compare As the min() method, but compares the absolute values of the operands. @@ -4888,7 +4895,7 @@ As the min() method, but compares the absolute values of the operands. static PyObject * _decimal_Decimal_min_mag_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=018562ad1c22aae3 input=351fa3c0e592746a]*/ +/*[clinic end generated code: output=018562ad1c22aae3 input=94c29817c7f16db7]*/ Dec_BinaryFuncVA(mpd_qmin_mag) /*[clinic input] @@ -4896,16 +4903,16 @@ _decimal.Decimal.next_toward = _decimal.Decimal.compare Returns the number closest to self, in the direction towards other. -If the two operands are unequal, return the number closest to the first -operand in the direction of the second operand. If both operands are -numerically equal, return a copy of the first operand with the sign set -to be the same as the sign of the second operand. +If the two operands are unequal, return the number closest to the +first operand in the direction of the second operand. If both +operands are numerically equal, return a copy of the first operand +with the sign set to be the same as the sign of the second operand. [clinic start generated code]*/ static PyObject * _decimal_Decimal_next_toward_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=71d879bca8bc1019 input=fdf0091ea6e9e416]*/ +/*[clinic end generated code: output=71d879bca8bc1019 input=adc5d453fc140341]*/ Dec_BinaryFuncVA(mpd_qnext_toward) /*[clinic input] @@ -4914,10 +4921,10 @@ _decimal.Decimal.remainder_near = _decimal.Decimal.compare Return the remainder from dividing self by other. This differs from self % other in that the sign of the remainder is -chosen so as to minimize its absolute value. More precisely, the return -value is self - n * other where n is the integer nearest to the exact -value of self / other, and if two integers are equally near then the -even one is chosen. +chosen so as to minimize its absolute value. More precisely, the +return value is self - n * other where n is the integer nearest to +the exact value of self / other, and if two integers are equally +near then the even one is chosen. If the result is zero then its sign will be the sign of self. [clinic start generated code]*/ @@ -4925,7 +4932,7 @@ If the result is zero then its sign will be the sign of self. static PyObject * _decimal_Decimal_remainder_near_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=d3fbb4985f2077fa input=eb5a8dfe3470b794]*/ +/*[clinic end generated code: output=d3fbb4985f2077fa input=dcb66d4afa0c77c3]*/ Dec_BinaryFuncVA(mpd_qrem_near) /* Ternary arithmetic functions, optional context arg */ @@ -4992,6 +4999,7 @@ _decimal_Decimal_is_infinite_impl(PyObject *self) Dec_BoolFunc(mpd_isinfinite) /*[clinic input] +@permit_long_summary _decimal.Decimal.is_nan Return True if the argument is a (quiet or signaling) NaN, else False. @@ -4999,7 +5007,7 @@ Return True if the argument is a (quiet or signaling) NaN, else False. static PyObject * _decimal_Decimal_is_nan_impl(PyObject *self) -/*[clinic end generated code: output=b704e8b49a164388 input=795e5dac85976994]*/ +/*[clinic end generated code: output=b704e8b49a164388 input=b7d8f0d59fe2332a]*/ Dec_BoolFunc(mpd_isnan) /*[clinic input] @@ -5153,13 +5161,13 @@ _decimal.Decimal.radix Return Decimal(10). -This is the radix (base) in which the Decimal class does -all its arithmetic. Included for compatibility with the specification. +This is the radix (base) in which the Decimal class does all its +arithmetic. Included for compatibility with the specification. [clinic start generated code]*/ static PyObject * _decimal_Decimal_radix_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=40a3bc7ec3d99228 input=b0d4cb9f870bbac1]*/ +/*[clinic end generated code: output=40a3bc7ec3d99228 input=d1cdbdbbbdefdec2]*/ { decimal_state *state = PyType_GetModuleState(cls); return _dec_mpd_radix(state); @@ -5250,15 +5258,15 @@ _decimal.Decimal.logb = _decimal.Decimal.exp Return the adjusted exponent of the operand as a Decimal instance. -If the operand is a zero, then Decimal('-Infinity') is returned and the -DivisionByZero condition is raised. If the operand is an infinity then -Decimal('Infinity') is returned. +If the operand is a zero, then Decimal('-Infinity') is returned and +the DivisionByZero condition is raised. If the operand is an +infinity then Decimal('Infinity') is returned. [clinic start generated code]*/ static PyObject * _decimal_Decimal_logb_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=36b0bda09e934245 input=a8df027d1b8a2b17]*/ +/*[clinic end generated code: output=36b0bda09e934245 input=eeafa6bbf8d8a013]*/ Dec_UnaryFuncVA(mpd_qlogb) /*[clinic input] @@ -5280,14 +5288,15 @@ The returned value is one of the following ten strings: * '+Normal', indicating that the operand is a positive normal number. * '+Infinity', indicating that the operand is positive infinity. - * 'NaN', indicating that the operand is a quiet NaN (Not a Number). + * 'NaN', indicating that the operand is a quiet NaN (Not a + Number). * 'sNaN', indicating that the operand is a signaling NaN. [clinic start generated code]*/ static PyObject * _decimal_Decimal_number_class_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=1ac82412e0849c52 input=447095d2677fa0ca]*/ +/*[clinic end generated code: output=1ac82412e0849c52 input=0b59852b43c521aa]*/ { const char *cp; @@ -5303,19 +5312,19 @@ _decimal.Decimal.to_eng_string = _decimal.Decimal.exp Convert to an engineering-type string. -Engineering notation has an exponent which is a multiple of 3, so there -are up to 3 digits left of the decimal place. For example, +Engineering notation has an exponent which is a multiple of 3, so +there are up to 3 digits left of the decimal place. For example, Decimal('123E+1') is converted to Decimal('1.23E+3'). -The value of context.capitals determines whether the exponent sign is -lower or upper case. Otherwise, the context does not affect the +The value of context.capitals determines whether the exponent sign +is lower or upper case. Otherwise, the context does not affect the operation. [clinic start generated code]*/ static PyObject * _decimal_Decimal_to_eng_string_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=901f128d437ae5c0 input=b2cb7e01e268e45d]*/ +/*[clinic end generated code: output=901f128d437ae5c0 input=111db4de6561f211]*/ { PyObject *result; mpd_ssize_t size; @@ -5343,31 +5352,31 @@ _decimal.Decimal.compare_total = _decimal.Decimal.compare Compare two operands using their abstract representation. -Similar to the compare() method, but the result -gives a total ordering on Decimal instances. Two Decimal instances with -the same numeric value but different representations compare unequal -in this ordering: +Similar to the compare() method, but the result gives a total +ordering on Decimal instances. Two Decimal instances with the same +numeric value but different representations compare unequal in this +ordering: >>> Decimal('12.0').compare_total(Decimal('12')) Decimal('-1') -Quiet and signaling NaNs are also included in the total ordering. The -result of this function is Decimal('0') if both operands have the same -representation, Decimal('-1') if the first operand is lower in the -total order than the second, and Decimal('1') if the first operand is -higher in the total order than the second operand. See the -specification for details of the total order. +Quiet and signaling NaNs are also included in the total ordering. +The result of this function is Decimal('0') if both operands have +the same representation, Decimal('-1') if the first operand is lower +in the total order than the second, and Decimal('1') if the first +operand is higher in the total order than the second operand. See +the specification for details of the total order. This operation is unaffected by context and is quiet: no flags are -changed and no rounding is performed. As an exception, the C version -may raise InvalidOperation if the second operand cannot be converted -exactly. +changed and no rounding is performed. As an exception, the C +version may raise InvalidOperation if the second operand cannot be +converted exactly. [clinic start generated code]*/ static PyObject * _decimal_Decimal_compare_total_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=83649010bad7815f input=6f3111ec5fdbf3c1]*/ +/*[clinic end generated code: output=83649010bad7815f input=d795bf204b9ff2a8]*/ Dec_BinaryFuncVA_NO_CTX(mpd_compare_total) /*[clinic input] @@ -5516,18 +5525,19 @@ _decimal.Decimal.rotate = _decimal.Decimal.compare Returns a rotated copy of self's digits, value-of-other times. -The second operand must be an integer in the range -precision through -precision. The absolute value of the second operand gives the number of -places to rotate. If the second operand is positive then rotation is to -the left; otherwise rotation is to the right. The coefficient of the -first operand is padded on the left with zeros to length precision if -necessary. The sign and exponent of the first operand are unchanged. +The second operand must be an integer in the range -precision +through precision. The absolute value of the second operand gives +the number of places to rotate. If the second operand is positive +then rotation is to the left; otherwise rotation is to the right. +The coefficient of the first operand is padded on the left with +zeros to length precision if necessary. The sign and exponent of +the first operand are unchanged. [clinic start generated code]*/ static PyObject * _decimal_Decimal_rotate_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=09f2737082882b83 input=cde7b032eac43f0b]*/ +/*[clinic end generated code: output=09f2737082882b83 input=4bc840d51842934c]*/ Dec_BinaryFuncVA(mpd_qrotate) /*[clinic input] @@ -5550,18 +5560,18 @@ _decimal.Decimal.shift = _decimal.Decimal.compare Returns a shifted copy of self's digits, value-of-other times. -The second operand must be an integer in the range -precision through -precision. The absolute value of the second operand gives the number -of places to shift. If the second operand is positive, then the shift -is to the left; otherwise the shift is to the right. Digits shifted -into the coefficient are zeros. The sign and exponent of the first -operand are unchanged. +The second operand must be an integer in the range -precision +through precision. The absolute value of the second operand gives +the number of places to shift. If the second operand is positive, +then the shift is to the left; otherwise the shift is to the right. +Digits shifted into the coefficient are zeros. The sign and +exponent of the first operand are unchanged. [clinic start generated code]*/ static PyObject * _decimal_Decimal_shift_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=82e061a0d9ecc4f5 input=501759c2522cb78e]*/ +/*[clinic end generated code: output=82e061a0d9ecc4f5 input=c05f3fd69fc1f9f9]*/ Dec_BinaryFuncVA(mpd_qshift) /*[clinic input] @@ -5589,18 +5599,18 @@ that of the right-hand operand. Also unlike other operations, quantize never signals Underflow, even if the result is subnormal and inexact. -If the exponent of the second operand is larger than that of the first, -then rounding may be necessary. In this case, the rounding mode is -determined by the rounding argument if given, else by the given context -argument; if neither argument is given, the rounding mode of the -current thread's context is used. +If the exponent of the second operand is larger than that of the +first, then rounding may be necessary. In this case, the rounding +mode is determined by the rounding argument if given, else by the +given context argument; if neither argument is given, the rounding +mode of the current thread's context is used. [clinic start generated code]*/ static PyObject * _decimal_Decimal_quantize_impl(PyObject *self, PyTypeObject *cls, PyObject *w, PyObject *rounding, PyObject *context) -/*[clinic end generated code: output=fc51edf458559913 input=1166e6311e047b74]*/ +/*[clinic end generated code: output=fc51edf458559913 input=7838b0a5f684adb8]*/ { PyObject *a, *b; PyObject *result; @@ -6629,14 +6639,14 @@ _decimal.Context.remainder_near = _decimal.Context.add Return x - y * n. -Here n is the integer nearest the exact value of x / y (if the result -is 0 then its sign will be the sign of x). +Here n is the integer nearest the exact value of x / y (if the +result is 0 then its sign will be the sign of x). [clinic start generated code]*/ static PyObject * _decimal_Context_remainder_near_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=7f18c535a12cf8ac input=bafb6327bb314c5c]*/ +/*[clinic end generated code: output=7f18c535a12cf8ac input=60342558000d4be6]*/ DecCtx_BinaryFunc(mpd_qrem_near) /*[clinic input] @@ -6723,13 +6733,14 @@ restrictions hold: * all three arguments must be integral * 'b' must be nonnegative * at least one of 'a' or 'b' must be nonzero - * modulo must be nonzero and less than 10**prec in absolute value + * modulo must be nonzero and less than 10**prec in absolute + value [clinic start generated code]*/ static PyObject * _decimal_Context_power_impl(PyObject *context, PyTypeObject *cls, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=d06d40c37cdd69dc input=2a70edd03317c666]*/ +/*[clinic end generated code: output=d06d40c37cdd69dc input=178a254468ec189b]*/ { PyObject *a, *b, *c = NULL; PyObject *result; @@ -7276,6 +7287,7 @@ _decimal_Context_copy_sign_impl(PyObject *context, PyTypeObject *cls, } /*[clinic input] +@permit_long_docstring_body _decimal.Context.logical_and = _decimal.Context.add Applies the logical operation 'and' between each operand's digits. @@ -7305,7 +7317,7 @@ The operands must be both logical numbers. static PyObject * _decimal_Context_logical_and_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=009dfa08ecaa2ac8 input=bcb7d3d6ab7530de]*/ +/*[clinic end generated code: output=009dfa08ecaa2ac8 input=9f8a93a31b9d7088]*/ DecCtx_BinaryFunc(mpd_qand) /*[clinic input] @@ -7342,6 +7354,7 @@ _decimal_Context_logical_or_impl(PyObject *context, PyTypeObject *cls, DecCtx_BinaryFunc(mpd_qor) /*[clinic input] +@permit_long_docstring_body _decimal.Context.logical_xor = _decimal.Context.add Applies the logical operation 'xor' between each operand's digits. @@ -7371,7 +7384,7 @@ The operands must be both logical numbers. static PyObject * _decimal_Context_logical_xor_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=23cd81fdcd865d5a input=fcaaf828c1d2d089]*/ +/*[clinic end generated code: output=23cd81fdcd865d5a input=119412854ae58440]*/ DecCtx_BinaryFunc(mpd_qxor) /*[clinic input] diff --git a/Modules/_decimal/clinic/_decimal.c.h b/Modules/_decimal/clinic/_decimal.c.h index b09200845d12e98..c803006ad443825 100644 --- a/Modules/_decimal/clinic/_decimal.c.h +++ b/Modules/_decimal/clinic/_decimal.c.h @@ -36,8 +36,8 @@ PyDoc_STRVAR(_decimal_Context_Etop__doc__, "\n" "Return a value equal to Emax - prec + 1.\n" "\n" -"This is the maximum exponent if the _clamp field of the context is set\n" -"to 1 (IEEE clamp mode). Etop() must not be negative."); +"This is the maximum exponent if the _clamp field of the context is\n" +"set to 1 (IEEE clamp mode). Etop() must not be negative."); #define _DECIMAL_CONTEXT_ETOP_METHODDEF \ {"Etop", (PyCFunction)_decimal_Context_Etop, METH_NOARGS, _decimal_Context_Etop__doc__}, @@ -1092,10 +1092,10 @@ PyDoc_STRVAR(_decimal_Decimal_to_integral_exact__doc__, "\n" "Round to the nearest integer.\n" "\n" -"Decimal.to_integral_exact() signals Inexact or Rounded as appropriate\n" -"if rounding occurs. The rounding mode is determined by the rounding\n" -"parameter if given, else by the given context. If neither parameter is\n" -"given, then the rounding mode of the current default context is used."); +"This method signals Inexact or Rounded as appropriate if rounding\n" +"occurs. The rounding mode is determined by the rounding parameter\n" +"if given, else by the given context. If neither parameter is given,\n" +"then the rounding mode of the current default context is used."); #define _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF \ {"to_integral_exact", _PyCFunction_CAST(_decimal_Decimal_to_integral_exact), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_exact__doc__}, @@ -1638,7 +1638,8 @@ PyDoc_STRVAR(_decimal_Decimal_sqrt__doc__, "\n" "Return the square root of the argument to full precision.\n" "\n" -"The result is correctly rounded using the ROUND_HALF_EVEN rounding mode."); +"The result is correctly rounded using the ROUND_HALF_EVEN rounding\n" +"mode."); #define _DECIMAL_DECIMAL_SQRT_METHODDEF \ {"sqrt", _PyCFunction_CAST(_decimal_Decimal_sqrt), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_sqrt__doc__}, @@ -2113,10 +2114,10 @@ PyDoc_STRVAR(_decimal_Decimal_next_toward__doc__, "\n" "Returns the number closest to self, in the direction towards other.\n" "\n" -"If the two operands are unequal, return the number closest to the first\n" -"operand in the direction of the second operand. If both operands are\n" -"numerically equal, return a copy of the first operand with the sign set\n" -"to be the same as the sign of the second operand."); +"If the two operands are unequal, return the number closest to the\n" +"first operand in the direction of the second operand. If both\n" +"operands are numerically equal, return a copy of the first operand\n" +"with the sign set to be the same as the sign of the second operand."); #define _DECIMAL_DECIMAL_NEXT_TOWARD_METHODDEF \ {"next_toward", _PyCFunction_CAST(_decimal_Decimal_next_toward), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_next_toward__doc__}, @@ -2185,10 +2186,10 @@ PyDoc_STRVAR(_decimal_Decimal_remainder_near__doc__, "Return the remainder from dividing self by other.\n" "\n" "This differs from self % other in that the sign of the remainder is\n" -"chosen so as to minimize its absolute value. More precisely, the return\n" -"value is self - n * other where n is the integer nearest to the exact\n" -"value of self / other, and if two integers are equally near then the\n" -"even one is chosen.\n" +"chosen so as to minimize its absolute value. More precisely, the\n" +"return value is self - n * other where n is the integer nearest to\n" +"the exact value of self / other, and if two integers are equally\n" +"near then the even one is chosen.\n" "\n" "If the result is zero then its sign will be the sign of self."); @@ -2671,8 +2672,8 @@ PyDoc_STRVAR(_decimal_Decimal_radix__doc__, "\n" "Return Decimal(10).\n" "\n" -"This is the radix (base) in which the Decimal class does\n" -"all its arithmetic. Included for compatibility with the specification."); +"This is the radix (base) in which the Decimal class does all its\n" +"arithmetic. Included for compatibility with the specification."); #define _DECIMAL_DECIMAL_RADIX_METHODDEF \ {"radix", _PyCFunction_CAST(_decimal_Decimal_radix), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_radix__doc__}, @@ -2812,9 +2813,9 @@ PyDoc_STRVAR(_decimal_Decimal_logb__doc__, "\n" "Return the adjusted exponent of the operand as a Decimal instance.\n" "\n" -"If the operand is a zero, then Decimal(\'-Infinity\') is returned and the\n" -"DivisionByZero condition is raised. If the operand is an infinity then\n" -"Decimal(\'Infinity\') is returned."); +"If the operand is a zero, then Decimal(\'-Infinity\') is returned and\n" +"the DivisionByZero condition is raised. If the operand is an\n" +"infinity then Decimal(\'Infinity\') is returned."); #define _DECIMAL_DECIMAL_LOGB_METHODDEF \ {"logb", _PyCFunction_CAST(_decimal_Decimal_logb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logb__doc__}, @@ -2894,7 +2895,8 @@ PyDoc_STRVAR(_decimal_Decimal_number_class__doc__, " * \'+Normal\', indicating that the operand is a positive normal\n" " number.\n" " * \'+Infinity\', indicating that the operand is positive infinity.\n" -" * \'NaN\', indicating that the operand is a quiet NaN (Not a Number).\n" +" * \'NaN\', indicating that the operand is a quiet NaN (Not a\n" +" Number).\n" " * \'sNaN\', indicating that the operand is a signaling NaN."); #define _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF \ @@ -2961,12 +2963,12 @@ PyDoc_STRVAR(_decimal_Decimal_to_eng_string__doc__, "\n" "Convert to an engineering-type string.\n" "\n" -"Engineering notation has an exponent which is a multiple of 3, so there\n" -"are up to 3 digits left of the decimal place. For example,\n" +"Engineering notation has an exponent which is a multiple of 3, so\n" +"there are up to 3 digits left of the decimal place. For example,\n" "Decimal(\'123E+1\') is converted to Decimal(\'1.23E+3\').\n" "\n" -"The value of context.capitals determines whether the exponent sign is\n" -"lower or upper case. Otherwise, the context does not affect the\n" +"The value of context.capitals determines whether the exponent sign\n" +"is lower or upper case. Otherwise, the context does not affect the\n" "operation."); #define _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF \ @@ -3033,25 +3035,25 @@ PyDoc_STRVAR(_decimal_Decimal_compare_total__doc__, "\n" "Compare two operands using their abstract representation.\n" "\n" -"Similar to the compare() method, but the result\n" -"gives a total ordering on Decimal instances. Two Decimal instances with\n" -"the same numeric value but different representations compare unequal\n" -"in this ordering:\n" +"Similar to the compare() method, but the result gives a total\n" +"ordering on Decimal instances. Two Decimal instances with the same\n" +"numeric value but different representations compare unequal in this\n" +"ordering:\n" "\n" " >>> Decimal(\'12.0\').compare_total(Decimal(\'12\'))\n" " Decimal(\'-1\')\n" "\n" -"Quiet and signaling NaNs are also included in the total ordering. The\n" -"result of this function is Decimal(\'0\') if both operands have the same\n" -"representation, Decimal(\'-1\') if the first operand is lower in the\n" -"total order than the second, and Decimal(\'1\') if the first operand is\n" -"higher in the total order than the second operand. See the\n" -"specification for details of the total order.\n" +"Quiet and signaling NaNs are also included in the total ordering.\n" +"The result of this function is Decimal(\'0\') if both operands have\n" +"the same representation, Decimal(\'-1\') if the first operand is lower\n" +"in the total order than the second, and Decimal(\'1\') if the first\n" +"operand is higher in the total order than the second operand. See\n" +"the specification for details of the total order.\n" "\n" "This operation is unaffected by context and is quiet: no flags are\n" -"changed and no rounding is performed. As an exception, the C version\n" -"may raise InvalidOperation if the second operand cannot be converted\n" -"exactly."); +"changed and no rounding is performed. As an exception, the C\n" +"version may raise InvalidOperation if the second operand cannot be\n" +"converted exactly."); #define _DECIMAL_DECIMAL_COMPARE_TOTAL_METHODDEF \ {"compare_total", _PyCFunction_CAST(_decimal_Decimal_compare_total), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare_total__doc__}, @@ -3544,12 +3546,13 @@ PyDoc_STRVAR(_decimal_Decimal_rotate__doc__, "\n" "Returns a rotated copy of self\'s digits, value-of-other times.\n" "\n" -"The second operand must be an integer in the range -precision through\n" -"precision. The absolute value of the second operand gives the number of\n" -"places to rotate. If the second operand is positive then rotation is to\n" -"the left; otherwise rotation is to the right. The coefficient of the\n" -"first operand is padded on the left with zeros to length precision if\n" -"necessary. The sign and exponent of the first operand are unchanged."); +"The second operand must be an integer in the range -precision\n" +"through precision. The absolute value of the second operand gives\n" +"the number of places to rotate. If the second operand is positive\n" +"then rotation is to the left; otherwise rotation is to the right.\n" +"The coefficient of the first operand is padded on the left with\n" +"zeros to length precision if necessary. The sign and exponent of\n" +"the first operand are unchanged."); #define _DECIMAL_DECIMAL_ROTATE_METHODDEF \ {"rotate", _PyCFunction_CAST(_decimal_Decimal_rotate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_rotate__doc__}, @@ -3686,12 +3689,12 @@ PyDoc_STRVAR(_decimal_Decimal_shift__doc__, "\n" "Returns a shifted copy of self\'s digits, value-of-other times.\n" "\n" -"The second operand must be an integer in the range -precision through\n" -"precision. The absolute value of the second operand gives the number\n" -"of places to shift. If the second operand is positive, then the shift\n" -"is to the left; otherwise the shift is to the right. Digits shifted\n" -"into the coefficient are zeros. The sign and exponent of the first\n" -"operand are unchanged."); +"The second operand must be an integer in the range -precision\n" +"through precision. The absolute value of the second operand gives\n" +"the number of places to shift. If the second operand is positive,\n" +"then the shift is to the left; otherwise the shift is to the right.\n" +"Digits shifted into the coefficient are zeros. The sign and\n" +"exponent of the first operand are unchanged."); #define _DECIMAL_DECIMAL_SHIFT_METHODDEF \ {"shift", _PyCFunction_CAST(_decimal_Decimal_shift), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_shift__doc__}, @@ -3774,11 +3777,11 @@ PyDoc_STRVAR(_decimal_Decimal_quantize__doc__, "Also unlike other operations, quantize never signals Underflow, even\n" "if the result is subnormal and inexact.\n" "\n" -"If the exponent of the second operand is larger than that of the first,\n" -"then rounding may be necessary. In this case, the rounding mode is\n" -"determined by the rounding argument if given, else by the given context\n" -"argument; if neither argument is given, the rounding mode of the\n" -"current thread\'s context is used."); +"If the exponent of the second operand is larger than that of the\n" +"first, then rounding may be necessary. In this case, the rounding\n" +"mode is determined by the rounding argument if given, else by the\n" +"given context argument; if neither argument is given, the rounding\n" +"mode of the current thread\'s context is used."); #define _DECIMAL_DECIMAL_QUANTIZE_METHODDEF \ {"quantize", _PyCFunction_CAST(_decimal_Decimal_quantize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_quantize__doc__}, @@ -5197,8 +5200,8 @@ PyDoc_STRVAR(_decimal_Context_remainder_near__doc__, "\n" "Return x - y * n.\n" "\n" -"Here n is the integer nearest the exact value of x / y (if the result\n" -"is 0 then its sign will be the sign of x)."); +"Here n is the integer nearest the exact value of x / y (if the\n" +"result is 0 then its sign will be the sign of x)."); #define _DECIMAL_CONTEXT_REMAINDER_NEAR_METHODDEF \ {"remainder_near", _PyCFunction_CAST(_decimal_Context_remainder_near), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_remainder_near__doc__}, @@ -5336,7 +5339,8 @@ PyDoc_STRVAR(_decimal_Context_power__doc__, " * all three arguments must be integral\n" " * \'b\' must be nonnegative\n" " * at least one of \'a\' or \'b\' must be nonzero\n" -" * modulo must be nonzero and less than 10**prec in absolute value"); +" * modulo must be nonzero and less than 10**prec in absolute\n" +" value"); #define _DECIMAL_CONTEXT_POWER_METHODDEF \ {"power", _PyCFunction_CAST(_decimal_Context_power), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_power__doc__}, @@ -6980,4 +6984,4 @@ _decimal_Context_same_quantum(PyObject *context, PyTypeObject *cls, PyObject *co #ifndef _DECIMAL_CONTEXT_APPLY_METHODDEF #define _DECIMAL_CONTEXT_APPLY_METHODDEF #endif /* !defined(_DECIMAL_CONTEXT_APPLY_METHODDEF) */ -/*[clinic end generated code: output=b288181c82fdc9f1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0eb835634388294e input=a9049054013a1b77]*/ diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index cbd1e026df27227..f827274eeffba83 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2297,6 +2297,10 @@ elementiter_next(PyObject *op) return NULL; } if (it->gettext) { + if (elem->tag != Py_None && !PyUnicode_Check(elem->tag)) { + Py_DECREF(elem); + continue; + } text = element_get_text(elem); goto gettext; } @@ -2570,6 +2574,7 @@ treebuilder_dealloc(PyObject *self) /* helpers for handling of arbitrary element-like objects */ /*[clinic input] +@permit_long_summary _elementtree._set_factories comment_factory: object @@ -2584,7 +2589,7 @@ For internal use only. static PyObject * _elementtree__set_factories_impl(PyObject *module, PyObject *comment_factory, PyObject *pi_factory) -/*[clinic end generated code: output=813b408adee26535 input=99d17627aea7fb3b]*/ +/*[clinic end generated code: output=813b408adee26535 input=0f415cb6b821f768]*/ { elementtreestate *st = get_elementtree_state(module); PyObject *old; @@ -3735,8 +3740,12 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, PyErr_NoMemory(); return -1; } - /* expat < 2.1.0 has no XML_SetHashSalt() */ - if (EXPAT(st, SetHashSalt) != NULL) { + // Prefer 16-byte entropy, only expat >= 2.8.0. See gh-149018 + if (EXPAT(st, SetHashSalt16Bytes) != NULL) { + EXPAT(st, SetHashSalt16Bytes)(self->parser, + _Py_HashSecret.expat.hashsalt16); + } + else if (EXPAT(st, SetHashSalt) != NULL) { EXPAT(st, SetHashSalt)(self->parser, (unsigned long)_Py_HashSecret.expat.hashsalt); } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 19bdf3d47c2fad5..b4595c55d519b93 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -860,7 +860,8 @@ static PyMethodDef partial_methods[] = { {"__reduce__", partial_reduce, METH_NOARGS}, {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("partial is generic over the wrapped function's return type")}, {NULL, NULL} /* sentinel */ }; @@ -1060,19 +1061,18 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _functools.reduce function as func: object iterable as seq: object / - initial as result: object = NULL + initial as result: object(c_default="NULL") = functools._initial_missing Apply a function of two arguments cumulatively to the items of an iterable, from left to right. -This effectively reduces the iterable to a single value. If initial is present, -it is placed before the items of the iterable in the calculation, and serves as -a default when the iterable is empty. +This effectively reduces the iterable to a single value. If initial is +present, it is placed before the items of the iterable in the +calculation, and serves as a default when the iterable is empty. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates ((((1 + 2) + 3) + 4) + 5). @@ -1081,7 +1081,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=4ccfb74548ce5170]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=ff4d5c73100e72e8]*/ { PyObject *args, *it; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index faffe8d28c5b5e7..20d482021656a50 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -520,14 +520,14 @@ _gdbm.gdbm.firstkey Return the starting key for the traversal. -It's possible to loop over every key in the database using this method -and the nextkey() method. The traversal is ordered by GDBM's internal -hash values, and won't be sorted by the key values. +It's possible to loop over every key in the database using this +method and the nextkey() method. The traversal is ordered by GDBM's +internal hash values, and won't be sorted by the key values. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=139275e9c8b60827 input=aad5a7c886c542f5]*/ +/*[clinic end generated code: output=139275e9c8b60827 input=ba40f0d81eae0f35]*/ { PyObject *v; datum key; @@ -556,8 +556,8 @@ _gdbm.gdbm.nextkey Returns the key that follows key in the traversal. -The following code prints every key in the database db, without having -to create a list in memory that contains them all: +The following code prints every key in the database db, without +having to create a list in memory that contains them all: k = db.firstkey() while k is not None: @@ -568,7 +568,7 @@ to create a list in memory that contains them all: static PyObject * _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=c81a69300ef41766 input=181f1130d5bfeb1e]*/ +/*[clinic end generated code: output=c81a69300ef41766 input=78293a913b02387e]*/ { PyObject *v; datum dbm_key, nextkey; @@ -599,14 +599,14 @@ Reorganize the database. If you have carried out a lot of deletions and would like to shrink the space used by the GDBM file, this routine will reorganize the -database. GDBM will not shorten the length of a database file except -by using this reorganization; otherwise, deleted file space will be -kept and reused as new (key,value) pairs are added. +database. GDBM will not shorten the length of a database file +except by using this reorganization; otherwise, deleted file space +will be kept and reused as new (key,value) pairs are added. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=d77c69e8e3dd644a input=3e3ca0d2ea787861]*/ +/*[clinic end generated code: output=d77c69e8e3dd644a input=d7fcf03051c6f7cd]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index fa3eceb74d16943..f895c9037485c43 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2369,18 +2369,17 @@ _hashlib_HMAC_digest_impl(HMACobject *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _hashlib.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. [clinic start generated code]*/ static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self) -/*[clinic end generated code: output=80d825be1eaae6a7 input=5e48db83ab1a4d19]*/ +/*[clinic end generated code: output=80d825be1eaae6a7 input=e37a84c36a43787c]*/ { unsigned char buf[EVP_MAX_MD_SIZE]; Py_ssize_t n = _hmac_digest(self, buf); @@ -2540,8 +2539,8 @@ _hashlib.get_fips_mode -> int Determine the OpenSSL FIPS mode of operation. For OpenSSL 3.0.0 and newer it returns the state of the default provider -in the default OSSL context. It's not quite the same as FIPS_mode() but good -enough for unittests. +in the default OSSL context. It's not quite the same as FIPS_mode() but +good enough for unittests. Effectively any non-zero return value indicates FIPS mode; values other than 1 may have additional significance. @@ -2549,7 +2548,7 @@ values other than 1 may have additional significance. static int _hashlib_get_fips_mode_impl(PyObject *module) -/*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/ +/*[clinic end generated code: output=87eece1bab4d3fa9 input=a6cdb6901421d122]*/ { #ifdef Py_HAS_OPENSSL3_SUPPORT diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index c705376f4edbf09..014c838694975d5 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -247,6 +247,7 @@ _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@permit_long_summary @critical_section heap _heapq.heappushpop @@ -262,7 +263,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=491178a1c7d417ba]*/ { PyObject *returnitem; int cmp; @@ -593,13 +594,13 @@ _heapq.heappushpop_max Maxheap variant of heappushpop. -The combined action runs more efficiently than heappush_max() followed by -a separate call to heappop_max(). +The combined action runs more efficiently than heappush_max() +followed by a separate call to heappop_max(). [clinic start generated code]*/ static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=24d0defa6fd6df4a]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=52030929667a4c08]*/ { PyObject *returnitem; int cmp; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 3c356cb40d2bca6..c6d107d243dda0e 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -2586,6 +2586,7 @@ static PyObject * _channelid_from_xid(_PyXIData_t *data) { struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2595,11 +2596,10 @@ _channelid_from_xid(_PyXIData_t *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 777b68547498847..9979cd3457e1014 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1363,6 +1363,7 @@ _queueobj_from_xid(_PyXIData_t *data) if (mod == NULL) { mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { + Py_DECREF(qidobj); return NULL; } } @@ -1552,12 +1553,13 @@ _interpqueues.destroy Clear and destroy the queue. -Afterward attempts to use the queue will behave as though it never existed. +Afterward attempts to use the queue will behave as though it never +existed. [clinic start generated code]*/ static PyObject * _interpqueues_destroy_impl(PyObject *module, int64_t qid) -/*[clinic end generated code: output=46b35623f080cbff input=8632bba87f81e3e9]*/ +/*[clinic end generated code: output=46b35623f080cbff input=75136ad807e28677]*/ { int err = queue_destroy(&_globals.queues, qid); if (handle_queue_error(err, module, qid)) { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 4c9be1d525d5871..e7a91ced48f1760 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1219,7 +1219,8 @@ _interpreters.run_func Execute the body of the provided function in the identified interpreter. Code objects are also supported. In both cases, closures and args -are not supported. Methods and other callables are not supported either. +are not supported. Methods and other callables are not supported +either. (See _interpreters.exec().) [clinic start generated code]*/ @@ -1227,7 +1228,7 @@ are not supported. Methods and other callables are not supported either. static PyObject * _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, PyObject *shared, int restricted) -/*[clinic end generated code: output=131f7202ca4a0c5e input=2d62bb9b9eaf4948]*/ +/*[clinic end generated code: output=131f7202ca4a0c5e input=162b29823b33d5cc]*/ { #define FUNCNAME MODULE_NAME_STR ".run_func" PyThreadState *tstate = _PyThreadState_GET(); @@ -1374,6 +1375,7 @@ _interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted) /*[clinic input] +@permit_long_summary _interpreters.get_config id: object * @@ -1384,7 +1386,7 @@ Return a representation of the config used to initialize the interpreter. static PyObject * _interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted) -/*[clinic end generated code: output=56773353b9b7224a input=59519a01c22d96d1]*/ +/*[clinic end generated code: output=56773353b9b7224a input=8272d9ea9e4fb42a]*/ { if (id == Py_None) { id = NULL; @@ -1490,19 +1492,19 @@ _interpreters_decref_impl(PyObject *module, PyObject *id, int restricted) /*[clinic input] -@permit_long_docstring_body _interpreters.capture_exception exc as exc_arg: object = None Return a snapshot of an exception. -If "exc" is None then the current exception, if any, is used (but not cleared). -The returned snapshot is the same as what _interpreters.exec() returns. +If "exc" is None then the current exception, if any, is used (but not +cleared). The returned snapshot is the same as what +_interpreters.exec() returns. [clinic start generated code]*/ static PyObject * _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) -/*[clinic end generated code: output=ef3f5393ef9c88a6 input=6c4dcb78fb722217]*/ +/*[clinic end generated code: output=ef3f5393ef9c88a6 input=4e6289f8f2a47b5b]*/ { PyObject *exc = exc_arg; if (exc == NULL || exc == Py_None) { diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 32c55f8e225ed91..03e6fbe08889d47 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -70,7 +70,6 @@ PyDoc_STRVAR(module_doc, /*[clinic input] module _io -@permit_long_docstring_body _io.open file: object mode: str = "r" @@ -86,112 +85,113 @@ Open file and return a stream. Raise OSError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to be opened or an integer file descriptor of the file to be -wrapped. (If a file descriptor is given, it is closed when the +wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) mode is an optional string that specifies the mode in which the file -is opened. It defaults to 'r' which means open for reading in text +is opened. It defaults to 'r' which means open for reading in text mode. Other common values are 'w' for writing (truncating the file if it already exists), 'x' for creating and writing to a new file, and 'a' for appending (which on some Unix systems, means that all writes append to the end of the file regardless of the current seek position). In text mode, if encoding is not specified the encoding used is platform -dependent: locale.getencoding() is called to get the current locale encoding. -(For reading and writing raw bytes use binary mode and leave encoding -unspecified.) The available modes are: +dependent: locale.getencoding() is called to get the current locale +encoding. (For reading and writing raw bytes use binary mode and leave +encoding unspecified.) The available modes are: -========= =============================================================== +========= ========================================================== Character Meaning ---------- --------------------------------------------------------------- +--------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing -'a' open for writing, appending to the end of the file if it exists +'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) -========= =============================================================== +========= ========================================================== -The default mode is 'rt' (open for reading text). For binary random +The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while -'r+b' opens the file without truncation. The 'x' mode implies 'w' and +'r+b' opens the file without truncation. The 'x' mode implies 'w' and raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, -even when the underlying operating system doesn't. Files opened in +even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as -bytes objects without any decoding. In text mode (the default, or when +bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. -Pass 0 to switch buffering off (only allowed in binary mode), 1 to select -line buffering (only usable in text mode), and an integer > 1 to indicate -the size of a fixed-size chunk buffer. When no buffering argument is -given, the default buffering policy works as follows: +Pass 0 to switch buffering off (only allowed in binary mode), 1 to +select line buffering (only usable in text mode), and an integer > 1 to +indicate the size of a fixed-size chunk buffer. When no buffering +argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above for binary files. encoding is the name of the encoding used to decode or encode the -file. This should only be used in text mode. The default encoding is +file. This should only be used in text mode. The default encoding is platform dependent, but any encoding supported by Python can be passed. See the codecs module for the list of supported encodings. errors is an optional string that specifies how encoding errors are to -be handled---this argument should not be used in binary mode. Pass +be handled---this argument should not be used in binary mode. Pass 'strict' to raise a ValueError exception if there is an encoding error (the default of None has the same effect), or pass 'ignore' to ignore -errors. (Note that ignoring encoding errors can lead to data loss.) +errors. (Note that ignoring encoding errors can lead to data loss.) See the documentation for codecs.register or run 'help(codecs.Codec)' for a list of the permitted encoding error strings. newline controls how universal newlines works (it only applies to text -mode). It can be None, '', '\n', '\r', and '\r\n'. It works as +mode). It can be None, '', '\n', '\r', and '\r\n'. It works as follows: -* On input, if newline is None, universal newlines mode is - enabled. Lines in the input can end in '\n', '\r', or '\r\n', and - these are translated into '\n' before being returned to the - caller. If it is '', universal newline mode is enabled, but line - endings are returned to the caller untranslated. If it has any of - the other legal values, input lines are only terminated by the given - string, and the line ending is returned to the caller untranslated. +* On input, if newline is None, universal newlines mode is enabled. + Lines in the input can end in '\n', '\r', or '\r\n', and these are + translated into '\n' before being returned to the caller. If it is + '', universal newline mode is enabled, but line endings are returned + to the caller untranslated. If it has any of the other legal values, + input lines are only terminated by the given string, and the line + ending is returned to the caller untranslated. * On output, if newline is None, any '\n' characters written are - translated to the system default line separator, os.linesep. If - newline is '' or '\n', no translation takes place. If newline is any + translated to the system default line separator, os.linesep. If + newline is '' or '\n', no translation takes place. If newline is any of the other legal values, any '\n' characters written are translated to the given string. If closefd is False, the underlying file descriptor will be kept open -when the file is closed. This does not work when a file name is given +when the file is closed. This does not work when a file name is given and must be True in that case. -A custom opener can be used by passing a callable as *opener*. The +A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by -calling *opener* with (*file*, *flags*). *opener* must return an open +calling *opener* with (*file*, *flags*). *opener* must return an open file descriptor (passing os.open as *opener* results in functionality similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing -are performed. When open() is used to open a file in a text mode ('w', -'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open +are performed. When open() is used to open a file in a text mode ('w', +'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open a file in a binary mode, the returned class varies: in read binary mode, it returns a BufferedReader; in write binary and append binary modes, it returns a BufferedWriter, and in read/write mode, it returns a BufferedRandom. It is also possible to use a string or bytearray as a file for both -reading and writing. For strings StringIO can be used like a file +reading and writing. For strings StringIO can be used like a file opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. [clinic start generated code]*/ @@ -200,7 +200,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=8629579a442a99e3]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=b3cefa70bef404b3]*/ { size_t i; @@ -499,21 +499,20 @@ _io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel) /*[clinic input] -@permit_long_docstring_body _io.open_code path : unicode Opens the provided file with the intent to import the contents. -This may perform extra validation beyond open(), but is otherwise interchangeable -with calling open(path, 'rb'). +This may perform extra validation beyond open(), but is otherwise +interchangeable with calling open(path, 'rb'). [clinic start generated code]*/ static PyObject * _io_open_code_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=53d38a37d780d034]*/ +/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=2803c35aeb63c719]*/ { return PyFile_OpenCodeObject(path); } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0fdae7b2d210040..5537947f6a51c11 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1509,7 +1509,9 @@ buffered_iternext(PyObject *op) tp == state->PyBufferedRandom_Type) { /* Skip method call overhead for speed */ + Py_BEGIN_CRITICAL_SECTION(self); line = _buffered_readline(self, -1); + Py_END_CRITICAL_SECTION(); } else { line = PyObject_CallMethodNoArgs((PyObject *)self, diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d088bb0efac797a..8cdcbd0d89c718e 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -488,13 +488,13 @@ _io.BytesIO.read1 Read at most size bytes, returned as a bytes object. -If the size argument is negative or omitted, read until EOF is reached. -Return an empty bytes object at EOF. +If the size argument is negative or omitted, read until EOF is +reached. Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=a08fc9e507ab380c]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=796ff4e0efccc4d9]*/ { return _io_BytesIO_read_impl(self, size); } @@ -792,13 +792,13 @@ _io.BytesIO.writelines Write lines to the file. Note that newlines are not added. lines can be any iterable object -producing bytes-like objects. This is equivalent to calling write() for -each element. +producing bytes-like objects. This is equivalent to calling write() +for each element. [clinic start generated code]*/ static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=5d6a616ae39dc9ca]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=d265f76533b058e7]*/ { PyObject *it, *item; diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 90b80af3018fb0f..f03638064385e28 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -18,112 +18,113 @@ PyDoc_STRVAR(_io_open__doc__, "file is either a text or byte string giving the name (and the path\n" "if the file isn\'t in the current working directory) of the file to\n" "be opened or an integer file descriptor of the file to be\n" -"wrapped. (If a file descriptor is given, it is closed when the\n" +"wrapped. (If a file descriptor is given, it is closed when the\n" "returned I/O object is closed, unless closefd is set to False.)\n" "\n" "mode is an optional string that specifies the mode in which the file\n" -"is opened. It defaults to \'r\' which means open for reading in text\n" +"is opened. It defaults to \'r\' which means open for reading in text\n" "mode. Other common values are \'w\' for writing (truncating the file if\n" "it already exists), \'x\' for creating and writing to a new file, and\n" "\'a\' for appending (which on some Unix systems, means that all writes\n" "append to the end of the file regardless of the current seek position).\n" "In text mode, if encoding is not specified the encoding used is platform\n" -"dependent: locale.getencoding() is called to get the current locale encoding.\n" -"(For reading and writing raw bytes use binary mode and leave encoding\n" -"unspecified.) The available modes are:\n" +"dependent: locale.getencoding() is called to get the current locale\n" +"encoding. (For reading and writing raw bytes use binary mode and leave\n" +"encoding unspecified.) The available modes are:\n" "\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "Character Meaning\n" -"--------- ---------------------------------------------------------------\n" +"--------- ----------------------------------------------------------\n" "\'r\' open for reading (default)\n" "\'w\' open for writing, truncating the file first\n" "\'x\' create a new file and open it for writing\n" -"\'a\' open for writing, appending to the end of the file if it exists\n" +"\'a\' open for writing, appending to the end of the file if it\n" +" exists\n" "\'b\' binary mode\n" "\'t\' text mode (default)\n" "\'+\' open a disk file for updating (reading and writing)\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "\n" -"The default mode is \'rt\' (open for reading text). For binary random\n" +"The default mode is \'rt\' (open for reading text). For binary random\n" "access, the mode \'w+b\' opens and truncates the file to 0 bytes, while\n" -"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" +"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" "raises an `FileExistsError` if the file already exists.\n" "\n" "Python distinguishes between files opened in binary and text modes,\n" -"even when the underlying operating system doesn\'t. Files opened in\n" +"even when the underlying operating system doesn\'t. Files opened in\n" "binary mode (appending \'b\' to the mode argument) return contents as\n" -"bytes objects without any decoding. In text mode (the default, or when\n" +"bytes objects without any decoding. In text mode (the default, or when\n" "\'t\' is appended to the mode argument), the contents of the file are\n" "returned as strings, the bytes having been first decoded using a\n" "platform-dependent encoding or using the specified encoding if given.\n" "\n" "buffering is an optional integer used to set the buffering policy.\n" -"Pass 0 to switch buffering off (only allowed in binary mode), 1 to select\n" -"line buffering (only usable in text mode), and an integer > 1 to indicate\n" -"the size of a fixed-size chunk buffer. When no buffering argument is\n" -"given, the default buffering policy works as follows:\n" +"Pass 0 to switch buffering off (only allowed in binary mode), 1 to\n" +"select line buffering (only usable in text mode), and an integer > 1 to\n" +"indicate the size of a fixed-size chunk buffer. When no buffering\n" +"argument is given, the default buffering policy works as follows:\n" "\n" "* Binary files are buffered in fixed-size chunks; the size of the buffer\n" -" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n" -" when the device block size is available.\n" -" On most systems, the buffer will typically be 128 kilobytes long.\n" +" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device\n" +" block size is available.\n" +" On most systems, the buffer will typically be 128 kilobytes long.\n" "\n" "* \"Interactive\" text files (files for which isatty() returns True)\n" " use line buffering. Other text files use the policy described above\n" " for binary files.\n" "\n" "encoding is the name of the encoding used to decode or encode the\n" -"file. This should only be used in text mode. The default encoding is\n" +"file. This should only be used in text mode. The default encoding is\n" "platform dependent, but any encoding supported by Python can be\n" "passed. See the codecs module for the list of supported encodings.\n" "\n" "errors is an optional string that specifies how encoding errors are to\n" -"be handled---this argument should not be used in binary mode. Pass\n" +"be handled---this argument should not be used in binary mode. Pass\n" "\'strict\' to raise a ValueError exception if there is an encoding error\n" "(the default of None has the same effect), or pass \'ignore\' to ignore\n" -"errors. (Note that ignoring encoding errors can lead to data loss.)\n" +"errors. (Note that ignoring encoding errors can lead to data loss.)\n" "See the documentation for codecs.register or run \'help(codecs.Codec)\'\n" "for a list of the permitted encoding error strings.\n" "\n" "newline controls how universal newlines works (it only applies to text\n" -"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" +"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" "follows:\n" "\n" -"* On input, if newline is None, universal newlines mode is\n" -" enabled. Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and\n" -" these are translated into \'\\n\' before being returned to the\n" -" caller. If it is \'\', universal newline mode is enabled, but line\n" -" endings are returned to the caller untranslated. If it has any of\n" -" the other legal values, input lines are only terminated by the given\n" -" string, and the line ending is returned to the caller untranslated.\n" +"* On input, if newline is None, universal newlines mode is enabled.\n" +" Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and these are\n" +" translated into \'\\n\' before being returned to the caller. If it is\n" +" \'\', universal newline mode is enabled, but line endings are returned\n" +" to the caller untranslated. If it has any of the other legal values,\n" +" input lines are only terminated by the given string, and the line\n" +" ending is returned to the caller untranslated.\n" "\n" "* On output, if newline is None, any \'\\n\' characters written are\n" -" translated to the system default line separator, os.linesep. If\n" -" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" +" translated to the system default line separator, os.linesep. If\n" +" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" " of the other legal values, any \'\\n\' characters written are translated\n" " to the given string.\n" "\n" "If closefd is False, the underlying file descriptor will be kept open\n" -"when the file is closed. This does not work when a file name is given\n" +"when the file is closed. This does not work when a file name is given\n" "and must be True in that case.\n" "\n" -"A custom opener can be used by passing a callable as *opener*. The\n" +"A custom opener can be used by passing a callable as *opener*. The\n" "underlying file descriptor for the file object is then obtained by\n" -"calling *opener* with (*file*, *flags*). *opener* must return an open\n" +"calling *opener* with (*file*, *flags*). *opener* must return an open\n" "file descriptor (passing os.open as *opener* results in functionality\n" "similar to passing None).\n" "\n" "open() returns a file object whose type depends on the mode, and\n" "through which the standard file operations such as reading and writing\n" -"are performed. When open() is used to open a file in a text mode (\'w\',\n" -"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" +"are performed. When open() is used to open a file in a text mode (\'w\',\n" +"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" "a file in a binary mode, the returned class varies: in read binary\n" "mode, it returns a BufferedReader; in write binary and append binary\n" "modes, it returns a BufferedWriter, and in read/write mode, it returns\n" "a BufferedRandom.\n" "\n" "It is also possible to use a string or bytearray as a file for both\n" -"reading and writing. For strings StringIO can be used like a file\n" +"reading and writing. For strings StringIO can be used like a file\n" "opened in a text mode, and for bytes a BytesIO can be used like a file\n" "opened in a binary mode."); @@ -352,8 +353,8 @@ PyDoc_STRVAR(_io_open_code__doc__, "\n" "Opens the provided file with the intent to import the contents.\n" "\n" -"This may perform extra validation beyond open(), but is otherwise interchangeable\n" -"with calling open(path, \'rb\')."); +"This may perform extra validation beyond open(), but is otherwise\n" +"interchangeable with calling open(path, \'rb\')."); #define _IO_OPEN_CODE_METHODDEF \ {"open_code", _PyCFunction_CAST(_io_open_code), METH_FASTCALL|METH_KEYWORDS, _io_open_code__doc__}, @@ -410,4 +411,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=7a8e032c0424bce2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5190d11f0803bfe8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 6595dc937bbcf0f..fad11ea6c9f6cf6 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -254,8 +254,8 @@ PyDoc_STRVAR(_io_BytesIO_read1__doc__, "\n" "Read at most size bytes, returned as a bytes object.\n" "\n" -"If the size argument is negative or omitted, read until EOF is reached.\n" -"Return an empty bytes object at EOF."); +"If the size argument is negative or omitted, read until EOF is\n" +"reached. Return an empty bytes object at EOF."); #define _IO_BYTESIO_READ1_METHODDEF \ {"read1", _PyCFunction_CAST(_io_BytesIO_read1), METH_FASTCALL, _io_BytesIO_read1__doc__}, @@ -529,8 +529,8 @@ PyDoc_STRVAR(_io_BytesIO_writelines__doc__, "Write lines to the file.\n" "\n" "Note that newlines are not added. lines can be any iterable object\n" -"producing bytes-like objects. This is equivalent to calling write() for\n" -"each element."); +"producing bytes-like objects. This is equivalent to calling write()\n" +"for each element."); #define _IO_BYTESIO_WRITELINES_METHODDEF \ {"writelines", (PyCFunction)_io_BytesIO_writelines, METH_O, _io_BytesIO_writelines__doc__}, @@ -637,4 +637,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=daa81dfdae5ccc57 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eac3911e207aaf45 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 96c31ce8d6f415a..890b6bc3fac9d55 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -15,8 +15,8 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "\n" "Close the file.\n" "\n" -"A closed file cannot be used for further I/O operations. close() may be\n" -"called more than once without error."); +"A closed file cannot be used for further I/O operations. close()\n" +"may be called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, @@ -41,16 +41,19 @@ PyDoc_STRVAR(_io_FileIO___init____doc__, "Open a file.\n" "\n" "The mode can be \'r\' (default), \'w\', \'x\' or \'a\' for reading,\n" -"writing, exclusive creation or appending. The file will be created if it\n" -"doesn\'t exist when opened for writing or appending; it will be truncated\n" -"when opened for writing. A FileExistsError will be raised if it already\n" -"exists when opened for creating. Opening a file for creating implies\n" -"writing so this mode behaves in a similar way to \'w\'.Add a \'+\' to the mode\n" -"to allow simultaneous reading and writing. A custom opener can be used by\n" -"passing a callable as *opener*. The underlying file descriptor for the file\n" -"object is then obtained by calling opener with (*name*, *flags*).\n" -"*opener* must return an open file descriptor (passing os.open as *opener*\n" -"results in functionality similar to passing None)."); +"writing, exclusive creation or appending. The file will be created\n" +"if it doesn\'t exist when opened for writing or appending; it will be\n" +"truncated when opened for writing. A FileExistsError will be raised\n" +"if it already exists when opened for creating. Opening a file for\n" +"creating implies writing so this mode behaves in a similar way to\n" +"\'w\'. Add a \'+\' to the mode to allow simultaneous reading and\n" +"writing.\n" +"\n" +"A custom opener can be used by passing a callable as *opener*.\n" +"The underlying file descriptor for the file object is then obtained\n" +"by calling opener with (*name*, *flags*). *opener* must return\n" +"an open file descriptor (passing os.open as *opener* results in\n" +"functionality similar to passing None)."); static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, @@ -270,11 +273,13 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "\n" "Read all data from the file, returned as bytes.\n" "\n" -"Reads until either there is an error or read() returns size 0 (indicates EOF).\n" -"If the file is already at EOF, returns an empty bytes object.\n" +"Reads until either there is an error or read() returns size 0\n" +"(indicates EOF). If the file is already at EOF, returns an empty\n" +"bytes object.\n" "\n" -"In non-blocking mode, returns as much data as could be read before EAGAIN. If no\n" -"data is available (EAGAIN is returned before bytes are read) returns None."); +"In non-blocking mode, returns as much data as could be read before\n" +"EAGAIN. If no data is available (EAGAIN is returned before bytes\n" +"are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, @@ -298,14 +303,14 @@ PyDoc_STRVAR(_io_FileIO_read__doc__, "\n" "Read at most size bytes, returned as bytes.\n" "\n" -"If size is less than 0, read all bytes in the file making multiple read calls.\n" -"See ``FileIO.readall``.\n" +"If size is less than 0, read all bytes in the file making multiple\n" +"read calls. See ``FileIO.readall``.\n" "\n" -"Attempts to make only one system call, retrying only per PEP 475 (EINTR). This\n" -"means less data may be returned than requested.\n" +"Attempts to make only one system call, retrying only per PEP 475\n" +"(EINTR). This means less data may be returned than requested.\n" "\n" -"In non-blocking mode, returns None if no data is available. Return an empty\n" -"bytes object at EOF."); +"In non-blocking mode, returns None if no data is available. Return\n" +"an empty bytes object at EOF."); #define _IO_FILEIO_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io_FileIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_read__doc__}, @@ -358,8 +363,8 @@ PyDoc_STRVAR(_io_FileIO_write__doc__, "Write buffer b to file, return number of bytes written.\n" "\n" "Only makes one system call, so not all of the data may be written.\n" -"The number of bytes actually written is returned. In non-blocking mode,\n" -"returns None if the write would block."); +"The number of bytes actually written is returned. In non-blocking\n" +"mode, returns None if the write would block."); #define _IO_FILEIO_WRITE_METHODDEF \ {"write", _PyCFunction_CAST(_io_FileIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_write__doc__}, @@ -412,11 +417,12 @@ PyDoc_STRVAR(_io_FileIO_seek__doc__, "\n" "Move to new file position and return the file position.\n" "\n" -"Argument offset is a byte count. Optional argument whence defaults to\n" -"SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\n" -"are SEEK_CUR or 1 (move relative to current position, positive or negative),\n" -"and SEEK_END or 2 (move relative to end of file, usually negative, although\n" -"many platforms allow seeking beyond the end of a file).\n" +"Argument offset is a byte count. Optional argument whence defaults\n" +"to SEEK_SET or 0 (offset from start of file, offset should be >= 0);\n" +"other values are SEEK_CUR or 1 (move relative to current position,\n" +"positive or negative), and SEEK_END or 2 (move relative to end of\n" +"file, usually negative, although many platforms allow seeking beyond\n" +"the end of a file).\n" "\n" "Note that not all file objects are seekable."); @@ -547,4 +553,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=453d584e2e72f986 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 402448545dfc516..e4438c26431aa8e 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -19,11 +19,13 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, " whence\n" " The relative position to seek from.\n" "\n" -"The offset is interpreted relative to the position indicated by whence.\n" -"Values for whence are:\n" +"The offset is interpreted relative to the position indicated by\n" +"whence. Values for whence are:\n" "\n" -"* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n" -"* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n" +"* os.SEEK_SET or 0 -- start of stream (the default); offset should\n" +" be zero or positive\n" +"* os.SEEK_CUR or 1 -- current stream position; offset may be\n" +" negative\n" "* os.SEEK_END or 2 -- end of stream; offset is usually negative\n" "\n" "Return the new absolute position."); @@ -103,8 +105,8 @@ PyDoc_STRVAR(_io__IOBase_truncate__doc__, "\n" "Truncate file to size bytes.\n" "\n" -"File pointer is left unchanged. Size defaults to the current IO position\n" -"as reported by tell(). Return the new size."); +"File pointer is left unchanged. Size defaults to the current IO\n" +"position as reported by tell(). Return the new size."); #define _IO__IOBASE_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, @@ -443,4 +445,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=9359e74d95534bef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28c06bb6db32c096 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 83165e5f7ad08bf..d6d4afb9b63c624 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -179,7 +179,8 @@ PyDoc_STRVAR(_io_StringIO_seek__doc__, "\n" "Change stream position.\n" "\n" -"Seek to character offset pos relative to position indicated by whence:\n" +"Seek to character offset pos relative to position indicated by\n" +"whence:\n" " 0 Start of stream (the default). pos should be >= 0;\n" " 1 Current position - pos must be 0;\n" " 2 End of stream - pos must be 0.\n" @@ -550,4 +551,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=bccc25ef8e6ce9ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=730c34b2a6c0500b input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 3898a9c29824364..9407076b850cee9 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "\n" "Separate the underlying buffer from the TextIOBase and return it.\n" "\n" -"After the underlying buffer has been detached, the TextIO is in an unusable state."); +"After the underlying buffer has been detached, the TextIO is in\n" +"an unusable state."); #define _IO__TEXTIOBASE_DETACH_METHODDEF \ {"detach", _PyCFunction_CAST(_io__TextIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_detach__doc__}, @@ -40,8 +41,8 @@ PyDoc_STRVAR(_io__TextIOBase_read__doc__, "\n" "Read at most size characters from stream.\n" "\n" -"Read from underlying buffer until we have size characters or we hit EOF.\n" -"If size is negative or omitted, read until EOF."); +"Read from underlying buffer until we have size characters or we hit\n" +"EOF. If size is negative or omitted, read until EOF."); #define _IO__TEXTIOBASE_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, @@ -964,8 +965,8 @@ PyDoc_STRVAR(_io_TextIOWrapper_tell__doc__, "\n" "Return the stream position as an opaque number.\n" "\n" -"The return value of tell() can be given as input to seek(), to restore a\n" -"previous stream position."); +"The return value of tell() can be given as input to seek(), to\n" +"restore a previous stream position."); #define _IO_TEXTIOWRAPPER_TELL_METHODDEF \ {"tell", (PyCFunction)_io_TextIOWrapper_tell, METH_NOARGS, _io_TextIOWrapper_tell__doc__}, @@ -1328,4 +1329,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUS return return_value; } -/*[clinic end generated code: output=c38e6cd5ff4b7eea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f900b42090c9781c input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 7af5923b6c17473..bd8073cd0af3f64 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -46,9 +46,9 @@ PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "\n" "Open a console buffer by file descriptor.\n" "\n" -"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" -"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" -"omitted. The *opener* parameter is always ignored."); +"The mode can be \'rb\' (default), or \'wb\' for reading or writing\n" +"bytes. All other mode characters will be ignored. Mode \'b\' will be\n" +"assumed if it is omitted. The *opener* parameter is always ignored."); static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, @@ -463,4 +463,4 @@ _io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=ce50bcd905f1f213 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dfe49dd71f4f4b1d input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 5d7741fdd830a53..3aeb30dfe24a357 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -153,13 +153,13 @@ _io.FileIO.close Close the file. -A closed file cannot be used for further I/O operations. close() may be -called more than once without error. +A closed file cannot be used for further I/O operations. close() +may be called more than once without error. [clinic start generated code]*/ static PyObject * _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=b405751dc4163da3]*/ { PyObject *res; int rc; @@ -231,22 +231,25 @@ _io.FileIO.__init__ Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, -writing, exclusive creation or appending. The file will be created if it -doesn't exist when opened for writing or appending; it will be truncated -when opened for writing. A FileExistsError will be raised if it already -exists when opened for creating. Opening a file for creating implies -writing so this mode behaves in a similar way to 'w'.Add a '+' to the mode -to allow simultaneous reading and writing. A custom opener can be used by -passing a callable as *opener*. The underlying file descriptor for the file -object is then obtained by calling opener with (*name*, *flags*). -*opener* must return an open file descriptor (passing os.open as *opener* -results in functionality similar to passing None). +writing, exclusive creation or appending. The file will be created +if it doesn't exist when opened for writing or appending; it will be +truncated when opened for writing. A FileExistsError will be raised +if it already exists when opened for creating. Opening a file for +creating implies writing so this mode behaves in a similar way to +'w'. Add a '+' to the mode to allow simultaneous reading and +writing. + +A custom opener can be used by passing a callable as *opener*. +The underlying file descriptor for the file object is then obtained +by calling opener with (*name*, *flags*). *opener* must return +an open file descriptor (passing os.open as *opener* results in +functionality similar to passing None). [clinic start generated code]*/ static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=bac4efcd8f930bf3]*/ { #ifdef MS_WINDOWS wchar_t *widename = NULL; @@ -725,7 +728,6 @@ new_buffersize(fileio *self, size_t currentsize) } /*[clinic input] -@permit_long_docstring_body _io.FileIO.readall cls: defining_class @@ -733,16 +735,18 @@ _io.FileIO.readall Read all data from the file, returned as bytes. -Reads until either there is an error or read() returns size 0 (indicates EOF). -If the file is already at EOF, returns an empty bytes object. +Reads until either there is an error or read() returns size 0 +(indicates EOF). If the file is already at EOF, returns an empty +bytes object. -In non-blocking mode, returns as much data as could be read before EAGAIN. If no -data is available (EAGAIN is returned before bytes are read) returns None. +In non-blocking mode, returns as much data as could be read before +EAGAIN. If no data is available (EAGAIN is returned before bytes +are read) returns None. [clinic start generated code]*/ static PyObject * _io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ +/*[clinic end generated code: output=d546737ec895c462 input=65d05bd0169f2df5]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -850,7 +854,6 @@ _io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) } /*[clinic input] -@permit_long_docstring_body _io.FileIO.read cls: defining_class size: Py_ssize_t(accept={int, NoneType}) = -1 @@ -858,19 +861,19 @@ _io.FileIO.read Read at most size bytes, returned as bytes. -If size is less than 0, read all bytes in the file making multiple read calls. -See ``FileIO.readall``. +If size is less than 0, read all bytes in the file making multiple +read calls. See ``FileIO.readall``. -Attempts to make only one system call, retrying only per PEP 475 (EINTR). This -means less data may be returned than requested. +Attempts to make only one system call, retrying only per PEP 475 +(EINTR). This means less data may be returned than requested. -In non-blocking mode, returns None if no data is available. Return an empty -bytes object at EOF. +In non-blocking mode, returns None if no data is available. Return +an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) -/*[clinic end generated code: output=bbd749c7c224143e input=752d1ad3db8564a5]*/ +/*[clinic end generated code: output=bbd749c7c224143e input=c7baa3b440af9337]*/ { if (self->fd < 0) return err_closed(); @@ -916,13 +919,13 @@ _io.FileIO.write Write buffer b to file, return number of bytes written. Only makes one system call, so not all of the data may be written. -The number of bytes actually written is returned. In non-blocking mode, -returns None if the write would block. +The number of bytes actually written is returned. In non-blocking +mode, returns None if the write would block. [clinic start generated code]*/ static PyObject * _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b) -/*[clinic end generated code: output=927e25be80f3b77b input=2776314f043088f5]*/ +/*[clinic end generated code: output=927e25be80f3b77b input=233f1f70f9e8b09e]*/ { Py_ssize_t n; int err; @@ -1016,7 +1019,6 @@ portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_er } /*[clinic input] -@permit_long_docstring_body _io.FileIO.seek pos: object whence: int = 0 @@ -1024,18 +1026,19 @@ _io.FileIO.seek Move to new file position and return the file position. -Argument offset is a byte count. Optional argument whence defaults to -SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values -are SEEK_CUR or 1 (move relative to current position, positive or negative), -and SEEK_END or 2 (move relative to end of file, usually negative, although -many platforms allow seeking beyond the end of a file). +Argument offset is a byte count. Optional argument whence defaults +to SEEK_SET or 0 (offset from start of file, offset should be >= 0); +other values are SEEK_CUR or 1 (move relative to current position, +positive or negative), and SEEK_END or 2 (move relative to end of +file, usually negative, although many platforms allow seeking beyond +the end of a file). Note that not all file objects are seekable. [clinic start generated code]*/ static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence) -/*[clinic end generated code: output=c976acdf054e6655 input=f077c492a84c9e62]*/ +/*[clinic end generated code: output=c976acdf054e6655 input=f165a1b4f5d494ad]*/ { if (self->fd < 0) return err_closed(); @@ -1063,6 +1066,7 @@ _io_FileIO_tell_impl(fileio *self) #ifdef HAVE_FTRUNCATE /*[clinic input] +@permit_long_summary _io.FileIO.truncate cls: defining_class size as posobj: object = None @@ -1076,7 +1080,7 @@ The current file position is changed to the value of size. static PyObject * _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) -/*[clinic end generated code: output=d936732a49e8d5a2 input=c367fb45d6bb2c18]*/ +/*[clinic end generated code: output=d936732a49e8d5a2 input=8f22152bcf900ed2]*/ { Py_off_t pos; int ret; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index f036ea503b11e86..1253f124108bdbf 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -93,7 +93,6 @@ iobase_unsupported(_PyIO_State *state, const char *message) /* Positioning */ /*[clinic input] -@permit_long_docstring_body _io._IOBase.seek cls: defining_class offset: int(unused=True) @@ -104,11 +103,13 @@ _io._IOBase.seek Change the stream position to the given byte offset. -The offset is interpreted relative to the position indicated by whence. -Values for whence are: +The offset is interpreted relative to the position indicated by +whence. Values for whence are: -* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive -* os.SEEK_CUR or 1 -- current stream position; offset may be negative +* os.SEEK_SET or 0 -- start of stream (the default); offset should + be zero or positive +* os.SEEK_CUR or 1 -- current stream position; offset may be + negative * os.SEEK_END or 2 -- end of stream; offset is usually negative Return the new absolute position. @@ -117,7 +118,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=a21b5aad416ff6a9]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=22eaf07a7a0ee289]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -144,14 +145,14 @@ _io._IOBase.truncate Truncate file to size bytes. -File pointer is left unchanged. Size defaults to the current IO position -as reported by tell(). Return the new size. +File pointer is left unchanged. Size defaults to the current IO +position as reported by tell(). Return the new size. [clinic start generated code]*/ static PyObject * _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *Py_UNUSED(size)) -/*[clinic end generated code: output=2013179bff1fe8ef input=660ac20936612c27]*/ +/*[clinic end generated code: output=2013179bff1fe8ef input=5b3b6ab3c7abd806]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 5debae5b42480b8..b8601383ad0a26f 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -225,7 +225,9 @@ write_str(stringio *self, PyObject *obj) if (self->state == STATE_ACCUMULATING) { if (self->string_size == self->pos) { - if (PyUnicodeWriter_WriteStr(self->writer, decoded)) + // gh-149046: Avoid PyUnicodeWriter_WriteStr() which calls str(obj) + // on str subclasses + if (_PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)self->writer, decoded)) goto fail; goto success; } @@ -497,7 +499,8 @@ _io.StringIO.seek Change stream position. -Seek to character offset pos relative to position indicated by whence: +Seek to character offset pos relative to position indicated by +whence: 0 Start of stream (the default). pos should be >= 0; 1 Current position - pos must be 0; 2 End of stream - pos must be 0. @@ -506,7 +509,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=c75ced09343a00d7]*/ +/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=ffef24668fd71a5d]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 347bfe976619e89..e80b75066c59a61 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -53,19 +53,19 @@ _unsupported(_PyIO_State *state, const char *message) } /*[clinic input] -@permit_long_docstring_body _io._TextIOBase.detach cls: defining_class / Separate the underlying buffer from the TextIOBase and return it. -After the underlying buffer has been detached, the TextIO is in an unusable state. +After the underlying buffer has been detached, the TextIO is in +an unusable state. [clinic start generated code]*/ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=50915f40c609eaa4 input=8cd0652c17d7f015]*/ +/*[clinic end generated code: output=50915f40c609eaa4 input=8099c088abcb87d8]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); @@ -79,14 +79,14 @@ _io._TextIOBase.read Read at most size characters from stream. -Read from underlying buffer until we have size characters or we hit EOF. -If size is negative or omitted, read until EOF. +Read from underlying buffer until we have size characters or we hit +EOF. If size is negative or omitted, read until EOF. [clinic start generated code]*/ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(size)) -/*[clinic end generated code: output=51a5178a309ce647 input=f5e37720f9fc563f]*/ +/*[clinic end generated code: output=51a5178a309ce647 input=c9fd4cc1cf1b4614]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); @@ -2727,13 +2727,13 @@ _io.TextIOWrapper.tell Return the stream position as an opaque number. -The return value of tell() can be given as input to seek(), to restore a -previous stream position. +The return value of tell() can be given as input to seek(), to +restore a previous stream position. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=415d6b4e4f8e6e8c]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=aeece020f747fd92]*/ { PyObject *res; PyObject *posobj = NULL; diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 677d7e85d4e626f..4cd71094e8f459e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -315,16 +315,16 @@ _io._WindowsConsoleIO.__init__ Open a console buffer by file descriptor. -The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All -other mode characters will be ignored. Mode 'b' will be assumed if it is -omitted. The *opener* parameter is always ignored. +The mode can be 'rb' (default), or 'wb' for reading or writing +bytes. All other mode characters will be ignored. Mode 'b' will be +assumed if it is omitted. The *opener* parameter is always ignored. [clinic start generated code]*/ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=f31100e2cd724617]*/ { const char *s; wchar_t *name = NULL; @@ -673,12 +673,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; diff --git a/Modules/_json.c b/Modules/_json.c index 1f454768355cc0c..6c4f38834631d30 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -657,14 +657,14 @@ JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. -Returns a tuple of the decoded string and the index of the character in s -after the end quote. +Returns a tuple of the decoded string and the index of the character in +s after the end quote. [clinic start generated code]*/ static PyObject * py_scanstring_impl(PyObject *module, PyObject *pystr, Py_ssize_t end, int strict) -/*[clinic end generated code: output=961740cfae07cdb3 input=cff59e47498f4d8e]*/ +/*[clinic end generated code: output=961740cfae07cdb3 input=6d5abb5947ccc297]*/ { Py_ssize_t next_end = -1; PyObject *rval = scanstring_unicode(pystr, end, strict, &next_end); diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 00ee68dcea2d0d9..237aae544a847bf 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1106,7 +1106,6 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) } /*[clinic input] -@permit_long_docstring_body _lzma.LZMADecompressor.decompress data: Py_buffer @@ -1114,24 +1113,25 @@ _lzma.LZMADecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=d5cbd45801b4b8b0]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=0eb62669c4315dee]*/ { PyObject *result = NULL; diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index a545ff4d80f067d..a4d7273aea718ae 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -50,9 +50,9 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "\n" "Remove a shared memory object (similar to unlink()).\n" "\n" -"Remove a shared memory object name, and, once all processes have unmapped\n" -"the object, de-allocates and destroys the contents of the associated memory\n" -"region."); +"Remove a shared memory object name, and, once all processes have\n" +"unmapped the object, de-allocates and destroys the contents of the\n" +"associated memory region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, @@ -86,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e69afacce7b0595e input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index ab45e4136c7d46e..22b4af212662b3c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -81,15 +81,15 @@ _posixshmem.shm_unlink Remove a shared memory object (similar to unlink()). -Remove a shared memory object name, and, once all processes have unmapped -the object, de-allocates and destroys the contents of the associated memory -region. +Remove a shared memory object name, and, once all processes have +unmapped the object, de-allocates and destroys the contents of the +associated memory region. [clinic start generated code]*/ static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=cf7a30ec6503cf78]*/ { int rv; int async_err = 0; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dedf17f76dfc9b7..2a34559fd1f4378 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -119,7 +119,7 @@ _opcode_has_const_impl(PyObject *module, int opcode) } /*[clinic input] - +@permit_long_summary _opcode.has_name -> bool opcode: int @@ -129,7 +129,7 @@ Return True if the opcode accesses an attribute by name, False otherwise. static int _opcode_has_name_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ +/*[clinic end generated code: output=b49a83555c2fa517 input=8faf669024d97fad]*/ { return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 9874f9475ac0296..6219706f9031593 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2055,22 +2055,34 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * return NULL; } if (PyDict_CheckExact(modules)) { + PyObject *found_name = NULL; + int error = 0; i = 0; + Py_BEGIN_CRITICAL_SECTION(modules); while (PyDict_Next(modules, &i, &module_name, &module)) { Py_INCREF(module_name); Py_INCREF(module); if (_checkmodule(module_name, module, global, dotted_path) == 0) { Py_DECREF(module); - Py_DECREF(modules); - return module_name; + found_name = module_name; + break; } Py_DECREF(module); Py_DECREF(module_name); if (PyErr_Occurred()) { - Py_DECREF(modules); - return NULL; + error = 1; + break; } } + Py_END_CRITICAL_SECTION(); + if (error) { + Py_DECREF(modules); + return NULL; + } + if (found_name != NULL) { + Py_DECREF(modules); + return found_name; + } } else { PyObject *iterator = PyObject_GetIter(modules); @@ -3452,7 +3464,7 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +batch_dict_exact_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3525,6 +3537,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) return -1; } +/* gh-146452: Wrap the dict iteration in a critical section to prevent + concurrent mutation from invalidating PyDict_Next() iteration state. */ +static int +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(obj); + ret = batch_dict_exact_impl(state, self, obj); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b7f39ea3d499e4c..bcee56339877976 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -962,7 +962,6 @@ do_fork_exec(char *const exec_array[], } /*[clinic input] -@permit_long_docstring_body _posixsubprocess.fork_exec as subprocess_fork_exec args as process_args: object executable_list: object @@ -990,15 +989,15 @@ _posixsubprocess.fork_exec as subprocess_fork_exec Spawn a fresh new child process. -Fork a child process, close parent file descriptors as appropriate in the -child and duplicate the few that are needed before calling exec() in the -child process. +Fork a child process, close parent file descriptors as appropriate in +the child and duplicate the few that are needed before calling exec() in +the child process. -If close_fds is True, close file descriptors 3 and higher, except those listed -in the sorted tuple pass_fds. +If close_fds is True, close file descriptors 3 and higher, except those +listed in the sorted tuple pass_fds. -The preexec_fn, if supplied, will be called immediately before closing file -descriptors and exec. +The preexec_fn, if supplied, will be called immediately before closing +file descriptors and exec. WARNING: preexec_fn is NOT SAFE if your application uses threads. It may trigger infrequent, difficult to debug deadlocks. @@ -1023,7 +1022,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, PyObject *preexec_fn) -/*[clinic end generated code: output=288464dc56e373c7 input=58e0db771686f4f6]*/ +/*[clinic end generated code: output=288464dc56e373c7 input=5e56eac3e036e349]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index ed925f3525a9a7d..af54e42a6af584c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -154,8 +154,6 @@ RingBuf_Get(RingBuf *buf) } // Returns 0 on success or -1 if the buffer failed to grow. -// -// Steals a reference to item. static int RingBuf_Put(RingBuf *buf, PyObject *item) { @@ -165,11 +163,10 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); - Py_DECREF(item); return -1; } } - buf->items[buf->put_idx] = item; + buf->items[buf->put_idx] = Py_NewRef(item); buf->put_idx = (buf->put_idx + 1) % buf->items_cap; buf->num_items++; return 0; @@ -276,16 +273,13 @@ maybe_handoff_item(void *arg, void *park_arg, int has_more_waiters) { HandoffData *data = (HandoffData*)arg; PyObject **item = (PyObject**)park_arg; - if (item == NULL) { - // No threads were waiting - data->handed_off = false; - } - else { + data->queue->has_threads_waiting = has_more_waiters; + + data->handed_off = item != NULL; + if (data->handed_off) { // There was at least one waiting thread, hand off the item - *item = data->item; - data->handed_off = true; + *item = Py_NewRef(data->item); } - data->queue->has_threads_waiting = has_more_waiters; } /*[clinic input] @@ -297,31 +291,33 @@ _queue.SimpleQueue.put Put the item on the queue. -The optional 'block' and 'timeout' arguments are ignored, as this method -never blocks. They are provided for compatibility with the Queue class. +The optional 'block' and 'timeout' arguments are ignored, as this +method never blocks. They are provided for compatibility with the +Queue class. [clinic start generated code]*/ static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=9f9ff270a74670c3]*/ { - HandoffData data = { - .handed_off = 0, - .item = Py_NewRef(item), - .queue = self, - }; if (self->has_threads_waiting) { + HandoffData data = { + .handed_off = 0, + .item = item, + .queue = self, + }; // Try to hand the item off directly if there are threads waiting _PyParkingLot_Unpark(&self->has_threads_waiting, maybe_handoff_item, &data); - } - if (!data.handed_off) { - if (RingBuf_Put(&self->buf, item) < 0) { - return NULL; + if (data.handed_off) { + Py_RETURN_NONE; } } + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -365,10 +361,11 @@ _queue.SimpleQueue.get Remove and return an item from the queue. -If optional args 'block' is true and 'timeout' is None (the default), -block if necessary until an item is available. If 'timeout' is -a non-negative number, it blocks at most 'timeout' seconds and raises -the Empty exception if no item was available within that time. +If optional args 'block' is true and 'timeout' is None (the +default), block if necessary until an item is available. If +'timeout' is a non-negative number, it blocks at most 'timeout' +seconds and raises the Empty exception if no item was available +within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). @@ -378,7 +375,7 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=afa0889bbc6b4761]*/ { PyTime_t endtime = 0; @@ -553,7 +550,7 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF _QUEUE_SIMPLEQUEUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("SimpleQueues are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0fb734816517485..a06966be23be1ef 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -123,9 +123,9 @@ typedef struct { /*[clinic input] module _random -class _random.Random "RandomObject *" "_randomstate_type(type)->Random_Type" +class _random.Random "RandomObject *" "(PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=70a2c99619474983]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f04bcbfba61a322e]*/ /* Random methods */ @@ -549,27 +549,20 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) return result; } -static int -random_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *arg = NULL; - _randomstate *state = _randomstate_type(Py_TYPE(self)); - - if ((Py_IS_TYPE(self, (PyTypeObject *)state->Random_Type) || - Py_TYPE(self)->tp_init == ((PyTypeObject*)state->Random_Type)->tp_init) && - !_PyArg_NoKeywords("Random", kwds)) { - return -1; - } - - if (PyTuple_GET_SIZE(args) > 1) { - PyErr_SetString(PyExc_TypeError, "Random() requires 0 or 1 argument"); - return -1; - } +/*[clinic input] +@critical_section +@text_signature "($self, [seed])" +_random.Random.__init__ as random_init - if (PyTuple_GET_SIZE(args) == 1) - arg = PyTuple_GET_ITEM(args, 0); + seed: object = NULL + / +[clinic start generated code]*/ - return random_seed(RandomObject_CAST(self), arg); +static int +random_init_impl(RandomObject *self, PyObject *seed) +/*[clinic end generated code: output=260734a3739c394f input=e516bf32e8a05e28]*/ +{ + return random_seed(self, seed); } diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7369cd1514c296d..635e6e208902af5 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -30,6 +30,7 @@ extern "C" { #include "internal/pycore_llist.h" // struct llist_node #include "internal/pycore_long.h" // _PyLong_GetZero #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause +#include "internal/pycore_pyhash.h" // _Py_HashPointerRaw #include "internal/pycore_stackref.h" // Py_TAG_BITS #include "../../Python/remote_debug.h" @@ -179,7 +180,7 @@ typedef enum _WIN32_THREADSTATE { #define set_exception_cause(unwinder, exc_type, message) \ do { \ assert(PyErr_Occurred() && "function returned -1 without setting exception"); \ - if (unwinder->debug) { \ + if (unwinder->debug && !_Py_RemoteDebug_HasPermissionError()) { \ _set_debug_exception_cause(exc_type, message); \ } \ } while (0) @@ -215,6 +216,8 @@ typedef struct { PyObject *file_name; int first_lineno; PyObject *linetable; // bytes + PyObject *last_frame_info; + ptrdiff_t last_addrq; uintptr_t addr_code_adaptive; } CachedCodeMetadata; @@ -224,11 +227,41 @@ typedef struct { typedef struct { uint64_t thread_id; // 0 = empty slot + uintptr_t thread_state_addr; uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; Py_ssize_t num_addrs; + PyObject *thread_id_obj; // owned reference, NULL if empty PyObject *frame_list; // owned reference, NULL if empty } FrameCacheEntry; +#define INTERPRETER_THREAD_CACHE_SIZE 32 +#if (INTERPRETER_THREAD_CACHE_SIZE & (INTERPRETER_THREAD_CACHE_SIZE - 1)) != 0 +# error "INTERPRETER_THREAD_CACHE_SIZE must be a power of two" +#endif + +// The two per-interpreter L2 caches below are split into per-field tables so +// that a writer rebinding one slot cannot leave stale data in a field owned by +// the other when the slot is reused across interpreters. +typedef struct { + uintptr_t interpreter_addr; + uintptr_t thread_state_addr; +} InterpreterTstateCacheEntry; +typedef struct { + uintptr_t interpreter_addr; + uint64_t code_object_generation; +} InterpreterGenerationCacheEntry; + +// Carries already-read thread state and/or frame buffers across helpers so the +// downstream callee can skip a remote read. Address fields are caller-supplied +// inputs; buffer pointers (tstate, frame) are NULL unless a prior batched read +// successfully populated them. +typedef struct { + const char *tstate; + uintptr_t tstate_addr; + const char *frame; + uintptr_t frame_addr; +} RemoteReadPrefetch; + /* Statistics for profiling performance analysis */ typedef struct { uint64_t total_samples; // Total number of get_stack_trace calls @@ -242,14 +275,44 @@ typedef struct { uint64_t code_object_cache_hits; // Code object cache hits uint64_t code_object_cache_misses; // Code object cache misses uint64_t stale_cache_invalidations; // Times stale entries were cleared + uint64_t batched_read_attempts; // Batched remote-read attempts + uint64_t batched_read_successes; // Attempts that read all requested segments + uint64_t batched_read_misses; // Attempts that fell back or partially read + uint64_t batched_read_segments_requested; // Segments requested by batched reads + uint64_t batched_read_segments_completed; // Segments completed by batched reads } UnwinderStats; +#if defined(__GNUC__) || defined(__clang__) +# define REMOTE_DEBUG_UNLIKELY(value) __builtin_expect(!!(value), 0) +#else +# define REMOTE_DEBUG_UNLIKELY(value) (value) +#endif + /* Stats tracking macros - no-op when stats collection is disabled */ #define STATS_INC(unwinder, field) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field++; } while(0) #define STATS_ADD(unwinder, field, val) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field += (val); } while(0) + +#if HAVE_PROCESS_VM_READV +# define STATS_BATCHED_READ(unwinder, requested, completed) \ + do { \ + if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) { \ + (unwinder)->stats.batched_read_attempts++; \ + (unwinder)->stats.batched_read_segments_requested += (uint64_t)(requested); \ + (unwinder)->stats.batched_read_segments_completed += (uint64_t)(completed); \ + if ((completed) == (requested)) { \ + (unwinder)->stats.batched_read_successes++; \ + } \ + else { \ + (unwinder)->stats.batched_read_misses++; \ + } \ + } \ + } while(0) +#else +# define STATS_BATCHED_READ(unwinder, requested, completed) ((void)0) +#endif typedef struct { PyTypeObject *RemoteDebugging_Type; @@ -290,7 +353,6 @@ typedef struct { struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; uintptr_t interpreter_addr; uintptr_t tstate_addr; - uint64_t code_object_generation; _Py_hashtable_t *code_object_cache; int debug; int only_active_thread; @@ -302,9 +364,17 @@ typedef struct { int cache_frames; int collect_stats; // whether to collect statistics uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale + // L1 single-entry shortcut over cached_tstates[]: most workloads sample one + // interpreter, so check these pairs before hashing into the table below. + uintptr_t cached_tstate_interpreter_addr; + uintptr_t cached_tstate_addr; + uintptr_t cached_generation_interpreter_addr; + uint64_t cached_code_object_generation; RemoteDebuggingState *cached_state; FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries UnwinderStats stats; // statistics for performance analysis + InterpreterTstateCacheEntry cached_tstates[INTERPRETER_THREAD_CACHE_SIZE]; + InterpreterGenerationCacheEntry cached_generations[INTERPRETER_THREAD_CACHE_SIZE]; #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; @@ -361,11 +431,13 @@ typedef struct { typedef struct { /* Inputs */ uintptr_t frame_addr; // Starting frame address + uintptr_t thread_state_addr; // Owning thread state address uintptr_t base_frame_addr; // Sentinel at bottom (for validation) uintptr_t gc_frame; // GC frame address (0 if not tracking) uintptr_t last_profiled_frame; // Last cached frame (0 if no cache) StackChunkList *chunks; // Pre-copied stack chunks int skip_first_frame; // Skip frame_addr itself (continue from its caller) + RemoteReadPrefetch prefetch; // Optional already-read thread/frame buffers /* Outputs */ PyObject *frame_info; // List to append FrameInfo objects @@ -548,6 +620,7 @@ extern int process_frame_chain( extern int frame_cache_init(RemoteUnwinderObject *unwinder); extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern FrameCacheEntry *frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr); extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); extern int frame_cache_lookup_and_extend( @@ -566,6 +639,7 @@ extern int frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited); @@ -605,7 +679,8 @@ extern PyObject* unwind_stack_for_thread( uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ); /* Thread stopping functions (for blocking mode) */ diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index fc7487d4044bfb2..44a9a3cbce0061a 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -22,35 +22,38 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", NULL); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__linux__) && HAVE_PROCESS_VM_READV // On Linux, search for asyncio debug in executable or DLL address = search_linux_map_for_section(handle, "AsyncioDebug", "python", NULL); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__APPLE__) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python address = search_map_for_section(handle, "AsyncioDebug", "libpython", NULL); - if (address == 0) { + if (address == 0 && !_Py_RemoteDebug_HasPermissionError()) { PyErr_Clear(); address = search_map_for_section(handle, "AsyncioDebug", "python", NULL); } if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #else Py_UNREACHABLE(); @@ -96,10 +99,12 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) return -1; } if (result < 0) { - PyErr_Clear(); - PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); - set_exception_cause(unwinder, PyExc_RuntimeError, - "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyErr_Clear(); + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + } return -1; } @@ -218,7 +223,7 @@ parse_task_name( if ((GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_LONG_SUBCLASS)) { long res = read_py_long(unwinder, task_name_addr); - if (res == -1) { + if (res == -1 && PyErr_Occurred()) { set_exception_cause(unwinder, PyExc_RuntimeError, "Task name PyLong parsing failed"); return NULL; } diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 87a54371c774f1a..d4188335c0b6d0a 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -253,7 +253,6 @@ typedef struct { /* Main binary writer structure */ typedef struct { FILE *fp; - char *filename; /* Write buffer for batched I/O */ uint8_t *write_buffer; @@ -311,10 +310,7 @@ typedef struct { /* Main binary reader structure */ typedef struct { - char *filename; - #if USE_MMAP - int fd; uint8_t *mapped_data; size_t mapped_size; #else @@ -522,7 +518,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * Create a new binary writer. * * Arguments: - * filename: Path to output file + * path: Path to output file * sample_interval_us: Sampling interval in microseconds * compression_type: COMPRESSION_NONE or COMPRESSION_ZSTD * start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) @@ -531,7 +527,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * New BinaryWriter* on success, NULL on failure (PyErr set) */ BinaryWriter *binary_writer_create( - const char *filename, + PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us @@ -583,12 +579,12 @@ void binary_writer_destroy(BinaryWriter *writer); * Open a binary file for reading. * * Arguments: - * filename: Path to input file + * path: Path to input file * * Returns: * New BinaryReader* on success, NULL on failure (PyErr set) */ -BinaryReader *binary_reader_open(const char *filename); +BinaryReader *binary_reader_open(PyObject *path); /* * Replay samples from binary file through a collector. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 551530b519952c0..ce1c3d232c94e0f 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -358,7 +358,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ } BinaryReader * -binary_reader_open(const char *filename) +binary_reader_open(PyObject *path) { BinaryReader *reader = PyMem_Calloc(1, sizeof(BinaryReader)); if (!reader) { @@ -366,32 +366,36 @@ binary_reader_open(const char *filename) return NULL; } -#if USE_MMAP - reader->fd = -1; /* Explicit initialization for cleanup safety */ -#endif - - reader->filename = PyMem_Malloc(strlen(filename) + 1); - if (!reader->filename) { - PyMem_Free(reader); - PyErr_NoMemory(); - return NULL; - } - strcpy(reader->filename, filename); - #if USE_MMAP /* Open with mmap on Unix */ - reader->fd = open(filename, O_RDONLY); - if (reader->fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + FILE *fp = Py_fopen(path, "rb"); + if (!fp) { goto error; } + int fd = fileno(fp); struct stat st; - if (fstat(reader->fd, &st) < 0) { + if (fstat(fd, &st) < 0) { PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); + goto error; + } + if (st.st_size < 0) { + PyErr_SetString(PyExc_IOError, "Invalid negative file size"); + Py_fclose(fp); + goto error; + } + if ((uintmax_t)st.st_size > SIZE_MAX) { + PyErr_SetString(PyExc_OverflowError, "File is too large to map"); + Py_fclose(fp); goto error; } reader->mapped_size = st.st_size; + if (reader->mapped_size == 0) { + PyErr_SetString(PyExc_ValueError, "File too small for header"); + Py_fclose(fp); + goto error; + } /* Map the file into memory. * MAP_POPULATE (Linux-only) pre-faults all pages at mmap time, which: @@ -400,14 +404,15 @@ binary_reader_open(const char *filename) */ #ifdef __linux__ reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE | MAP_POPULATE, reader->fd, 0); + MAP_PRIVATE | MAP_POPULATE, fd, 0); #else reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE, reader->fd, 0); + MAP_PRIVATE, fd, 0); #endif if (reader->mapped_data == MAP_FAILED) { reader->mapped_data = NULL; PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } @@ -428,19 +433,23 @@ binary_reader_open(const char *filename) /* Add file descriptor-level hints for better kernel I/O scheduling */ #if defined(__linux__) && defined(POSIX_FADV_SEQUENTIAL) - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); if (reader->mapped_size > (64 * 1024 * 1024)) { - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_WILLNEED); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); } #endif + if (Py_fclose(fp) != 0) { + PyErr_SetFromErrno(PyExc_IOError); + goto error; + } + uint8_t *data = reader->mapped_data; size_t file_size = reader->mapped_size; #else /* Use stdio on Windows */ - reader->fp = fopen(filename, "rb"); + reader->fp = Py_fopen(path, "rb"); if (!reader->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -453,7 +462,15 @@ binary_reader_open(const char *filename) PyErr_SetFromErrno(PyExc_IOError); goto error; } + if ((uint64_t)file_size_off > SIZE_MAX) { + PyErr_SetString(PyExc_OverflowError, "File is too large to read"); + goto error; + } reader->file_size = (size_t)file_size_off; + if (reader->file_size == 0) { + PyErr_SetString(PyExc_ValueError, "File too small for header"); + goto error; + } if (FSEEK64(reader->fp, 0, SEEK_SET) != 0) { PyErr_SetFromErrno(PyExc_IOError); goto error; @@ -465,8 +482,18 @@ binary_reader_open(const char *filename) goto error; } - if (fread(reader->file_data, 1, reader->file_size, reader->fp) != reader->file_size) { - PyErr_SetFromErrno(PyExc_IOError); + size_t nread = fread(reader->file_data, 1, reader->file_size, reader->fp); + if (nread != reader->file_size) { + int err = errno; + if (ferror(reader->fp) && err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_IOError); + } + else { + PyErr_Format(PyExc_ValueError, + "Unexpected end of file: read %zu of %zu bytes", + nread, reader->file_size); + } goto error; } @@ -953,10 +980,16 @@ invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint32_t total) Py_ssize_t binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progress_callback) { - if (!PyObject_HasAttrString(collector, "collect")) { + PyObject *collect_method; + int has_collect = PyObject_GetOptionalAttrString(collector, "collect", &collect_method); + if (has_collect < 0) { + return -1; + } + if (has_collect == 0) { PyErr_SetString(PyExc_TypeError, "Collector must have a collect() method"); return -1; } + Py_DECREF(collect_method); /* Get module state for struct sequence types */ PyObject *module = PyImport_ImportModule("_remote_debugging"); @@ -982,7 +1015,10 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre while (offset < reader->sample_data_size) { /* Read thread_id (8 bytes) + interpreter_id (4 bytes) + encoding byte */ if (reader->sample_data_size - offset < SAMPLE_HEADER_FIXED_SIZE) { - break; /* End of data */ + PyErr_Format(PyExc_ValueError, + "Truncated sample data: %zu trailing bytes", + reader->sample_data_size - offset); + return -1; } /* Use memcpy to avoid strict aliasing violations, then byte-swap if needed */ @@ -1028,6 +1064,11 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre count, max_possible_samples); return -1; } + if ((uint64_t)count > (uint64_t)PY_SSIZE_T_MAX - (uint64_t)replayed) { + PyErr_SetString(PyExc_OverflowError, + "Sample count exceeds Py_ssize_t maximum"); + return -1; + } reader->stats.repeat_records++; reader->stats.repeat_samples += count; @@ -1158,6 +1199,11 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre return -1; } Py_DECREF(timestamps_list); + if (replayed == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "Sample count exceeds Py_ssize_t maximum"); + return -1; + } replayed++; reader->stats.total_samples++; break; @@ -1176,6 +1222,13 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre } } + if ((uint64_t)replayed != reader->sample_count) { + PyErr_Format(PyExc_ValueError, + "Sample count mismatch: header declares %u samples but replay decoded %zd", + reader->sample_count, replayed); + return -1; + } + /* Final progress callback at 100% */ if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { return -1; @@ -1263,8 +1316,6 @@ binary_reader_close(BinaryReader *reader) return; } - PyMem_Free(reader->filename); - #if USE_MMAP if (reader->mapped_data) { munmap(reader->mapped_data, reader->mapped_size); @@ -1274,13 +1325,9 @@ binary_reader_close(BinaryReader *reader) /* Clear sample_data which may point into the now-unmapped region */ reader->sample_data = NULL; reader->sample_data_size = 0; - if (reader->fd >= 0) { - close(reader->fd); - reader->fd = -1; /* Mark as closed */ - } #else if (reader->fp) { - fclose(reader->fp); + Py_fclose(reader->fp); reader->fp = NULL; } if (reader->file_data) { diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 4cfed7300ac5ab2..341f9f7dc8ac457 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -108,7 +108,15 @@ fwrite_checked_allow_threads(const void *data, size_t size, FILE *fp) written = fwrite(data, 1, size, fp); Py_END_ALLOW_THREADS if (written != size) { - PyErr_SetFromErrno(PyExc_IOError); + int err = errno; + if (ferror(fp) && err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_IOError); + } + else { + PyErr_Format(PyExc_IOError, + "short write: wrote %zu of %zu bytes", written, size); + } return -1; } return 0; @@ -366,6 +374,11 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) return 0; } + if (writer->string_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many strings for binary format"); + return -1; + } if (writer->string_count >= writer->string_capacity) { if (grow_parallel_arrays((void **)&writer->strings, (void **)&writer->string_lengths, @@ -380,6 +393,12 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) if (!str_data) { return -1; } + if ((uintmax_t)str_len > UINT32_MAX) { + PyErr_Format(PyExc_OverflowError, + "string length %zd exceeds binary format maximum %u", + str_len, UINT32_MAX); + return -1; + } char *str_copy = PyMem_Malloc(str_len + 1); if (!str_copy) { @@ -422,6 +441,11 @@ writer_intern_frame(BinaryWriter *writer, const FrameEntry *entry, uint32_t *ind return 0; } + if (writer->frame_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many frames for binary format"); + return -1; + } if (GROW_ARRAY(writer->frame_entries, writer->frame_count, writer->frame_capacity, FrameEntry) < 0) { return -1; @@ -466,6 +490,11 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, } } + if (writer->thread_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many threads for binary format"); + return NULL; + } if (writer->thread_count >= writer->thread_capacity) { ThreadEntry *new_entries = grow_array(writer->thread_entries, &writer->thread_capacity, @@ -600,6 +629,11 @@ flush_pending_rle(BinaryWriter *writer, ThreadEntry *entry) if (!entry->has_pending_rle || entry->pending_rle_count == 0) { return 0; } + if (entry->pending_rle_count > UINT32_MAX - writer->total_samples) { + PyErr_SetString(PyExc_OverflowError, + "too many samples for binary format"); + return -1; + } /* Write RLE record: * [thread_id: 8] [interpreter_id: 4] [STACK_REPEAT: 1] [count: varint] @@ -644,6 +678,12 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, const uint32_t *frame_indices, size_t stack_depth, size_t shared_count, size_t pop_count, size_t push_count) { + if (writer->total_samples == UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many samples for binary format"); + return -1; + } + /* Header: thread_id(8) + interpreter_id(4) + encoding(1) + delta(varint) + status(1) */ uint8_t header_buf[SAMPLE_HEADER_MAX_SIZE]; memcpy(header_buf + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); @@ -717,7 +757,7 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, } BinaryWriter * -binary_writer_create(const char *filename, uint64_t sample_interval_us, int compression_type, +binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us) { BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); @@ -726,14 +766,6 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp return NULL; } - writer->filename = PyMem_Malloc(strlen(filename) + 1); - if (!writer->filename) { - PyMem_Free(writer); - PyErr_NoMemory(); - return NULL; - } - strcpy(writer->filename, filename); - writer->start_time_us = start_time_us; writer->sample_interval_us = sample_interval_us; writer->compression_type = compression_type; @@ -799,9 +831,8 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp } } - writer->fp = fopen(filename, "wb"); + writer->fp = Py_fopen(path, "wb"); if (!writer->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -1193,7 +1224,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } - if (fclose(writer->fp) != 0) { + if (Py_fclose(writer->fp) != 0) { writer->fp = NULL; PyErr_SetFromErrno(PyExc_IOError); return -1; @@ -1211,10 +1242,9 @@ binary_writer_destroy(BinaryWriter *writer) } if (writer->fp) { - fclose(writer->fp); + Py_fclose(writer->fp); } - PyMem_Free(writer->filename); PyMem_Free(writer->write_buffer); #ifdef HAVE_ZSTD diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 1133db808efaec3..d01f3d13e85f09f 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -21,33 +21,37 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "\n" "Args:\n" " pid: Process ID of the target Python process to debug\n" -" all_threads: If True, initialize state for all threads in the process.\n" -" If False, only initialize for the main thread.\n" +" all_threads: If True, initialize state for all threads in the\n" +" process. If False, only initialize for the main thread.\n" " only_active_thread: If True, only sample the thread holding the GIL.\n" -" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time).\n" -" Cannot be used together with all_threads=True.\n" -" debug: If True, chain exceptions to explain the sequence of events that\n" -" lead to the exception.\n" -" skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" -" If False, include all threads regardless of mode.\n" -" native: If True, include artificial \"<native>\" frames to denote calls to\n" -" non-Python code.\n" -" gc: If True, include artificial \"<GC>\" frames to denote active garbage\n" -" collection.\n" -" opcodes: If True, gather bytecode opcode information for instruction-level\n" -" profiling.\n" -" cache_frames: If True, enable frame caching optimization to avoid re-reading\n" -" unchanged parent frames between samples.\n" -" stats: If True, collect statistics about cache hits, memory reads, etc.\n" -" Use get_stats() to retrieve the collected statistics.\n" -"\n" -"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" -"process, including examining thread states, stack frames and other runtime data.\n" +" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL\n" +" (gil-time). Cannot be used together with all_threads=True.\n" +" debug: If True, chain exceptions to explain the sequence of events\n" +" that lead to the exception.\n" +" skip_non_matching_threads: If True, skip threads that don\'t match\n" +" the selected mode. If False, include all threads regardless of\n" +" mode.\n" +" native: If True, include artificial \"<native>\" frames to denote\n" +" calls to non-Python code.\n" +" gc: If True, include artificial \"<GC>\" frames to denote active\n" +" garbage collection.\n" +" opcodes: If True, gather bytecode opcode information for\n" +" instruction-level profiling.\n" +" cache_frames: If True, enable frame caching optimization to avoid\n" +" re-reading unchanged parent frames between samples.\n" +" stats: If True, collect statistics about cache hits, memory reads,\n" +" etc. Use get_stats() to retrieve the collected statistics.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running\n" +"Python process, including examining thread states, stack frames and\n" +"other runtime data.\n" "\n" "Raises:\n" " PermissionError: If access to the target process is denied\n" -" OSError: If unable to attach to the target process or access its memory\n" -" RuntimeError: If unable to read debug information from the target process\n" +" OSError: If unable to attach to the target process or access its\n" +" memory\n" +" RuntimeError: If unable to read debug information from the target\n" +" process\n" " ValueError: If both all_threads and only_active_thread are True"); static int @@ -217,16 +221,21 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, "\n" "Returns stack traces for all interpreters and threads in process.\n" "\n" -"Each element in the returned list is a tuple of (interpreter_id, thread_list), where:\n" +"Each element in the returned list is a tuple of (interpreter_id,\n" +"thread_list), where:\n" "- interpreter_id is the interpreter identifier\n" -"- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter\n" +"- thread_list is a list of tuples (thread_id, frame_list) for\n" +" threads in that interpreter\n" " - thread_id is the OS thread identifier\n" -" - frame_list is a list of tuples (function_name, filename, line_number) representing\n" -" the Python stack frames for that thread, ordered from most recent to oldest\n" +" - frame_list is a list of tuples (function_name, filename,\n" +" line_number) representing the Python stack frames for that\n" +" thread, ordered from most recent to oldest\n" "\n" "The threads returned depend on the initialization parameters:\n" -"- If only_active_thread was True: returns only the thread holding the GIL across all interpreters\n" -"- If all_threads was True: returns all threads across all interpreters\n" +"- If only_active_thread was True: returns only the thread holding\n" +" the GIL across all interpreters\n" +"- If all_threads was True: returns all threads across all\n" +" interpreters\n" "- Otherwise: returns only the main thread of each interpreter\n" "\n" "Example:\n" @@ -250,10 +259,12 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, " ]\n" "\n" "Raises:\n" -" RuntimeError: If there is an error copying memory from the target process\n" +" RuntimeError: If there is an error copying memory from the\n" +" target process\n" " OSError: If there is an error accessing the target process\n" " PermissionError: If access to the target process is denied\n" -" UnicodeDecodeError: If there is an error decoding strings from the target process"); +" UnicodeDecodeError: If there is an error decoding strings from\n" +" the target process"); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, @@ -279,20 +290,25 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, "\n" "Get all tasks and their awaited_by relationships from the remote process.\n" "\n" -"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"This provides a tree structure showing which tasks are waiting for\n" +"other tasks.\n" "\n" "For each task, returns:\n" -"1. The call stack frames leading to where the task is currently executing\n" +"1. The call stack frames leading to where the task is currently\n" +" executing\n" "2. The name of the task\n" -"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"3. A list of tasks that this task is waiting for, with their own\n" +" frames/names/etc\n" "\n" "Returns a list of [frames, task_name, subtasks] where:\n" -"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- frames: List of (func_name, filename, lineno) showing the call\n" +" stack\n" "- task_name: String identifier for the task\n" "- subtasks: List of tasks being awaited by this task, in same format\n" "\n" "Raises:\n" -" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" RuntimeError: If AsyncioDebug section is not available in the\n" +" remote process\n" " MemoryError: If memory allocation fails\n" " OSError: If reading from the remote process fails\n" "\n" @@ -336,14 +352,16 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "\n" "Get the currently running async tasks and their dependency graphs from the remote process.\n" "\n" -"This returns information about running tasks and all tasks that are waiting for them,\n" -"forming a complete dependency graph for each thread\'s active task.\n" +"This returns information about running tasks and all tasks that are\n" +"waiting for them, forming a complete dependency graph for each\n" +"thread\'s active task.\n" "\n" -"For each thread with a running task, returns the running task plus all tasks that\n" -"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" -"those tasks, etc.).\n" +"For each thread with a running task, returns the running task plus\n" +"all tasks that transitively depend on it (tasks waiting for the\n" +"running task, tasks waiting for those tasks, etc.).\n" "\n" -"Returns a list of per-thread results, where each thread result contains:\n" +"Returns a list of per-thread results, where each thread result\n" +"contains:\n" "- Thread ID\n" "- List of task information for the running task and all its waiters\n" "\n" @@ -354,11 +372,13 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "- List of tasks waiting for this task (recursive structure)\n" "\n" "Raises:\n" -" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" RuntimeError: If AsyncioDebug section is not available in the\n" +" target process\n" " MemoryError: If memory allocation fails\n" " OSError: If reading from the remote process fails\n" "\n" -"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"Example output (similar structure to get_all_awaited_by but only for\n" +"running tasks):\n" "[\n" " (140234, [\n" " (4345585712, \'main_task\',\n" @@ -403,16 +423,34 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, " - total_samples: Total number of get_stack_trace calls\n" " - frame_cache_hits: Full cache hits (entire stack unchanged)\n" " - frame_cache_misses: Cache misses requiring full walk\n" -" - frame_cache_partial_hits: Partial hits (stopped at cached frame)\n" +" - frame_cache_partial_hits: Partial hits (stopped at cached\n" +" frame)\n" " - frames_read_from_cache: Total frames retrieved from cache\n" -" - frames_read_from_memory: Total frames read from remote memory\n" +" - frames_read_from_memory: Total frames read from remote\n" +" memory\n" " - memory_reads: Total remote memory read operations\n" " - memory_bytes_read: Total bytes read from remote memory\n" " - code_object_cache_hits: Code object cache hits\n" " - code_object_cache_misses: Code object cache misses\n" -" - stale_cache_invalidations: Times stale cache entries were cleared\n" -" - frame_cache_hit_rate: Percentage of samples that hit the cache\n" -" - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +" - stale_cache_invalidations: Times stale cache entries were\n" +" cleared\n" +" - batched_read_attempts: Batched remote-read attempts\n" +" - batched_read_successes: Attempts that read all requested\n" +" segments\n" +" - batched_read_misses: Attempts that fell back or partially\n" +" read\n" +" - batched_read_segments_requested: Segments requested by\n" +" batched reads\n" +" - batched_read_segments_completed: Segments completed by\n" +" batched reads\n" +" - frame_cache_hit_rate: Percentage of samples that hit the\n" +" cache\n" +" - code_object_cache_hit_rate: Percentage of code object\n" +" lookups that hit cache\n" +" - batched_read_success_rate: Percentage of batched reads\n" +" that completed all segments\n" +" - batched_read_segment_completion_rate: Percentage of\n" +" requested segments read by batched reads\n" "\n" "Raises:\n" " RuntimeError: If stats collection was not enabled (stats=False)"); @@ -442,9 +480,11 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_pause_threads__doc__, "Pause all threads in the target process.\n" "\n" "This stops all threads in the target process to allow for consistent\n" -"memory reads during sampling. Must be paired with a call to resume_threads().\n" +"memory reads during sampling. Must be paired with a call to\n" +"resume_threads().\n" "\n" -"Returns True if threads were successfully paused, False if they were already paused.\n" +"Returns True if threads were successfully paused, False if they were\n" +"already paused.\n" "\n" "Raises:\n" " RuntimeError: If there is an error stopping the threads"); @@ -473,9 +513,11 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_resume_threads__doc__, "\n" "Resume all threads in the target process.\n" "\n" -"This resumes threads that were previously paused with pause_threads().\n" +"This resumes threads that were previously paused with\n" +"pause_threads().\n" "\n" -"Returns True if threads were successfully resumed, False if they were not paused."); +"Returns True if threads were successfully resumed, False if they\n" +"were not paused."); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_RESUME_THREADS_METHODDEF \ {"resume_threads", (PyCFunction)_remote_debugging_RemoteUnwinder_resume_threads, METH_NOARGS, _remote_debugging_RemoteUnwinder_resume_threads__doc__}, @@ -503,16 +545,18 @@ PyDoc_STRVAR(_remote_debugging_GCMonitor___init____doc__, "\n" "Args:\n" " pid: Process ID of the target Python process to monitor\n" -" debug: If True, chain exceptions to explain the sequence of events that\n" -" lead to the exception.\n" +" debug: If True, chain exceptions to explain the sequence of\n" +" events that lead to the exception.\n" "\n" -"The GCMonitor provides functionality to read GC statistics from a running\n" -"Python process.\n" +"The GCMonitor provides functionality to read GC statistics from\n" +"a running Python process.\n" "\n" "Raises:\n" " PermissionError: If access to the target process is denied\n" -" OSError: If unable to attach to the target process or access its memory\n" -" RuntimeError: If unable to read debug information from the target process"); +" OSError: If unable to attach to the target process or access\n" +" its memory\n" +" RuntimeError: If unable to read debug information from the\n" +" target process"); static int _remote_debugging_GCMonitor___init___impl(GCMonitorObject *self, int pid, @@ -605,8 +649,8 @@ PyDoc_STRVAR(_remote_debugging_GCMonitor_get_gc_stats__doc__, " - duration: Total collection time, in seconds.\n" "\n" "Raises:\n" -" RuntimeError: If the target process cannot be inspected or if its\n" -" debug offsets or GC stats layout are incompatible."); +" RuntimeError: If the target process cannot be inspected or if\n" +" its debug offsets or GC stats layout are incompatible."); #define _REMOTE_DEBUGGING_GCMONITOR_GET_GC_STATS_METHODDEF \ {"get_gc_stats", _PyCFunction_CAST(_remote_debugging_GCMonitor_get_gc_stats), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_GCMonitor_get_gc_stats__doc__}, @@ -681,14 +725,15 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__, "Arguments:\n" " filename: Path to output file\n" " sample_interval_us: Sampling interval in microseconds\n" -" start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6)\n" +" start_time_us: Start timestamp in microseconds (from\n" +" time.monotonic() * 1e6)\n" " compression: 0=none, 1=zstd (default: 0)\n" "\n" "Use as a context manager or call finalize() when done."); static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression); @@ -728,7 +773,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; - const char *filename; + PyObject *filename; unsigned long long sample_interval_us; unsigned long long start_time_us; int compression = 0; @@ -738,19 +783,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryWriter", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; if (!_PyLong_UnsignedLongLong_Converter(fastargs[1], &sample_interval_us)) { goto exit; } @@ -779,7 +812,8 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter_write_sample__doc__, "\n" "Arguments:\n" " stack_frames: List of InterpreterInfo objects\n" -" timestamp_us: Current timestamp in microseconds (from time.monotonic() * 1e6)"); +" timestamp_us: Current timestamp in microseconds (from\n" +" time.monotonic() * 1e6)"); #define _REMOTE_DEBUGGING_BINARYWRITER_WRITE_SAMPLE_METHODDEF \ {"write_sample", _PyCFunction_CAST(_remote_debugging_BinaryWriter_write_sample), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_BinaryWriter_write_sample__doc__}, @@ -981,8 +1015,9 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter_get_stats__doc__, "\n" "Get encoding statistics for the writer.\n" "\n" -"Returns a dict with encoding statistics including repeat/full/suffix/pop-push\n" -"record counts, frames written/saved, and compression ratio."); +"Returns a dict with encoding statistics including\n" +"repeat/full/suffix/pop-push record counts, frames written/saved, and\n" +"compression ratio."); #define _REMOTE_DEBUGGING_BINARYWRITER_GET_STATS_METHODDEF \ {"get_stats", (PyCFunction)_remote_debugging_BinaryWriter_get_stats, METH_NOARGS, _remote_debugging_BinaryWriter_get_stats__doc__}, @@ -1009,7 +1044,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader___init____doc__, static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename); + PyObject *filename); static int _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1045,26 +1080,14 @@ _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - const char *filename; + PyObject *filename; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryReader", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; return_value = _remote_debugging_BinaryReader___init___impl((BinaryReaderObject *)self, filename); exit: @@ -1172,8 +1195,8 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader_get_stats__doc__, "\n" "Get reconstruction statistics from replay.\n" "\n" -"Returns a dict with statistics about record types decoded and samples\n" -"reconstructed during replay."); +"Returns a dict with statistics about record types decoded and\n" +"samples reconstructed during replay."); #define _REMOTE_DEBUGGING_BINARYREADER_GET_STATS_METHODDEF \ {"get_stats", (PyCFunction)_remote_debugging_BinaryReader_get_stats, METH_NOARGS, _remote_debugging_BinaryReader_get_stats__doc__}, @@ -1336,11 +1359,12 @@ PyDoc_STRVAR(_remote_debugging_get_child_pids__doc__, " If True, return all descendants (children, grandchildren, etc.).\n" " If False, return only direct children.\n" "\n" -"Returns a list of child process IDs. Returns an empty list if no children\n" -"are found.\n" +"Returns a list of child process IDs. Returns an empty list if no\n" +"children are found.\n" "\n" -"This function provides a snapshot of child processes at a moment in time.\n" -"Child processes may exit or new ones may be created after the list is returned.\n" +"This function provides a snapshot of child processes at a moment in\n" +"time. Child processes may exit or new ones may be created after the\n" +"list is returned.\n" "\n" "Raises:\n" " OSError: If unable to enumerate processes\n" @@ -1564,4 +1588,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=36674f4cb8a653f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3df14a6ab7f2998 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 7b95c0f2d4fa8da..ab889a130ee4e7e 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -47,7 +47,6 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t // Read the TLBC array pointer if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0) { - PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array pointer"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array pointer"); return 0; // Read error } @@ -61,7 +60,6 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t // Read the TLBC array size Py_ssize_t tlbc_size; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0) { - PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array size"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array size"); return 0; // Read error } @@ -405,6 +403,8 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->func_name = func; meta->file_name = file; meta->linetable = linetable; + meta->last_frame_info = NULL; + meta->last_addrq = -1; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; @@ -432,7 +432,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, #ifdef Py_GIL_DISABLED // Handle thread-local bytecode (TLBC) in free threading builds - if (ctx->tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + if (ctx->tlbc_index == 0 || unwinder == NULL || unwinder->debug_offsets.code_object.co_tlbc == 0) { // No TLBC or no unwinder - use main bytecode directly addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; goto done_tlbc; @@ -482,6 +482,12 @@ parse_code_object(RemoteUnwinderObject *unwinder, addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; #endif ; // Empty statement to avoid C23 extension warning + + if (!unwinder->opcodes && meta->last_frame_info != NULL && meta->last_addrq == addrq) { + *result = Py_NewRef(meta->last_frame_info); + return 0; + } + LocationInfo info = {0}; bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), PyBytes_GET_SIZE(meta->linetable), @@ -529,6 +535,11 @@ parse_code_object(RemoteUnwinderObject *unwinder, goto error; } + if (!unwinder->opcodes) { + Py_XSETREF(meta->last_frame_info, Py_NewRef(tuple)); + meta->last_addrq = addrq; + } + *result = tuple; return 0; diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c index b6566d7cff7b543..19fc406bca9ac96 100644 --- a/Modules/_remote_debugging/frame_cache.c +++ b/Modules/_remote_debugging/frame_cache.c @@ -30,6 +30,7 @@ frame_cache_cleanup(RemoteUnwinderObject *unwinder) return; } for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); } PyMem_Free(unwinder->frame_cache); @@ -53,6 +54,21 @@ frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) return NULL; } +FrameCacheEntry * +frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr) +{ + if (!unwinder->frame_cache || tstate_addr == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_state_addr == tstate_addr) { + assert(unwinder->frame_cache[i].num_addrs <= FRAME_CACHE_MAX_FRAMES); + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + // Allocate a cache slot for a thread // Returns NULL if cache is full (graceful degradation) static FrameCacheEntry * @@ -127,8 +143,10 @@ frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) } if (!found) { // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].thread_state_addr = 0; unwinder->frame_cache[i].num_addrs = 0; STATS_INC(unwinder, stale_cache_invalidations); } @@ -216,6 +234,7 @@ frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited) { @@ -257,6 +276,13 @@ frame_cache_store( return -1; } entry->thread_id = thread_id; + entry->thread_state_addr = thread_state_addr; + if (entry->thread_id_obj == NULL) { + entry->thread_id_obj = PyLong_FromUnsignedLongLong(thread_id); + if (entry->thread_id_obj == NULL) { + return -1; + } + } memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); entry->num_addrs = num_addrs; assert(entry->num_addrs == num_addrs); diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index bbdfce3f7201d9d..d73cd080dc477f3 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -56,12 +56,14 @@ process_single_stack_chunk( return -1; } - this_chunk = PyMem_RawRealloc(this_chunk, actual_size); - if (!this_chunk) { + char *tmp = PyMem_RawRealloc(this_chunk, actual_size); + if (!tmp) { + PyMem_RawFree(this_chunk); PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to reallocate stack chunk buffer"); return -1; } + this_chunk = tmp; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { PyMem_RawFree(this_chunk); @@ -186,30 +188,16 @@ is_frame_valid( return 1; } -int -parse_frame_object( +static int +parse_frame_buffer( RemoteUnwinderObject *unwinder, PyObject** result, - uintptr_t address, + const char *frame, uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { - char frame[SIZEOF_INTERP_FRAME]; *address_of_code_object = 0; - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); - return -1; - } - STATS_INC(unwinder, memory_reads); - STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); @@ -237,6 +225,31 @@ parse_frame_object( return parse_code_object(unwinder, result, &code_ctx); } +int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); + return -1; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + + return parse_frame_buffer(unwinder, result, frame, address_of_code_object, previous_frame); +} + int parse_frame_from_chunks( RemoteUnwinderObject *unwinder, @@ -312,15 +325,32 @@ process_frame_chain( } assert(frame_count <= MAX_FRAMES); - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) < 0) { + if (ctx->chunks && ctx->chunks->count > 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) == 0) { + goto parsed_frame; + } PyErr_Clear(); + } + { uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object, &next_frame_addr) < 0) { + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { + parse_result = parse_frame_buffer( + unwinder, &frame, ctx->prefetch.frame, + &address_of_code_object, &next_frame_addr); + } + else { + parse_result = parse_frame_object( + unwinder, &frame, frame_addr, + &address_of_code_object, &next_frame_addr); + } + if (parse_result < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } } +parsed_frame: // Skip first frame if requested (used for cache miss continuation) if (ctx->skip_first_frame && frame_count == 1) { Py_XDECREF(frame); @@ -501,41 +531,37 @@ try_full_cache_hit( PyObject *current_frame = NULL; uintptr_t code_object_addr = 0; uintptr_t previous_frame = 0; - int parse_result = parse_frame_object(unwinder, &current_frame, ctx->frame_addr, + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == ctx->frame_addr) { + parse_result = parse_frame_buffer(unwinder, &current_frame, + ctx->prefetch.frame, &code_object_addr, &previous_frame); + } + else { + parse_result = parse_frame_object(unwinder, &current_frame, ctx->frame_addr, + &code_object_addr, &previous_frame); + } if (parse_result < 0) { return -1; } - Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); - PyObject *parent_slice = NULL; - if (cached_size > 1) { - parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); - if (!parent_slice) { - Py_XDECREF(current_frame); - return -1; - } - } - if (current_frame != NULL) { if (PyList_Append(ctx->frame_info, current_frame) < 0) { Py_DECREF(current_frame); - Py_XDECREF(parent_slice); return -1; } Py_DECREF(current_frame); STATS_ADD(unwinder, frames_read_from_memory, 1); } - if (parent_slice) { - Py_ssize_t cur_size = PyList_GET_SIZE(ctx->frame_info); - int result = PyList_SetSlice(ctx->frame_info, cur_size, cur_size, parent_slice); - Py_DECREF(parent_slice); - if (result < 0) { + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + for (Py_ssize_t i = 1; i < cached_size; i++) { + PyObject *cached_frame = PyList_GET_ITEM(entry->frame_list, i); + if (PyList_Append(ctx->frame_info, cached_frame) < 0) { return -1; } - STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); } + STATS_ADD(unwinder, frames_read_from_cache, cached_size > 1 ? cached_size - 1 : 0); STATS_INC(unwinder, frame_cache_hits); return 1; @@ -606,7 +632,8 @@ collect_frames_with_cache( } if (frame_cache_store(unwinder, thread_id, ctx->frame_info, ctx->frame_addrs, ctx->num_addrs, - ctx->base_frame_addr, ctx->last_frame_visited) < 0) { + ctx->thread_state_addr, ctx->base_frame_addr, + ctx->last_frame_visited) < 0) { return -1; } diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index 172f8711a2a2a08..984213d18817523 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -166,6 +166,7 @@ cached_code_metadata_destroy(void *ptr) Py_DECREF(meta->func_name); Py_DECREF(meta->file_name); Py_DECREF(meta->linetable); + Py_XDECREF(meta->last_frame_info); PyMem_RawFree(meta); } @@ -279,7 +280,6 @@ class _remote_debugging.RemoteUnwinder "RemoteUnwinderObject *" "&RemoteUnwinder /*[clinic input] @permit_long_summary -@permit_long_docstring_body _remote_debugging.RemoteUnwinder.__init__ pid: int * @@ -298,33 +298,37 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. Args: pid: Process ID of the target Python process to debug - all_threads: If True, initialize state for all threads in the process. - If False, only initialize for the main thread. + all_threads: If True, initialize state for all threads in the + process. If False, only initialize for the main thread. only_active_thread: If True, only sample the thread holding the GIL. - mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time). - Cannot be used together with all_threads=True. - debug: If True, chain exceptions to explain the sequence of events that - lead to the exception. - skip_non_matching_threads: If True, skip threads that don't match the selected mode. - If False, include all threads regardless of mode. - native: If True, include artificial "<native>" frames to denote calls to - non-Python code. - gc: If True, include artificial "<GC>" frames to denote active garbage - collection. - opcodes: If True, gather bytecode opcode information for instruction-level - profiling. - cache_frames: If True, enable frame caching optimization to avoid re-reading - unchanged parent frames between samples. - stats: If True, collect statistics about cache hits, memory reads, etc. - Use get_stats() to retrieve the collected statistics. - -The RemoteUnwinder provides functionality to inspect and debug a running Python -process, including examining thread states, stack frames and other runtime data. + mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL + (gil-time). Cannot be used together with all_threads=True. + debug: If True, chain exceptions to explain the sequence of events + that lead to the exception. + skip_non_matching_threads: If True, skip threads that don't match + the selected mode. If False, include all threads regardless of + mode. + native: If True, include artificial "<native>" frames to denote + calls to non-Python code. + gc: If True, include artificial "<GC>" frames to denote active + garbage collection. + opcodes: If True, gather bytecode opcode information for + instruction-level profiling. + cache_frames: If True, enable frame caching optimization to avoid + re-reading unchanged parent frames between samples. + stats: If True, collect statistics about cache hits, memory reads, + etc. Use get_stats() to retrieve the collected statistics. + +The RemoteUnwinder provides functionality to inspect and debug a running +Python process, including examining thread states, stack frames and +other runtime data. Raises: PermissionError: If access to the target process is denied - OSError: If unable to attach to the target process or access its memory - RuntimeError: If unable to read debug information from the target process + OSError: If unable to attach to the target process or access its + memory + RuntimeError: If unable to read debug information from the target + process ValueError: If both all_threads and only_active_thread are True [clinic start generated code]*/ @@ -337,7 +341,7 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int native, int gc, int opcodes, int cache_frames, int stats) -/*[clinic end generated code: output=0031f743f4b9ad52 input=8fb61b24102dec6e]*/ +/*[clinic end generated code: output=0031f743f4b9ad52 input=9d25ae328d62626d]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -360,6 +364,10 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, self->cache_frames = cache_frames; self->collect_stats = stats; self->stale_invalidation_counter = 0; + self->cached_tstate_interpreter_addr = 0; + self->cached_tstate_addr = 0; + memset(self->cached_tstates, 0, sizeof(self->cached_tstates)); + memset(self->cached_generations, 0, sizeof(self->cached_generations)); self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -403,6 +411,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return -1; } if (async_debug_result < 0) { + if (_Py_RemoteDebug_HasPermissionError()) { + return -1; + } PyErr_Clear(); memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); self->async_debug_offsets_available = 0; @@ -473,23 +484,193 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return 0; } +static inline size_t +interpreter_thread_cache_index(uintptr_t interpreter_addr) +{ + // Direct-mapped table indexed by the remote interpreter address. Each entry + // stores the full address and verifies it on lookup, so hash collisions + // degrade to misses and cannot return a value from the wrong interpreter. + return (size_t)_Py_HashPointerRaw((const void *)interpreter_addr) + & (INTERPRETER_THREAD_CACHE_SIZE - 1); +} + +static inline uintptr_t +get_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + if (interpreter_addr == 0) { + return 0; + } + + if (self->cached_tstate_interpreter_addr == interpreter_addr) { + return self->cached_tstate_addr; + } + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + if (entry->interpreter_addr == interpreter_addr) { + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = entry->thread_state_addr; + return entry->thread_state_addr; + } + return 0; +} + +static inline void +set_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + uintptr_t thread_state_addr) +{ + if (interpreter_addr == 0 || thread_state_addr == 0) { + return; + } + + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = thread_state_addr; + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + entry->interpreter_addr = interpreter_addr; + entry->thread_state_addr = thread_state_addr; +} + +static void +refresh_generation_caches_from_interp_state( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + const char *interp_state_buffer) +{ + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (self->cached_generation_interpreter_addr == interpreter_addr) { + if (code_object_generation != self->cached_code_object_generation) { + self->cached_code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + } + else { + InterpreterGenerationCacheEntry *entry = + &self->cached_generations[interpreter_thread_cache_index(interpreter_addr)]; + // A slot rebound from another interpreter must be treated as changed: + // the code_object_cache is global, so even if the new generation + // numerically matches what the previous occupant had, stale entries + // from that occupant could still be served. + int changed = entry->interpreter_addr != interpreter_addr + || entry->code_object_generation != code_object_generation; + entry->interpreter_addr = interpreter_addr; + entry->code_object_generation = code_object_generation; + if (changed) { + _Py_hashtable_clear(self->code_object_cache); + } + self->cached_generation_interpreter_addr = interpreter_addr; + self->cached_code_object_generation = code_object_generation; + } + +#ifdef Py_GIL_DISABLED + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif +} + +static int +refresh_generation_caches_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_ReadRemoteMemory( + &self->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, + "Failed to read interpreter state buffer"); + return -1; + } + refresh_generation_caches_from_interp_state(self, interpreter_addr, interp_state_buffer); + return 0; +} + +static int +read_interp_state_and_maybe_thread_frame( + RemoteUnwinderObject *unwinder, + uintptr_t interpreter_addr, + char *interp_state_buffer, + char *tstate_buffer, + char *frame_buffer, + RemoteReadPrefetch *prefetch) +{ + prefetch->tstate = NULL; + prefetch->frame = NULL; + if (prefetch->tstate_addr != 0) { + size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size; + _Py_RemoteReadSegment segments[3] = { + {interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE}, + {prefetch->tstate_addr, tstate_buffer, tstate_size}, + {prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + int nsegs = prefetch->frame_addr != 0 ? 3 : 2; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, nsegs); + int completed = 0; + if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) { + completed = 1; + Py_ssize_t with_tstate = (Py_ssize_t)INTERP_STATE_BUFFER_SIZE + + (Py_ssize_t)tstate_size; + if (nread >= with_tstate) { + completed = 2; + } + if (nsegs == 3 + && nread == with_tstate + (Py_ssize_t)SIZEOF_INTERP_FRAME) { + completed = 3; + } + } + STATS_BATCHED_READ(unwinder, nsegs, completed); + if (completed >= 1) { + if (completed >= 2) { + prefetch->tstate = tstate_buffer; + } + if (completed >= 3) { + prefetch->frame = frame_buffer; + } + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer); +} + /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_stack_trace Returns stack traces for all interpreters and threads in process. -Each element in the returned list is a tuple of (interpreter_id, thread_list), where: +Each element in the returned list is a tuple of (interpreter_id, +thread_list), where: - interpreter_id is the interpreter identifier -- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter +- thread_list is a list of tuples (thread_id, frame_list) for + threads in that interpreter - thread_id is the OS thread identifier - - frame_list is a list of tuples (function_name, filename, line_number) representing - the Python stack frames for that thread, ordered from most recent to oldest + - frame_list is a list of tuples (function_name, filename, + line_number) representing the Python stack frames for that + thread, ordered from most recent to oldest The threads returned depend on the initialization parameters: -- If only_active_thread was True: returns only the thread holding the GIL across all interpreters -- If all_threads was True: returns all threads across all interpreters +- If only_active_thread was True: returns only the thread holding + the GIL across all interpreters +- If all_threads was True: returns all threads across all + interpreters - Otherwise: returns only the main thread of each interpreter Example: @@ -513,16 +694,18 @@ The threads returned depend on the initialization parameters: ] Raises: - RuntimeError: If there is an error copying memory from the target process + RuntimeError: If there is an error copying memory from the + target process OSError: If there is an error accessing the target process PermissionError: If access to the target process is denied - UnicodeDecodeError: If there is an error decoding strings from the target process + UnicodeDecodeError: If there is an error decoding strings from + the target process [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=666192b90c69d567 input=bcff01c73cccc1c0]*/ +/*[clinic end generated code: output=666192b90c69d567 input=86a992b853f48aa9]*/ { STATS_INC(self, total_samples); @@ -537,15 +720,32 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self while (current_interpreter != 0) { // Read interpreter state to get the interpreter ID char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; - if (_Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, + char prefetched_tstate[SIZEOF_THREAD_STATE]; + char prefetched_frame[SIZEOF_INTERP_FRAME]; + RemoteReadPrefetch prefetch = {0}; + if (self->cache_frames) { + prefetch.tstate_addr = get_cached_tstate_for_interpreter( + self, current_interpreter); + } + if (prefetch.tstate_addr != 0) { + FrameCacheEntry *entry = frame_cache_find_by_tstate(self, prefetch.tstate_addr); + if (entry && entry->num_addrs > 0) { + prefetch.frame_addr = entry->addrs[0]; + } + } + + if (read_interp_state_and_maybe_thread_frame( + self, current_interpreter, - INTERP_STATE_BUFFER_SIZE, - interp_state_buffer) < 0) { + interp_state_buffer, + prefetched_tstate, + prefetched_frame, + &prefetch) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); Py_CLEAR(result); goto exit; } + refresh_generation_caches_from_interp_state(self, current_interpreter, interp_state_buffer); uintptr_t gc_frame = 0; if (self->gc) { @@ -557,25 +757,6 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); - // Get code object generation from buffer - uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, - self->debug_offsets.interpreter_state.code_object_generation); - - if (code_object_generation != self->code_object_generation) { - self->code_object_generation = code_object_generation; - _Py_hashtable_clear(self->code_object_cache); - } - -#ifdef Py_GIL_DISABLED - // Check TLBC generation and invalidate cache if needed - uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, - self->debug_offsets.interpreter_state.tlbc_generation); - if (current_tlbc_generation != self->tlbc_generation) { - self->tlbc_generation = current_tlbc_generation; - _Py_hashtable_clear(self->tlbc_cache); - } -#endif - // Create a list to hold threads for this interpreter PyObject *interpreter_threads = PyList_New(0); if (!interpreter_threads) { @@ -611,6 +792,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self // Target specific thread (only process first interpreter) current_tstate = self->tstate_addr; } + if (current_tstate != 0 && self->cache_frames) { + set_cached_tstate_for_interpreter(self, current_interpreter, current_tstate); + } // Acquire main thread state information uintptr_t main_thread_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, @@ -621,7 +805,8 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self PyObject* frame_info = unwind_stack_for_thread(self, &current_tstate, gil_holder_tstate, gc_frame, - main_thread_tstate); + main_thread_tstate, + &prefetch); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || @@ -720,26 +905,30 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_all_awaited_by Get all tasks and their awaited_by relationships from the remote process. -This provides a tree structure showing which tasks are waiting for other tasks. +This provides a tree structure showing which tasks are waiting for +other tasks. For each task, returns: -1. The call stack frames leading to where the task is currently executing +1. The call stack frames leading to where the task is currently + executing 2. The name of the task -3. A list of tasks that this task is waiting for, with their own frames/names/etc +3. A list of tasks that this task is waiting for, with their own + frames/names/etc Returns a list of [frames, task_name, subtasks] where: -- frames: List of (func_name, filename, lineno) showing the call stack +- frames: List of (func_name, filename, lineno) showing the call + stack - task_name: String identifier for the task - subtasks: List of tasks being awaited by this task, in same format Raises: - RuntimeError: If AsyncioDebug section is not available in the remote process + RuntimeError: If AsyncioDebug section is not available in the + remote process MemoryError: If memory allocation fails OSError: If reading from the remote process fails @@ -766,11 +955,14 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6a49cd345e8aec53 input=307f754cbe38250c]*/ +/*[clinic end generated code: output=6a49cd345e8aec53 input=c22bfee0612e0b69]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -808,20 +1000,21 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_async_stack_trace Get the currently running async tasks and their dependency graphs from the remote process. -This returns information about running tasks and all tasks that are waiting for them, -forming a complete dependency graph for each thread's active task. +This returns information about running tasks and all tasks that are +waiting for them, forming a complete dependency graph for each +thread's active task. -For each thread with a running task, returns the running task plus all tasks that -transitively depend on it (tasks waiting for the running task, tasks waiting for -those tasks, etc.). +For each thread with a running task, returns the running task plus +all tasks that transitively depend on it (tasks waiting for the +running task, tasks waiting for those tasks, etc.). -Returns a list of per-thread results, where each thread result contains: +Returns a list of per-thread results, where each thread result +contains: - Thread ID - List of task information for the running task and all its waiters @@ -832,11 +1025,13 @@ Each task info contains: - List of tasks waiting for this task (recursive structure) Raises: - RuntimeError: If AsyncioDebug section is not available in the target process + RuntimeError: If AsyncioDebug section is not available in the + target process MemoryError: If memory allocation fails OSError: If reading from the remote process fails -Example output (similar structure to get_all_awaited_by but only for running tasks): +Example output (similar structure to get_all_awaited_by but only for +running tasks): [ # Thread 140234 results (140234, [ @@ -855,11 +1050,14 @@ Example output (similar structure to get_all_awaited_by but only for running tas static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=6129b7d509a887c9]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=bc802e4221c99399]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -881,7 +1079,6 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_stats @@ -896,16 +1093,34 @@ RemoteUnwinder was created with stats=True. - total_samples: Total number of get_stack_trace calls - frame_cache_hits: Full cache hits (entire stack unchanged) - frame_cache_misses: Cache misses requiring full walk - - frame_cache_partial_hits: Partial hits (stopped at cached frame) + - frame_cache_partial_hits: Partial hits (stopped at cached + frame) - frames_read_from_cache: Total frames retrieved from cache - - frames_read_from_memory: Total frames read from remote memory + - frames_read_from_memory: Total frames read from remote + memory - memory_reads: Total remote memory read operations - memory_bytes_read: Total bytes read from remote memory - code_object_cache_hits: Code object cache hits - code_object_cache_misses: Code object cache misses - - stale_cache_invalidations: Times stale cache entries were cleared - - frame_cache_hit_rate: Percentage of samples that hit the cache - - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + - stale_cache_invalidations: Times stale cache entries were + cleared + - batched_read_attempts: Batched remote-read attempts + - batched_read_successes: Attempts that read all requested + segments + - batched_read_misses: Attempts that fell back or partially + read + - batched_read_segments_requested: Segments requested by + batched reads + - batched_read_segments_completed: Segments completed by + batched reads + - frame_cache_hit_rate: Percentage of samples that hit the + cache + - code_object_cache_hit_rate: Percentage of code object + lookups that hit cache + - batched_read_success_rate: Percentage of batched reads + that completed all segments + - batched_read_segment_completion_rate: Percentage of + requested segments read by batched reads Raises: RuntimeError: If stats collection was not enabled (stats=False) @@ -913,7 +1128,7 @@ RemoteUnwinder was created with stats=True. static PyObject * _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +/*[clinic end generated code: output=21e36477122be2a0 input=87905c65038fb06e]*/ { if (!self->collect_stats) { PyErr_SetString(PyExc_RuntimeError, @@ -948,9 +1163,24 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) ADD_STAT(code_object_cache_hits); ADD_STAT(code_object_cache_misses); ADD_STAT(stale_cache_invalidations); + ADD_STAT(batched_read_attempts); + ADD_STAT(batched_read_successes); + ADD_STAT(batched_read_misses); + ADD_STAT(batched_read_segments_requested); + ADD_STAT(batched_read_segments_completed); #undef ADD_STAT +#define ADD_DERIVED_STAT(name, value) do { \ + PyObject *val = PyFloat_FromDouble(value); \ + if (!val || PyDict_SetItemString(result, name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + // Calculate and add derived statistics // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups double frame_cache_hit_rate = 0.0; @@ -959,41 +1189,49 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) / (double)total_cache_lookups; } - PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); - if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { - Py_XDECREF(hit_rate); - Py_DECREF(result); - return NULL; - } - Py_DECREF(hit_rate); + ADD_DERIVED_STAT("frame_cache_hit_rate", frame_cache_hit_rate); double code_object_hit_rate = 0.0; uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; if (total_code_lookups > 0) { code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; } - PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); - if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { - Py_XDECREF(code_hit_rate); - Py_DECREF(result); - return NULL; + ADD_DERIVED_STAT("code_object_cache_hit_rate", code_object_hit_rate); + + double batched_read_success_rate = 0.0; + if (self->stats.batched_read_attempts > 0) { + batched_read_success_rate = + 100.0 * (double)self->stats.batched_read_successes + / (double)self->stats.batched_read_attempts; } - Py_DECREF(code_hit_rate); + ADD_DERIVED_STAT("batched_read_success_rate", batched_read_success_rate); + + double batched_read_segment_completion_rate = 0.0; + if (self->stats.batched_read_segments_requested > 0) { + batched_read_segment_completion_rate = + 100.0 * (double)self->stats.batched_read_segments_completed + / (double)self->stats.batched_read_segments_requested; + } + ADD_DERIVED_STAT("batched_read_segment_completion_rate", + batched_read_segment_completion_rate); + +#undef ADD_DERIVED_STAT return result; } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.pause_threads Pause all threads in the target process. This stops all threads in the target process to allow for consistent -memory reads during sampling. Must be paired with a call to resume_threads(). +memory reads during sampling. Must be paired with a call to +resume_threads(). -Returns True if threads were successfully paused, False if they were already paused. +Returns True if threads were successfully paused, False if they were +already paused. Raises: RuntimeError: If there is an error stopping the threads @@ -1001,7 +1239,7 @@ Returns True if threads were successfully paused, False if they were already pau static PyObject * _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=aaf2bdc0a725750c input=d8a266f19a81c67e]*/ +/*[clinic end generated code: output=aaf2bdc0a725750c input=b91dde5517c9dde2]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (self->threads_stopped) { @@ -1023,20 +1261,21 @@ _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.resume_threads Resume all threads in the target process. -This resumes threads that were previously paused with pause_threads(). +This resumes threads that were previously paused with +pause_threads(). -Returns True if threads were successfully resumed, False if they were not paused. +Returns True if threads were successfully resumed, False if they +were not paused. [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_resume_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=8d6781ea37095536 input=16baaaab007f4259]*/ +/*[clinic end generated code: output=8d6781ea37095536 input=130758d55d46897a]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (!self->threads_stopped) { @@ -1174,7 +1413,6 @@ class _remote_debugging.GCMonitor "GCMonitorObject *" "&GCMonitor_Type" /*[clinic input] @permit_long_summary -@permit_long_docstring_body _remote_debugging.GCMonitor.__init__ pid: int * @@ -1184,22 +1422,24 @@ Initialize a new GCMonitor object for monitoring GC events from remote process. Args: pid: Process ID of the target Python process to monitor - debug: If True, chain exceptions to explain the sequence of events that - lead to the exception. + debug: If True, chain exceptions to explain the sequence of + events that lead to the exception. -The GCMonitor provides functionality to read GC statistics from a running -Python process. +The GCMonitor provides functionality to read GC statistics from +a running Python process. Raises: PermissionError: If access to the target process is denied - OSError: If unable to attach to the target process or access its memory - RuntimeError: If unable to read debug information from the target process + OSError: If unable to attach to the target process or access + its memory + RuntimeError: If unable to read debug information from the + target process [clinic start generated code]*/ static int _remote_debugging_GCMonitor___init___impl(GCMonitorObject *self, int pid, int debug) -/*[clinic end generated code: output=2cdf351c2f6335db input=1185a48535b808be]*/ +/*[clinic end generated code: output=2cdf351c2f6335db input=03da0b2d3282ae1b]*/ { return init_runtime_offsets(&self->offsets, pid, debug); } @@ -1230,14 +1470,14 @@ Returns a list of GCStatsInfo objects with GC statistics data. - duration: Total collection time, in seconds. Raises: - RuntimeError: If the target process cannot be inspected or if its - debug offsets or GC stats layout are incompatible. + RuntimeError: If the target process cannot be inspected or if + its debug offsets or GC stats layout are incompatible. [clinic start generated code]*/ static PyObject * _remote_debugging_GCMonitor_get_gc_stats_impl(GCMonitorObject *self, int all_interpreters) -/*[clinic end generated code: output=f73f365725224f7a input=12f7c1a288cf2741]*/ +/*[clinic end generated code: output=f73f365725224f7a input=ec016bc4be6dd003]*/ { RemoteDebuggingState *st = RemoteDebugging_GetStateFromType(Py_TYPE(self)); return get_gc_stats(&self->offsets, all_interpreters, st->GCStatsInfo_Type); @@ -1474,9 +1714,8 @@ class _remote_debugging.BinaryWriter "BinaryWriterObject *" "&PyBinaryWriter_Typ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e948838b90a2003c]*/ /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.__init__ - filename: str + filename: object sample_interval_us: unsigned_long_long start_time_us: unsigned_long_long * @@ -1487,7 +1726,8 @@ High-performance binary writer for profiling data. Arguments: filename: Path to output file sample_interval_us: Sampling interval in microseconds - start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) + start_time_us: Start timestamp in microseconds (from + time.monotonic() * 1e6) compression: 0=none, 1=zstd (default: 0) Use as a context manager or call finalize() when done. @@ -1495,11 +1735,11 @@ Use as a context manager or call finalize() when done. static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression) -/*[clinic end generated code: output=014c0306f1bacf4b input=3bdf01c1cc2f5a1d]*/ +/*[clinic end generated code: output=00446656ea2e5986 input=2e3f298c69fc7666]*/ { if (self->writer) { binary_writer_destroy(self->writer); @@ -1514,7 +1754,6 @@ _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, } /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.write_sample stack_frames: object timestamp_us: unsigned_long_long @@ -1523,14 +1762,15 @@ Write a sample to the binary file. Arguments: stack_frames: List of InterpreterInfo objects - timestamp_us: Current timestamp in microseconds (from time.monotonic() * 1e6) + timestamp_us: Current timestamp in microseconds (from + time.monotonic() * 1e6) [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, PyObject *stack_frames, unsigned long long timestamp_us) -/*[clinic end generated code: output=24d5b86679b4128f input=4e6d832d360bea46]*/ +/*[clinic end generated code: output=24d5b86679b4128f input=5033f1ae7fa135f1]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1653,18 +1893,18 @@ _remote_debugging_BinaryWriter___exit___impl(BinaryWriterObject *self, } /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.get_stats Get encoding statistics for the writer. -Returns a dict with encoding statistics including repeat/full/suffix/pop-push -record counts, frames written/saved, and compression ratio. +Returns a dict with encoding statistics including +repeat/full/suffix/pop-push record counts, frames written/saved, and +compression ratio. [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) -/*[clinic end generated code: output=06522cd52544df89 input=837c874ffdebd24c]*/ +/*[clinic end generated code: output=06522cd52544df89 input=a8bb8c8682ccd34b]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1742,7 +1982,7 @@ class _remote_debugging.BinaryReader "BinaryReaderObject *" "&PyBinaryReader_Typ /*[clinic input] _remote_debugging.BinaryReader.__init__ - filename: str + filename: object High-performance binary reader for profiling data. @@ -1754,8 +1994,8 @@ Use as a context manager or call close() when done. static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename) -/*[clinic end generated code: output=9699226f7ae052bb input=4201f9cc500ef2f6]*/ + PyObject *filename) +/*[clinic end generated code: output=f04b33ee5c5e6dbf input=9d7cbe8b4f1a97c9]*/ { if (self->reader) { binary_reader_close(self->reader); @@ -1829,13 +2069,13 @@ _remote_debugging.BinaryReader.get_stats Get reconstruction statistics from replay. -Returns a dict with statistics about record types decoded and samples -reconstructed during replay. +Returns a dict with statistics about record types decoded and +samples reconstructed during replay. [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryReader_get_stats_impl(BinaryReaderObject *self) -/*[clinic end generated code: output=628b9ab5e4c4fd36 input=d8dd6654abd6c3c0]*/ +/*[clinic end generated code: output=628b9ab5e4c4fd36 input=15b8d8f89ccf3726]*/ { if (!self->reader) { PyErr_SetString(PyExc_ValueError, "Reader is closed"); @@ -1987,7 +2227,6 @@ _remote_debugging_zstd_available_impl(PyObject *module) * ============================================================================ */ /*[clinic input] -@permit_long_docstring_body _remote_debugging.get_child_pids pid: int @@ -1999,11 +2238,12 @@ _remote_debugging.get_child_pids Get all child process IDs of the given process. -Returns a list of child process IDs. Returns an empty list if no children -are found. +Returns a list of child process IDs. Returns an empty list if no +children are found. -This function provides a snapshot of child processes at a moment in time. -Child processes may exit or new ones may be created after the list is returned. +This function provides a snapshot of child processes at a moment in +time. Child processes may exit or new ones may be created after the +list is returned. Raises: OSError: If unable to enumerate processes @@ -2013,7 +2253,7 @@ Child processes may exit or new ones may be created after the list is returned. static PyObject * _remote_debugging_get_child_pids_impl(PyObject *module, int pid, int recursive) -/*[clinic end generated code: output=1ae2289c6b953e4b input=19d8d5d6e2b59e6e]*/ +/*[clinic end generated code: output=1ae2289c6b953e4b input=c6437b52e2fdd880]*/ { return enumerate_child_pids((pid_t)pid, recursive); } diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index b63b103a2617acc..1cea96a2151fcc6 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include <limits.h> /* ============================================================================ * MEMORY READING FUNCTIONS @@ -264,26 +265,16 @@ read_py_long( Py_ssize_t inline_digits_space = SIZEOF_LONG_OBJ - ob_digit_offset; Py_ssize_t max_inline_digits = inline_digits_space / (Py_ssize_t)sizeof(digit); - // If the long object has inline digits that fit in our buffer, use them directly - digit *digits; + digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for PyLong"); + return -1; + } + if (size <= max_inline_digits && size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { - // For small integers, digits are inline in the long_value.ob_digit array - digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for small PyLong"); - return -1; - } memcpy(digits, long_obj + ob_digit_offset, size * sizeof(digit)); } else { - // For larger integers, we need to read the digits separately - digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for large PyLong"); - return -1; - } - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, @@ -296,19 +287,34 @@ read_py_long( } } - long long value = 0; + unsigned long limit = negative + ? (unsigned long)LONG_MAX + 1UL + : (unsigned long)LONG_MAX; + unsigned long value = 0; - // In theory this can overflow, but because of llvm/llvm-project#16778 - // we can't use __builtin_mul_overflow because it fails to link with - // __muloti4 on aarch64. In practice this is fine because all we're - // testing here are task numbers that would fit in a single byte. - for (Py_ssize_t i = 0; i < size; ++i) { - long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); - value += factor; + for (Py_ssize_t i = size; i-- > 0;) { + if (digits[i] >= PyLong_BASE) { + PyErr_Format(PyExc_RuntimeError, + "Invalid PyLong digit: %u (base %u)", digits[i], PyLong_BASE); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid PyLong digit (corrupted remote memory)"); + goto error; + } + if (value > ((limit - (unsigned long)digits[i]) >> shift)) { + PyErr_SetString(PyExc_OverflowError, + "Remote PyLong value does not fit in C long"); + set_exception_cause(unwinder, PyExc_OverflowError, + "Remote PyLong value is too large"); + goto error; + } + value = (value << shift) | (unsigned long)digits[i]; } PyMem_RawFree(digits); if (negative) { - value *= -1; + if (value == (unsigned long)LONG_MAX + 1UL) { + return LONG_MIN; + } + return -(long)value; } return (long)value; error: diff --git a/Modules/_remote_debugging/subprocess.c b/Modules/_remote_debugging/subprocess.c index 1b16dd8343f2a5b..cdad75e318be914 100644 --- a/Modules/_remote_debugging/subprocess.c +++ b/Modules/_remote_debugging/subprocess.c @@ -223,8 +223,19 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) } /* Single pass: collect PIDs and their PPIDs together */ - struct dirent *entry; - while ((entry = readdir(proc_dir)) != NULL) { + for (;;) { + errno = 0; + struct dirent *entry = readdir(proc_dir); + if (entry == NULL) { + if (errno != 0) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to read process directory '/proc': %s", + strerror(err)); + goto done; + } + break; + } /* Skip non-numeric entries (also skips . and ..) */ if (entry->d_name[0] < '1' || entry->d_name[0] > '9') { continue; @@ -245,7 +256,14 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) } } - closedir(proc_dir); + if (closedir(proc_dir) != 0) { + int err = errno; + proc_dir = NULL; + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to close process directory '/proc': %s", + strerror(err)); + goto done; + } proc_dir = NULL; if (find_children_bfs(target_pid, recursive, @@ -358,7 +376,8 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); if (snapshot == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); goto done; } @@ -373,13 +392,23 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) /* Single pass: collect PIDs and PPIDs together */ PROCESSENTRY32 pe; pe.dwSize = sizeof(PROCESSENTRY32); - if (Process32First(snapshot, &pe)) { - do { - if (pid_array_append(&all_pids, (pid_t)pe.th32ProcessID) < 0 || - pid_array_append(&ppids, (pid_t)pe.th32ParentProcessID) < 0) { - goto done; - } - } while (Process32Next(snapshot, &pe)); + if (!Process32First(snapshot, &pe)) { + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); + goto done; + } + + do { + if (pid_array_append(&all_pids, (pid_t)pe.th32ProcessID) < 0 || + pid_array_append(&ppids, (pid_t)pe.th32ParentProcessID) < 0) { + goto done; + } + } while (Process32Next(snapshot, &pe)); + + DWORD error = GetLastError(); + if (error != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(error); + goto done; } CloseHandle(snapshot); diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index d775234b8d78d72..81735e85395ac9e 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -289,28 +289,110 @@ typedef struct { unsigned int :24; } _thread_status; +static int +read_thread_state_and_maybe_frame( + RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + size_t tstate_size, + char *tstate_buffer, + uintptr_t predicted_frame_addr, + char *frame_buffer, + int *frame_read) +{ + *frame_read = 0; + if (predicted_frame_addr != 0) { + _Py_RemoteReadSegment segments[2] = { + {tstate_addr, tstate_buffer, tstate_size}, + {predicted_frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, 2); + int completed = 0; + if (nread >= (Py_ssize_t)tstate_size) { + completed = 1; + if (nread == (Py_ssize_t)(tstate_size + SIZEOF_INTERP_FRAME)) { + completed = 2; + } + } + STATS_BATCHED_READ(unwinder, 2, completed); + if (completed >= 1) { + *frame_read = completed == 2; + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, tstate_addr, tstate_size, tstate_buffer); +} + PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; PyObject *result = NULL; StackChunkList chunks = {0}; - char ts[SIZEOF_THREAD_STATE]; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); - goto error; + char local_ts[SIZEOF_THREAD_STATE]; + char local_prefetched_frame[SIZEOF_INTERP_FRAME]; + const char *ts; + RemoteReadPrefetch ctx_prefetch = {0}; + if (prefetch->tstate && prefetch->tstate_addr == *current_tstate) { + ts = prefetch->tstate; + if (prefetch->frame) { + ctx_prefetch.frame = prefetch->frame; + ctx_prefetch.frame_addr = prefetch->frame_addr; + } + } + else if (unwinder->cache_frames) { + uintptr_t predicted_frame_addr = 0; + int have_prefetched_frame = 0; + FrameCacheEntry *entry = frame_cache_find_by_tstate(unwinder, *current_tstate); + if (entry && entry->num_addrs > 0) { + predicted_frame_addr = entry->addrs[0]; + } + + int rc = read_thread_state_and_maybe_frame( + unwinder, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts, + predicted_frame_addr, + local_prefetched_frame, + &have_prefetched_frame); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; + if (have_prefetched_frame) { + ctx_prefetch.frame = local_prefetched_frame; + ctx_prefetch.frame_addr = predicted_frame_addr; + } + } + else { + int rc = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; } STATS_INC(unwinder, memory_reads); STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); + if (ctx_prefetch.frame) { + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + } long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); @@ -432,9 +514,11 @@ unwind_stack_for_thread( uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; FrameWalkContext ctx = { .frame_addr = frame_addr, + .thread_state_addr = *current_tstate, .base_frame_addr = base_frame_addr, .gc_frame = gc_frame, .chunks = &chunks, + .prefetch = ctx_prefetch, .frame_info = frame_info, .frame_addrs = addrs, .num_addrs = 0, @@ -450,12 +534,14 @@ unwind_stack_for_thread( set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to collect frames"); goto error; } - // Update last_profiled_frame for next sample - uintptr_t lpf_addr = - *current_tstate + (uintptr_t)unwinder->debug_offsets.thread_state.last_profiled_frame; - if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, - sizeof(uintptr_t), &frame_addr) < 0) { - PyErr_Clear(); // Non-fatal + // Update last_profiled_frame for next sample if it changed + if (frame_addr != ctx.last_profiled_frame) { + uintptr_t lpf_addr = + *current_tstate + (uintptr_t)unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &frame_addr) < 0) { + PyErr_Clear(); // Non-fatal + } } } else { // No caching - process entire frame chain with base_frame validation @@ -467,10 +553,18 @@ unwind_stack_for_thread( *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); - thread_id = PyLong_FromLongLong(tid); + if (unwinder->cache_frames) { + FrameCacheEntry *entry = frame_cache_find(unwinder, (uint64_t)tid); + if (entry && entry->thread_id_obj) { + thread_id = Py_NewRef(entry->thread_id_obj); + } + } if (thread_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); - goto error; + thread_id = PyLong_FromLongLong(tid); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } } RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); @@ -566,8 +660,7 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st DIR *dir = opendir(task_path); if (dir == NULL) { - st->tids = NULL; - st->count = 0; + _Py_RemoteDebug_InitThreadsState(unwinder, st); if (errno == ENOENT || errno == ESRCH) { PyErr_Format(PyExc_ProcessLookupError, "Process %d has terminated", unwinder->handle.pid); @@ -579,8 +672,21 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st st->count = 0; - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { + for (;;) { + errno = 0; + struct dirent *entry = readdir(dir); + if (entry == NULL) { + if (errno != 0) { + int err = errno; + closedir(dir); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to read process task directory '%s': %s", + task_path, strerror(err)); + return -1; + } + break; + } if (entry->d_name[0] < '1' || entry->d_name[0] > '9') { continue; } @@ -594,8 +700,7 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st pid_t *new_tids = PyMem_RawRealloc(unwinder->thread_tids, new_cap * sizeof(pid_t)); if (new_tids == NULL) { closedir(dir); - st->tids = NULL; - st->count = 0; + _Py_RemoteDebug_InitThreadsState(unwinder, st); PyErr_NoMemory(); return -1; } @@ -605,8 +710,15 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st unwinder->thread_tids[st->count++] = (pid_t)tid; } + if (closedir(dir) != 0) { + int err = errno; + _Py_RemoteDebug_InitThreadsState(unwinder, st); + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to close process task directory '%s': %s", + task_path, strerror(err)); + return -1; + } st->tids = unwinder->thread_tids; - closedir(dir); return 0; } @@ -619,28 +731,30 @@ detach_threads(_Py_RemoteDebug_ThreadsState *st, size_t up_to) } static int -seize_thread(pid_t tid) +seize_thread(pid_t tid, int *err) { if (ptrace(PTRACE_SEIZE, tid, NULL, 0) == 0) { return 0; } - if (errno == ESRCH) { + *err = errno; + if (*err == ESRCH) { return 1; // Thread gone, skip } - if (errno == EPERM) { + if (*err == EPERM) { // Thread may have exited, be in a special state, or already be traced. // Skip rather than fail - this avoids endless retry loops when // threads transiently become inaccessible. return 1; } - if (errno == EINVAL || errno == EIO) { + if (*err == EINVAL || *err == EIO) { // Fallback for older kernels if (ptrace(PTRACE_ATTACH, tid, NULL, NULL) == 0) { int status; waitpid(tid, &status, __WALL); return 0; } - if (errno == ESRCH || errno == EPERM) { + *err = errno; + if (*err == ESRCH || *err == EPERM) { return 1; // Thread gone or inaccessible } } @@ -654,39 +768,50 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T return -1; } - for (size_t i = 0; i < st->count; i++) { + size_t n_tids = st->count; + size_t seized = 0; + for (size_t i = 0; i < n_tids; i++) { pid_t tid = st->tids[i]; - int ret = seize_thread(tid); + int err = 0; + int ret = seize_thread(tid, &err); if (ret == 1) { continue; // Thread gone, skip } if (ret < 0) { - detach_threads(st, i); - PyErr_Format(PyExc_RuntimeError, "Failed to seize thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "Failed to seize thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } - - if (ptrace(PTRACE_INTERRUPT, tid, NULL, NULL) == -1 && errno != ESRCH) { - detach_threads(st, i + 1); - PyErr_Format(PyExc_RuntimeError, "Failed to interrupt thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; - return -1; + st->tids[seized++] = tid; + + if (ptrace(PTRACE_INTERRUPT, tid, NULL, NULL) == -1) { + err = errno; + if (err != ESRCH) { + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "Failed to interrupt thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + return -1; + } } int status; - if (waitpid(tid, &status, __WALL) == -1 && errno != ECHILD && errno != ESRCH) { - detach_threads(st, i + 1); - PyErr_Format(PyExc_RuntimeError, "waitpid failed for thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; - return -1; + if (waitpid(tid, &status, __WALL) == -1) { + err = errno; + if (err != ECHILD && err != ESRCH) { + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "waitpid failed for thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + return -1; + } } } + st->count = seized; return 0; } diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 8dad94556236bd6..d81784409e5d91a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -166,7 +166,6 @@ read_multiple(pysqlite_Blob *self, Py_ssize_t length, Py_ssize_t offset) /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.read as blob_read length: int = -1 @@ -175,14 +174,14 @@ _sqlite3.Blob.read as blob_read Read data at the current offset position. -If the end of the blob is reached, the data up to end of file will be returned. -When length is not specified, or is negative, Blob.read() will read until the -end of the blob. +If the end of the blob is reached, the data up to end of file will +be returned. When length is not specified, or is negative, +Blob.read() will read until the end of the blob. [clinic start generated code]*/ static PyObject * blob_read_impl(pysqlite_Blob *self, int length) -/*[clinic end generated code: output=1fc99b2541360dde input=e5715bcddbcfca5a]*/ +/*[clinic end generated code: output=1fc99b2541360dde input=6b745ad37720e556]*/ { if (!check_blob(self)) { return NULL; @@ -235,7 +234,6 @@ inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.write as blob_write data: Py_buffer @@ -243,13 +241,13 @@ _sqlite3.Blob.write as blob_write Write data at the current offset. -This function cannot change the blob length. Writing beyond the end of the -blob will result in an exception being raised. +This function cannot change the blob length. Writing beyond the end +of the blob will result in an exception being raised. [clinic start generated code]*/ static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data) -/*[clinic end generated code: output=b34cf22601b570b2 input=203d3458f244814b]*/ +/*[clinic end generated code: output=b34cf22601b570b2 input=0d372cb0240a5d49]*/ { if (!check_blob(self)) { return NULL; @@ -265,7 +263,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.seek as blob_seek offset: int @@ -274,14 +271,15 @@ _sqlite3.Blob.seek as blob_seek Set the current access position to offset. -The origin argument defaults to os.SEEK_SET (absolute blob positioning). -Other values for origin are os.SEEK_CUR (seek relative to the current position) -and os.SEEK_END (seek relative to the blob's end). +The origin argument defaults to os.SEEK_SET (absolute blob +positioning). Other values for origin are os.SEEK_CUR (seek +relative to the current position) and os.SEEK_END (seek relative to +the blob's end). [clinic start generated code]*/ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin) -/*[clinic end generated code: output=854c5a0e208547a5 input=ee4d88e1dc0b1048]*/ +/*[clinic end generated code: output=854c5a0e208547a5 input=84aea1b6b48607dd]*/ { if (!check_blob(self)) { return NULL; @@ -533,21 +531,25 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (len == 0) { - return 0; - } - Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; } - int rc = -1; if (vbuf.len != len) { PyErr_SetString(PyExc_IndexError, "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; } - else if (step == 1) { + + if (len == 0) { + PyBuffer_Release(&vbuf); + return 0; + } + + int rc = -1; + if (step == 1) { rc = inner_write(self, vbuf.buf, len, start); } else { diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 921e7cbd7ffcaba..929703257f04bee 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -31,9 +31,9 @@ PyDoc_STRVAR(blob_read__doc__, " length\n" " Read length in bytes.\n" "\n" -"If the end of the blob is reached, the data up to end of file will be returned.\n" -"When length is not specified, or is negative, Blob.read() will read until the\n" -"end of the blob."); +"If the end of the blob is reached, the data up to end of file will\n" +"be returned. When length is not specified, or is negative,\n" +"Blob.read() will read until the end of the blob."); #define BLOB_READ_METHODDEF \ {"read", _PyCFunction_CAST(blob_read), METH_FASTCALL, blob_read__doc__}, @@ -70,8 +70,8 @@ PyDoc_STRVAR(blob_write__doc__, "\n" "Write data at the current offset.\n" "\n" -"This function cannot change the blob length. Writing beyond the end of the\n" -"blob will result in an exception being raised."); +"This function cannot change the blob length. Writing beyond the end\n" +"of the blob will result in an exception being raised."); #define BLOB_WRITE_METHODDEF \ {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, @@ -105,9 +105,10 @@ PyDoc_STRVAR(blob_seek__doc__, "\n" "Set the current access position to offset.\n" "\n" -"The origin argument defaults to os.SEEK_SET (absolute blob positioning).\n" -"Other values for origin are os.SEEK_CUR (seek relative to the current position)\n" -"and os.SEEK_END (seek relative to the blob\'s end)."); +"The origin argument defaults to os.SEEK_SET (absolute blob\n" +"positioning). Other values for origin are os.SEEK_CUR (seek\n" +"relative to the current position) and os.SEEK_END (seek relative to\n" +"the blob\'s end)."); #define BLOB_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(blob_seek), METH_FASTCALL, blob_seek__doc__}, @@ -211,4 +212,4 @@ blob_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f03f4ba622b67ae0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b0e3d38063739b17 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index abb864eb0307578..b645bf3464bcea1 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -685,13 +685,14 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "\n" " progress_handler\n" " A callable that takes no arguments.\n" -" If the callable returns non-zero, the current query is terminated,\n" -" and an exception is raised.\n" +" If the callable returns non-zero, the current query is\n" +" terminated, and an exception is raised.\n" " n\n" " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled."); +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is\n" +"disabled."); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ {"set_progress_handler", _PyCFunction_CAST(pysqlite_connection_set_progress_handler), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, @@ -1303,10 +1304,10 @@ PyDoc_STRVAR(serialize__doc__, " name\n" " Which database to serialize.\n" "\n" -"For an ordinary on-disk database file, the serialization is just a copy of the\n" -"disk file. For an in-memory database or a \"temp\" database, the serialization is\n" -"the same sequence of bytes which would be written to disk if that database\n" -"were backed up to disk."); +"For an ordinary on-disk database file, the serialization is just\n" +"a copy of the disk file. For an in-memory database or a \"temp\"\n" +"database, the serialization is the same sequence of bytes which\n" +"would be written to disk if that database were backed up to disk."); #define SERIALIZE_METHODDEF \ {"serialize", _PyCFunction_CAST(serialize), METH_FASTCALL|METH_KEYWORDS, serialize__doc__}, @@ -1392,12 +1393,13 @@ PyDoc_STRVAR(deserialize__doc__, " name\n" " Which database to reopen with the deserialization.\n" "\n" -"The deserialize interface causes the database connection to disconnect from the\n" -"target database, and then reopen it as an in-memory database based on the given\n" -"serialized data.\n" +"The deserialize interface causes the database connection to\n" +"disconnect from the target database, and then reopen it as\n" +"an in-memory database based on the given serialized data.\n" "\n" -"The deserialize interface will fail with SQLITE_BUSY if the database is\n" -"currently in a read transaction or is involved in a backup operation."); +"The deserialize interface will fail with SQLITE_BUSY if the database\n" +"is currently in a read transaction or is involved in a backup\n" +"operation."); #define DESERIALIZE_METHODDEF \ {"deserialize", _PyCFunction_CAST(deserialize), METH_FASTCALL|METH_KEYWORDS, deserialize__doc__}, @@ -1518,7 +1520,8 @@ PyDoc_STRVAR(pysqlite_connection_exit__doc__, "\n" "Called when the connection is used as a context manager.\n" "\n" -"If there was any exception, a rollback takes place; otherwise we commit."); +"If there was any exception, a rollback takes place; otherwise we\n" +"commit."); #define PYSQLITE_CONNECTION_EXIT_METHODDEF \ {"__exit__", _PyCFunction_CAST(pysqlite_connection_exit), METH_FASTCALL, pysqlite_connection_exit__doc__}, @@ -1556,12 +1559,12 @@ PyDoc_STRVAR(setlimit__doc__, " category\n" " The limit category to be set.\n" " limit\n" -" The new limit. If the new limit is a negative number, the limit is\n" -" unchanged.\n" +" The new limit. If the new limit is a negative number, the limit\n" +" is unchanged.\n" "\n" -"Attempts to increase a limit above its hard upper bound are silently truncated\n" -"to the hard upper bound. Regardless of whether or not the limit was changed,\n" -"the prior value of the limit is returned."); +"Attempts to increase a limit above its hard upper bound are silently\n" +"truncated to the hard upper bound. Regardless of whether or not the\n" +"limit was changed, the prior value of the limit is returned."); #define SETLIMIT_METHODDEF \ {"setlimit", _PyCFunction_CAST(setlimit), METH_FASTCALL, setlimit__doc__}, @@ -1722,4 +1725,4 @@ getconfig(PyObject *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=16d44c1d8a45e622 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1418b72751ef68fc input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index bd44ff31b87c67b..892740b05e55c98 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -557,6 +557,47 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } +static PyObject * +connection_get_row_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->row_factory); +} + +static int +connection_set_row_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete row_factory attribute"); + return -1; + } + Py_XSETREF(self->row_factory, Py_NewRef(value)); + return 0; +} + +static PyObject * +connection_get_text_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->text_factory); +} + +static int +connection_set_text_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete text_factory attribute"); + return -1; + } + Py_XSETREF(self->text_factory, Py_NewRef(value)); + return 0; +} + + /*[clinic input] _sqlite3.Connection.blobopen as blobopen @@ -1561,14 +1602,13 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_handler cls: defining_class progress_handler as callable: object A callable that takes no arguments. - If the callable returns non-zero, the current query is terminated, - and an exception is raised. + If the callable returns non-zero, the current query is + terminated, and an exception is raised. / n: int The number of SQLite virtual machine instructions that are @@ -1576,14 +1616,15 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han Set progress handler callback. -If 'progress_handler' is None or 'n' is 0, the progress handler is disabled. +If 'progress_handler' is None or 'n' is 0, the progress handler is +disabled. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=3ecce6c915922ad4]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=fd0d5abb004f370f]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1606,6 +1647,7 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, } /*[clinic input] +@permit_long_summary _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback cls: defining_class @@ -1619,7 +1661,7 @@ static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=d91048c03bfcee05 input=f4f59bf2f87f2026]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=edfd7d890200a9cb]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2212,7 +2254,6 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, #ifdef PY_SQLITE_HAVE_SERIALIZE /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.serialize as serialize * @@ -2221,15 +2262,15 @@ _sqlite3.Connection.serialize as serialize Serialize a database into a byte string. -For an ordinary on-disk database file, the serialization is just a copy of the -disk file. For an in-memory database or a "temp" database, the serialization is -the same sequence of bytes which would be written to disk if that database -were backed up to disk. +For an ordinary on-disk database file, the serialization is just +a copy of the disk file. For an in-memory database or a "temp" +database, the serialization is the same sequence of bytes which +would be written to disk if that database were backed up to disk. [clinic start generated code]*/ static PyObject * serialize_impl(pysqlite_Connection *self, const char *name) -/*[clinic end generated code: output=97342b0e55239dd3 input=963e617cdf75c747]*/ +/*[clinic end generated code: output=97342b0e55239dd3 input=7e48654e8e082fa8]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2263,7 +2304,6 @@ serialize_impl(pysqlite_Connection *self, const char *name) } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.deserialize as deserialize data: Py_buffer(accept={buffer, str}) @@ -2275,18 +2315,19 @@ _sqlite3.Connection.deserialize as deserialize Load a serialized database. -The deserialize interface causes the database connection to disconnect from the -target database, and then reopen it as an in-memory database based on the given -serialized data. +The deserialize interface causes the database connection to +disconnect from the target database, and then reopen it as +an in-memory database based on the given serialized data. -The deserialize interface will fail with SQLITE_BUSY if the database is -currently in a read transaction or is involved in a backup operation. +The deserialize interface will fail with SQLITE_BUSY if the database +is currently in a read transaction or is involved in a backup +operation. [clinic start generated code]*/ static PyObject * deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name) -/*[clinic end generated code: output=e394c798b98bad89 input=037e94599aaa5b5c]*/ +/*[clinic end generated code: output=e394c798b98bad89 input=5d20e028d98c0686]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2359,13 +2400,14 @@ _sqlite3.Connection.__exit__ as pysqlite_connection_exit Called when the connection is used as a context manager. -If there was any exception, a rollback takes place; otherwise we commit. +If there was any exception, a rollback takes place; otherwise we +commit. [clinic start generated code]*/ static PyObject * pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=0705200e9321202a input=bd66f1532c9c54a7]*/ +/*[clinic end generated code: output=0705200e9321202a input=8fdb0392ee6f3466]*/ { int commit = 0; PyObject* result; @@ -2400,26 +2442,25 @@ pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.setlimit as setlimit category: int The limit category to be set. limit: int - The new limit. If the new limit is a negative number, the limit is - unchanged. + The new limit. If the new limit is a negative number, the limit + is unchanged. / Set connection run-time limits. -Attempts to increase a limit above its hard upper bound are silently truncated -to the hard upper bound. Regardless of whether or not the limit was changed, -the prior value of the limit is returned. +Attempts to increase a limit above its hard upper bound are silently +truncated to the hard upper bound. Regardless of whether or not the +limit was changed, the prior value of the limit is returned. [clinic start generated code]*/ static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit) -/*[clinic end generated code: output=0d208213f8d68ccd input=bf06e06a21eb37e2]*/ +/*[clinic end generated code: output=0d208213f8d68ccd input=5c2e430091206677]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2620,6 +2661,10 @@ static PyGetSetDef connection_getset[] = { {"in_transaction", pysqlite_connection_get_in_transaction, NULL}, {"autocommit", get_autocommit, set_autocommit}, {"__text_signature__", get_sig, NULL}, + {"row_factory", connection_get_row_factory, + connection_set_row_factory}, + {"text_factory", connection_get_text_factory, + connection_set_text_factory}, {NULL} }; @@ -2667,8 +2712,6 @@ static struct PyMemberDef connection_members[] = {"InternalError", _Py_T_OBJECT, offsetof(pysqlite_Connection, InternalError), Py_READONLY}, {"ProgrammingError", _Py_T_OBJECT, offsetof(pysqlite_Connection, ProgrammingError), Py_READONLY}, {"NotSupportedError", _Py_T_OBJECT, offsetof(pysqlite_Connection, NotSupportedError), Py_READONLY}, - {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, row_factory)}, - {"text_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, text_factory)}, {NULL} }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 044eb6e5f1fb662..d372d6a03ab4ee4 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -2587,6 +2587,7 @@ _pair(Py_ssize_t i1, Py_ssize_t i2) } /*[clinic input] +@permit_long_summary _sre.SRE_Match.span group: object(c_default="NULL") = 0 @@ -2597,7 +2598,7 @@ For match object m, return the 2-tuple (m.start(group), m.end(group)). static PyObject * _sre_SRE_Match_span_impl(MatchObject *self, PyObject *group) -/*[clinic end generated code: output=f02ae40594d14fe6 input=8fa6014e982d71d4]*/ +/*[clinic end generated code: output=f02ae40594d14fe6 input=834cfe444f0f55cf]*/ { Py_ssize_t index = match_getindex(self, group); @@ -3179,7 +3180,7 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Patterns are generic over the type of string they handle (str or bytes)")}, {NULL, NULL} }; @@ -3235,7 +3236,7 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Matches are generic over the type of string which was matched (str or bytes)")}, {NULL, NULL} }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 3224ca7d0f93b96..f451c0ce7364ab9 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -26,6 +26,7 @@ #define OPENSSL_NO_DEPRECATED 1 #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -2357,27 +2358,26 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) } /*[clinic input] -@permit_long_docstring_body @critical_section @getter _ssl._SSLSocket.context This changes the context associated with the SSLSocket. -This is typically used from within a callback function set by the sni_callback -on the SSLContext to change the certificate information associated with the -SSLSocket before the cryptographic exchange handshake messages. +This is typically used from within a callback function set by the +sni_callback on the SSLContext to change the certificate information +associated with the SSLSocket before the cryptographic exchange +handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=0cc8e773a079295e]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=b845dea1f9710ebe]*/ { return Py_NewRef(self->ctx); } /*[clinic input] -@permit_long_docstring_body @critical_section @setter _ssl._SSLSocket.context @@ -2385,7 +2385,7 @@ _ssl._SSLSocket.context static int _ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value) -/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=f7fc1674b660df96]*/ +/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=48ece77724fd9dd4]*/ { if (PyObject_TypeCheck(value, self->ctx->state->PySSLContext_Type)) { Py_SETREF(self->ctx, (PySSLContext *)Py_NewRef(value)); @@ -2635,7 +2635,6 @@ _ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) #ifdef BIO_get_ktls_send /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _ssl._SSLSocket.sendfile fd: int @@ -2646,9 +2645,9 @@ _ssl._SSLSocket.sendfile Write size bytes from offset in the file descriptor fd to the SSL connection. -This method uses the zero-copy technique and returns the number of bytes -written. It should be called only when Kernel TLS is used for sending data in -the connection. +This method uses the zero-copy technique and returns the number of +bytes written. It should be called only when Kernel TLS is used for +sending data in the connection. The meaning of flags is platform dependent. [clinic start generated code]*/ @@ -2656,7 +2655,7 @@ The meaning of flags is platform dependent. static PyObject * _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, size_t size, int flags) -/*[clinic end generated code: output=0c6815b0719ca8d5 input=1f193e681bbae664]*/ +/*[clinic end generated code: output=0c6815b0719ca8d5 input=68c7fbf90c9a8a1b]*/ { Py_ssize_t retval; int sockstate; @@ -3175,22 +3174,22 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } /*[clinic input] -@permit_long_docstring_body @critical_section _ssl._SSLSocket.get_channel_binding cb_type: str = "tls-unique" Get channel binding data for current connection. -Raise ValueError if the requested `cb_type` is not supported. Return bytes -of the data or None if the data is not available (e.g. before the handshake). +Raise ValueError if the requested `cb_type` is not supported. +Return bytes of the data or None if the data is not available (e.g. +before the handshake). Only 'tls-unique' channel binding data from RFC 5929 is supported. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=26fad522435ecca1]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=bed81ef7936535a0]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -5153,12 +5152,15 @@ _servername_callback(SSL *s, int *al, void *args) PyObject *result; /* The high-level ssl.SSLSocket object */ PyObject *ssl_socket; + PyObject *sni_cb; const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); PyGILState_STATE gstate = PyGILState_Ensure(); - if (sslctx->set_sni_cb == NULL) { - /* remove race condition in this the call back while if removing the - * callback is in progress */ + Py_BEGIN_CRITICAL_SECTION(sslctx); + sni_cb = Py_XNewRef(sslctx->set_sni_cb); + Py_END_CRITICAL_SECTION(); + + if (sni_cb == NULL) { PyGILState_Release(gstate); return SSL_TLSEXT_ERR_OK; } @@ -5185,7 +5187,7 @@ _servername_callback(SSL *s, int *al, void *args) goto error; if (servername == NULL) { - result = PyObject_CallFunctionObjArgs(sslctx->set_sni_cb, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, Py_None, sslctx, NULL); } else { @@ -5212,7 +5214,7 @@ _servername_callback(SSL *s, int *al, void *args) } Py_DECREF(servername_bytes); result = PyObject_CallFunctionObjArgs( - sslctx->set_sni_cb, ssl_socket, servername_str, + sni_cb, ssl_socket, servername_str, sslctx, NULL); Py_DECREF(servername_str); } @@ -5222,7 +5224,7 @@ _servername_callback(SSL *s, int *al, void *args) PyErr_FormatUnraisable("Exception ignored " "in ssl servername callback " "while calling set SNI callback %R", - sslctx->set_sni_cb); + sni_cb); *al = SSL_AD_HANDSHAKE_FAILURE; ret = SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -5247,11 +5249,13 @@ _servername_callback(SSL *s, int *al, void *args) Py_DECREF(result); } + Py_DECREF(sni_cb); PyGILState_Release(gstate); return ret; error: Py_XDECREF(ssl_socket); + Py_XDECREF(sni_cb); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; PyGILState_Release(gstate); @@ -5260,22 +5264,22 @@ _servername_callback(SSL *s, int *al, void *args) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section @getter _ssl._SSLContext.sni_callback Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. -If the argument is None then the callback is disabled. The method is called -with the SSLSocket, the server name as a string, and the SSLContext object. +If the argument is None then the callback is disabled. The method +is called with the SSLSocket, the server name as a string, and the +SSLContext object. See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=3aee06696b0874d9]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=a319bc8fc15d6fc8]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -5286,7 +5290,6 @@ _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section @setter _ssl._SSLContext.sni_callback @@ -5294,27 +5297,25 @@ _ssl._SSLContext.sni_callback static int _ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) -/*[clinic end generated code: output=b32736c6b891f61a input=332def1d8c81d549]*/ +/*[clinic end generated code: output=b32736c6b891f61a input=402b43fb06c1139e]*/ { if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { PyErr_SetString(PyExc_ValueError, "sni_callback cannot be set on TLS_CLIENT context"); return -1; } - Py_CLEAR(self->set_sni_cb); - if (value == Py_None) { + if (!PyCallable_Check(value)) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - } - else { - if (!PyCallable_Check(value)) { - SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - PyErr_SetString(PyExc_TypeError, - "not a callable object"); + Py_CLEAR(self->set_sni_cb); + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, "not a callable object"); return -1; } - self->set_sni_cb = Py_NewRef(value); - SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); + } + else { + Py_XSETREF(self->set_sni_cb, Py_NewRef(value)); SSL_CTX_set_tlsext_servername_arg(self->ctx, self); + SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); } return 0; } @@ -5369,16 +5370,16 @@ _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. -X.509 certificates with a CA extension and certificate revocation lists -inside the context's cert store. +X.509 certificates with a CA extension and certificate revocation +lists inside the context's cert store. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=9e5094e094b892a3]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -5421,16 +5422,16 @@ _ssl._SSLContext.get_ca_certs Returns a list of dicts with information of loaded CA certs. -If the optional argument is True, returns a DER-encoded copy of the CA -certificate. +If the optional argument is True, returns a DER-encoded copy of the +CA certificate. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=9f71af5aa4e67076]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -6319,13 +6320,13 @@ _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. -It is necessary to seed the PRNG with RAND_add() on some platforms before -using the ssl() function. +It is necessary to seed the PRNG with RAND_add() on some platforms +before using the ssl() function. [clinic start generated code]*/ static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=aba24a3f3af3b184]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=52b061f4a24ff3a1]*/ { return PyBool_FromLong(RAND_status()); } @@ -6621,16 +6622,16 @@ _ssl.enum_certificates Retrieve certificates from Windows' cert store. -store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide -more cert storages, too. The function returns a list of (bytes, -encoding_type, trust) tuples. The encoding_type flag can be interpreted -with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either -a set of OIDs or the boolean True. +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may +provide more cert storages, too. The function returns a list of +(bytes, encoding_type, trust) tuples. The encoding_type flag can be +interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The +trust setting is either a set of OIDs or the boolean True. [clinic start generated code]*/ static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=ef81b4bd1b7ab8e9]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; diff --git a/Modules/_struct.c b/Modules/_struct.c index 3a970d99bb3d6d8..8c611a708d02a9d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2721,6 +2721,7 @@ pack_into_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer, } /*[clinic input] +@permit_long_summary unpack format as s_object: cache_struct @@ -2735,12 +2736,13 @@ for more on format strings. static PyObject * unpack_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer) -/*[clinic end generated code: output=48ddd4d88eca8551 input=7df28c5d0b5b6f4e]*/ +/*[clinic end generated code: output=48ddd4d88eca8551 input=53a60a65830bd1e1]*/ { return Struct_unpack_impl(s_object, buffer); } /*[clinic input] +@permit_long_summary unpack_from format as s_object: cache_struct @@ -2757,7 +2759,7 @@ help(struct) for more on format strings. static PyObject * unpack_from_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=1042631674c6e0d3 input=599262b23559f6c5]*/ +/*[clinic end generated code: output=1042631674c6e0d3 input=3e46619756fb0293]*/ { return Struct_unpack_from_impl(s_object, buffer, offset); } diff --git a/Modules/_testcapi/import.c b/Modules/_testcapi/import.c index ebb1032fdd1c320..384a8f52da4b984 100644 --- a/Modules/_testcapi/import.c +++ b/Modules/_testcapi/import.c @@ -41,8 +41,6 @@ pyimport_setlazyimportsmode(PyObject *self, PyObject *args) PyImport_SetLazyImportsMode(PyImport_LAZY_NORMAL); } else if (strcmp(PyUnicode_AsUTF8(mode), "all") == 0) { PyImport_SetLazyImportsMode(PyImport_LAZY_ALL); - } else if (strcmp(PyUnicode_AsUTF8(mode), "none") == 0) { - PyImport_SetLazyImportsMode(PyImport_LAZY_NONE); } else { PyErr_SetString(PyExc_ValueError, "invalid mode"); return NULL; @@ -59,8 +57,6 @@ pyimport_getlazyimportsmode(PyObject *self, PyObject *args) return PyUnicode_FromString("normal"); case PyImport_LAZY_ALL: return PyUnicode_FromString("all"); - case PyImport_LAZY_NONE: - return PyUnicode_FromString("none"); default: PyErr_SetString(PyExc_ValueError, "unknown mode"); return NULL; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 6e5c8dcbb725fa5..09a548fd2e24489 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -560,10 +560,11 @@ pysentinel_new(PyObject *self, PyObject *args) { const char *name; const char *module_name = NULL; - if (!PyArg_ParseTuple(args, "s|s", &name, &module_name)) { + const char *repr = NULL; + if (!PyArg_ParseTuple(args, "s|ss", &name, &module_name, &repr)) { return NULL; } - return PySentinel_New(name, module_name); + return PySentinel_New(name, module_name, repr); } static PyObject * @@ -572,6 +573,12 @@ pysentinel_check(PyObject *self, PyObject *obj) return PyBool_FromLong(PySentinel_Check(obj)); } +static PyObject * +pysentinel_checkexact(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_CheckExact(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -604,6 +611,7 @@ static PyMethodDef test_methods[] = { {"pyobject_dump", pyobject_dump, METH_VARARGS}, {"pysentinel_new", pysentinel_new, METH_VARARGS}, {"pysentinel_check", pysentinel_check, METH_O}, + {"pysentinel_checkexact", pysentinel_checkexact, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index e0abf6b1845d8ef..71cdc54009017a7 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 238e17bea303d35..a2506524f0bb6dc 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -7946,8 +7946,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13056,8 +13057,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h index ebaeb53921a82f1..f9852aba266a570 100644 --- a/Modules/_testlimitedcapi/clinic/long.c.h +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -84,8 +84,8 @@ PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, "\n" "Test the PyLong_As{Size,Ssize}_t API.\n" "\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); +"At present this just tests that non-integer arguments are handled\n" +"correctly. It should be extended to test overflow handling."); #define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, @@ -140,4 +140,4 @@ PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, #define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb5c95bd0a4bdad8 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index 34bc7331da92477..99b9e96760d50df 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -451,13 +451,13 @@ _testlimitedcapi.test_long_as_size_t Test the PyLong_As{Size,Ssize}_t API. -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. +At present this just tests that non-integer arguments are handled +correctly. It should be extended to test overflow handling. [clinic start generated code]*/ static PyObject * _testlimitedcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +/*[clinic end generated code: output=297a9f14a42f55af input=692e73744b35bf6e]*/ { size_t out_u; Py_ssize_t out_s; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 128e3f79ecd99c2..57ba61e67f1d7f4 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -141,14 +141,14 @@ _testmultiphase.StateAccessType.get_defining_module Return the module of the defining class. -Also tests that result of PyType_GetModuleByDef matches defining_class's -module. +Also tests that result of PyType_GetModuleByDef matches +defining_class's module. [clinic start generated code]*/ static PyObject * _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=ba2a14284a5d0921 input=d2c7245c8a9d06f8]*/ +/*[clinic end generated code: output=ba2a14284a5d0921 input=903e7f66555d65ae]*/ { PyObject *retval; retval = PyType_GetModule(cls); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 135b53111014d1c..7a34c5a5a0ef6be 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -820,8 +820,8 @@ _thread.lock.acquire Lock the lock. -Without argument, this blocks if the lock is already -locked (even by the same thread), waiting for another thread to release +Without argument, this blocks if the lock is already locked +(even by the same thread), waiting for another thread to release the lock, and return True once the lock is acquired. With an argument, this will only block if the argument is true, and the return value reflects whether the lock is acquired. @@ -831,7 +831,7 @@ The blocking operation is interruptible. static PyObject * _thread_lock_acquire_impl(lockobject *self, int blocking, PyObject *timeoutobj) -/*[clinic end generated code: output=569d6b25d508bf6f input=13e999649bc1c798]*/ +/*[clinic end generated code: output=569d6b25d508bf6f input=73e75b3d2ec32677]*/ { PyTime_t timeout; @@ -1131,19 +1131,19 @@ _thread.RLock.release Release the lock. -Allows another thread that is blocked waiting for -the lock to acquire the lock. The lock must be in the locked state, +Allows another thread that is blocked waiting for the lock +to acquire the lock. The lock must be in the locked state, and must be locked by the same thread that unlocks it; otherwise a `RuntimeError` is raised. -Do note that if the lock was acquire()d several times in a row by the -current thread, release() needs to be called as many times for the lock -to be available for other threads. +Do note that if the lock was acquire()d several times in a row by +the current thread, release() needs to be called as many times for +the lock to be available for other threads. [clinic start generated code]*/ static PyObject * _thread_RLock_release_impl(rlockobject *self) -/*[clinic end generated code: output=51f4a013c5fae2c5 input=d425daf1a5782e63]*/ +/*[clinic end generated code: output=51f4a013c5fae2c5 input=7c188f60189be13a]*/ { if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index bbe2a428454e0cc..58fdabecf16ada7 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3219,7 +3219,6 @@ _tkinter_create_impl(PyObject *module, const char *screenName, /*[clinic input] @permit_long_summary -@permit_long_docstring_body _tkinter.setbusywaitinterval new_val: int @@ -3227,12 +3226,13 @@ _tkinter.setbusywaitinterval Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter. -It should be set to a divisor of the maximum time between frames in an animation. +It should be set to a divisor of the maximum time between frames in +an animation. [clinic start generated code]*/ static PyObject * _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) -/*[clinic end generated code: output=42bf7757dc2d0ab6 input=07b82a04b56625e1]*/ +/*[clinic end generated code: output=42bf7757dc2d0ab6 input=0360dd95c8bd8619]*/ { if (new_val < 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 56d83ea0dcb2a7d..a97b0e0bfeefba6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -11,6 +11,7 @@ module _tracemalloc /*[clinic input] +@permit_long_summary _tracemalloc.is_tracing Return True if the tracemalloc module is tracing Python memory allocations. @@ -18,7 +19,7 @@ Return True if the tracemalloc module is tracing Python memory allocations. static PyObject * _tracemalloc_is_tracing_impl(PyObject *module) -/*[clinic end generated code: output=2d763b42601cd3ef input=af104b0a00192f63]*/ +/*[clinic end generated code: output=2d763b42601cd3ef input=cac4fc9096babeac]*/ { return PyBool_FromLong(_PyTraceMalloc_IsTracing()); } @@ -153,6 +154,7 @@ _tracemalloc_get_tracemalloc_memory_impl(PyObject *module) /*[clinic input] +@permit_long_summary _tracemalloc.get_traced_memory Get the current size and peak size of memory blocks traced by tracemalloc. @@ -162,7 +164,7 @@ Returns a tuple: (current: int, peak: int). static PyObject * _tracemalloc_get_traced_memory_impl(PyObject *module) -/*[clinic end generated code: output=5b167189adb9e782 input=61ddb5478400ff66]*/ +/*[clinic end generated code: output=5b167189adb9e782 input=b06e7a1a4914fc21]*/ { return _PyTraceMalloc_GetTracedMemory(); } diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index 4f8d93fd9e867c6..6775ba4826a652b 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -21,8 +21,8 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, " zstd_dict\n" " A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" -"Thread-safe at method level. For one-shot compression, use the compress()\n" -"function instead."); +"Thread-safe at method level. For one-shot compression, use the\n" +"compress() function instead."); static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, @@ -105,9 +105,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_compress__doc__, " Can be these 3 values ZstdCompressor.CONTINUE,\n" " ZstdCompressor.FLUSH_BLOCK, ZstdCompressor.FLUSH_FRAME\n" "\n" -"Return a chunk of compressed data if possible, or b\'\' otherwise. When you have\n" -"finished providing data to the compressor, call the flush() method to finish\n" -"the compression process."); +"Return a chunk of compressed data if possible, or b\'\' otherwise.\n" +"When you have finished providing data to the compressor, call the\n" +"flush() method to finish the compression process."); #define _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF \ {"compress", _PyCFunction_CAST(_zstd_ZstdCompressor_compress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_compress__doc__}, @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since Zstandard data\n" -"consists of one or more independent frames, the compressor object can still\n" -"be used after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard\n" +"data consists of one or more independent frames, the compressor\n" +"object can still be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -262,13 +262,14 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__, " size\n" " The size of the uncompressed data to be provided to the compressor.\n" "\n" -"This method can be used to ensure the header of the frame about to be written\n" -"includes the size of the data, unless the CompressionParameter.content_size_flag\n" -"is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" +"This method can be used to ensure the header of the frame about to\n" +"be written includes the size of the data, unless the\n" +"CompressionParameter.content_size_flag is set to False.\n" +"If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" "\n" -"It is important to ensure that the pledged data size matches the actual data\n" -"size. If they do not match the compressed output data may be corrupted and the\n" -"final chunk written may be lost."); +"It is important to ensure that the pledged data size matches the\n" +"actual data size. If they do not match the compressed output data\n" +"may be corrupted and the final chunk written may be lost."); #define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \ {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__}, @@ -291,4 +292,4 @@ _zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=c1d5c2cf06a8becd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1a5e21476885866c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index c6fdae74ab0447b..fe3b76b8bb369df 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -20,8 +20,8 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" -"Thread-safe at method level. For one-shot decompression, use the decompress()\n" -"function instead."); +"Thread-safe at method level. For one-shot decompression, use the\n" +"decompress() function instead."); static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, @@ -91,7 +91,8 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_unused_data__doc__, "A bytes object of un-consumed input data.\n" "\n" "When ZstdDecompressor object stops after a frame is\n" -"decompressed, unused input data after the frame. Otherwise this will be b\'\'."); +"decompressed, unused input data after the frame. Otherwise this\n" +"will be b\'\'."); #if defined(_zstd_ZstdDecompressor_unused_data_DOCSTR) # undef _zstd_ZstdDecompressor_unused_data_DOCSTR #endif @@ -129,18 +130,19 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, " output buffer is unlimited. When it is nonnegative, returns at\n" " most max_length bytes of decompressed data.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of a frame is reached raises an\n" -"EOFError. Any data found after the end of the frame is ignored and saved in\n" -"the self.unused_data attribute."); +"Attempting to decompress data after the end of a frame is reached\n" +"raises an EOFError. Any data found after the end of the frame is\n" +"ignored and saved in the self.unused_data attribute."); #define _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_zstd_ZstdDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdDecompressor_decompress__doc__}, @@ -220,4 +222,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/ +/*[clinic end generated code: output=70bc308e86463751 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 166d925a542352d..18b049e3cbe37ef 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -21,8 +21,8 @@ PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, " advanced cases. Otherwise, check that the content represents\n" " a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"The dictionary can be used for compression or decompression, and can be shared\n" -"by multiple ZstdCompressor or ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be\n" +"shared by multiple ZstdCompressor or ZstdDecompressor objects."); static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, @@ -125,11 +125,11 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_digested_dict)\n" "\n" -"1. Some advanced compression parameters of compressor may be overridden\n" -" by parameters of digested dictionary.\n" -"2. ZstdDict has a digested dictionaries cache for each compression level.\n" -" It\'s faster when loading again a digested dictionary with the same\n" -" compression level.\n" +"1. Some advanced compression parameters of compressor may be\n" +" overridden by parameters of digested dictionary.\n" +"2. ZstdDict has a digested dictionaries cache for each compression\n" +" level. It\'s faster when loading again a digested dictionary with\n" +" the same compression level.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_digested_dict_DOCSTR) # undef _zstd_ZstdDict_as_digested_dict_DOCSTR @@ -161,9 +161,10 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_undigested_dict)\n" "\n" -"1. The advanced compression parameters of compressor will not be overridden.\n" -"2. Loading an undigested dictionary is costly. If load an undigested dictionary\n" -" multiple times, consider reusing a compressor object.\n" +"1. The advanced compression parameters of compressor will not be\n" +" overridden.\n" +"2. Loading an undigested dictionary is costly. If load an undigested\n" +" dictionary multiple times, consider reusing a compressor object.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_undigested_dict_DOCSTR) # undef _zstd_ZstdDict_as_undigested_dict_DOCSTR @@ -195,9 +196,10 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_prefix)\n" "\n" -"1. Prefix is compatible with long distance matching, while dictionary is not.\n" -"2. It only works for the first frame, then the compressor/decompressor will\n" -" return to no prefix state.\n" +"1. Prefix is compatible with long distance matching, while\n" +" dictionary is not.\n" +"2. It only works for the first frame, then the\n" +" compressor/decompressor will return to no prefix state.\n" "3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR @@ -222,4 +224,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=f41d9e2e2cc2928f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=49b66061b4fcdb5f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 8a3cd182ab15160..b2eb22d9ec8add8 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -332,14 +332,14 @@ _zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new Create a compressor object for compressing data incrementally. -Thread-safe at method level. For one-shot compression, use the compress() -function instead. +Thread-safe at method level. For one-shot compression, use the +compress() function instead. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/ +/*[clinic end generated code: output=cdef61eafecac3d7 input=bbfeeaa06fd3bd4d]*/ { ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { @@ -583,7 +583,6 @@ compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) } /*[clinic input] -@permit_long_docstring_body _zstd.ZstdCompressor.compress data: Py_buffer @@ -593,15 +592,15 @@ _zstd.ZstdCompressor.compress Provide data to the compressor object. -Return a chunk of compressed data if possible, or b'' otherwise. When you have -finished providing data to the compressor, call the flush() method to finish -the compression process. +Return a chunk of compressed data if possible, or b'' otherwise. +When you have finished providing data to the compressor, call the +flush() method to finish the compression process. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, int mode) -/*[clinic end generated code: output=ed7982d1cf7b4f98 input=6018ed6cc729cea6]*/ +/*[clinic end generated code: output=ed7982d1cf7b4f98 input=11726dff64d7b2f9]*/ { PyObject *ret; @@ -643,7 +642,6 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, } /*[clinic input] -@permit_long_docstring_body _zstd.ZstdCompressor.flush mode: int(c_default="ZSTD_e_end") = ZstdCompressor.FLUSH_FRAME @@ -652,14 +650,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since Zstandard data -consists of one or more independent frames, the compressor object can still -be used after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard +data consists of one or more independent frames, the compressor +object can still be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a9871ec742d79003]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=130e0b1eddf0f498]*/ { PyObject *ret; @@ -693,7 +691,7 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _zstd.ZstdCompressor.set_pledged_input_size size: zstd_contentsize @@ -702,19 +700,20 @@ _zstd.ZstdCompressor.set_pledged_input_size Set the uncompressed content size to be written into the frame header. -This method can be used to ensure the header of the frame about to be written -includes the size of the data, unless the CompressionParameter.content_size_flag -is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised. +This method can be used to ensure the header of the frame about to +be written includes the size of the data, unless the +CompressionParameter.content_size_flag is set to False. +If last_mode != FLUSH_FRAME, then a RuntimeError is raised. -It is important to ensure that the pledged data size matches the actual data -size. If they do not match the compressed output data may be corrupted and the -final chunk written may be lost. +It is important to ensure that the pledged data size matches the +actual data size. If they do not match the compressed output data +may be corrupted and the final chunk written may be lost. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, unsigned long long size) -/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=b4c87bcbd5ce6111]*/ +/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=714cd7a9aa10e2a8]*/ { // Error occurred while converting argument, should be unreachable assert(size != ZSTD_CONTENTSIZE_ERROR); diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 46682b483ad06ab..cb95ba89eb650ae 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -469,7 +469,6 @@ stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, /*[clinic input] -@permit_long_docstring_body @classmethod _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new zstd_dict: object = None @@ -479,14 +478,14 @@ _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new Create a decompressor object for decompressing data incrementally. -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. +Thread-safe at method level. For one-shot decompression, use the +decompress() function instead. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=590ca65c1102ff4a input=ed8891edfd14cdaa]*/ +/*[clinic end generated code: output=590ca65c1102ff4a input=73879de69bf89f59]*/ { ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { @@ -571,19 +570,19 @@ ZstdDecompressor_dealloc(PyObject *ob) } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDecompressor.unused_data A bytes object of un-consumed input data. When ZstdDecompressor object stops after a frame is -decompressed, unused input data after the frame. Otherwise this will be b''. +decompressed, unused input data after the frame. Otherwise this +will be b''. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) -/*[clinic end generated code: output=f3a20940f11b6b09 input=37c2c531ab56f914]*/ +/*[clinic end generated code: output=f3a20940f11b6b09 input=0462065c5e60ba01]*/ { PyObject *ret; @@ -613,7 +612,6 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _zstd.ZstdDecompressor.decompress data: Py_buffer @@ -625,25 +623,26 @@ _zstd.ZstdDecompressor.decompress Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of a frame is reached raises an -EOFError. Any data found after the end of the frame is ignored and saved in -the self.unused_data attribute. +Attempting to decompress data after the end of a frame is reached +raises an EOFError. Any data found after the end of the frame is +ignored and saved in the self.unused_data attribute. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=e5c905a774df1553]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=4ddda5a0bdd00673]*/ { PyObject *ret; /* Thread-safe code */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index b0bfbdc886e04fa..e1b9d998e697fb8 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -23,7 +23,6 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #define ZstdDict_CAST(op) ((ZstdDict *)op) /*[clinic input] -@permit_long_docstring_body @classmethod _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new dict_content: Py_buffer @@ -37,14 +36,14 @@ _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new Represents a Zstandard dictionary. -The dictionary can be used for compression or decompression, and can be shared -by multiple ZstdCompressor or ZstdDecompressor objects. +The dictionary can be used for compression or decompression, and can be +shared by multiple ZstdCompressor or ZstdDecompressor objects. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, int is_raw) -/*[clinic end generated code: output=685b7406a48b0949 input=b132ee40b784c293]*/ +/*[clinic end generated code: output=685b7406a48b0949 input=3bb66063c0240433]*/ { /* All dictionaries must be at least 8 bytes */ if (dict_content->len < 8) { @@ -154,7 +153,6 @@ _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self) } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_digested_dict @@ -163,23 +161,22 @@ Load as a digested dictionary to compressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict) -1. Some advanced compression parameters of compressor may be overridden - by parameters of digested dictionary. -2. ZstdDict has a digested dictionaries cache for each compression level. - It's faster when loading again a digested dictionary with the same - compression level. +1. Some advanced compression parameters of compressor may be + overridden by parameters of digested dictionary. +2. ZstdDict has a digested dictionaries cache for each compression + level. It's faster when loading again a digested dictionary with + the same compression level. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=8d01ff0b8b043f2e]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=a9417d40f1d7fedd]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_undigested_dict @@ -188,21 +185,21 @@ Load as an undigested dictionary to compressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict) -1. The advanced compression parameters of compressor will not be overridden. -2. Loading an undigested dictionary is costly. If load an undigested dictionary - multiple times, consider reusing a compressor object. +1. The advanced compression parameters of compressor will not be + overridden. +2. Loading an undigested dictionary is costly. If load an undigested + dictionary multiple times, consider reusing a compressor object. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=b1bdb306c3798ad4]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=56443c9c4e589cd5]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_prefix @@ -211,15 +208,16 @@ Load as a prefix to compressor/decompressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) -1. Prefix is compatible with long distance matching, while dictionary is not. -2. It only works for the first frame, then the compressor/decompressor will - return to no prefix state. +1. Prefix is compatible with long distance matching, while + dictionary is not. +2. It only works for the first frame, then the + compressor/decompressor will return to no prefix state. 3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=77966c012d15e6ab]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=192681a899c6fad0]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 472c59ea8c98826..aefdfe0edc0e1e4 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -1534,13 +1534,13 @@ array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. -The length should be multiplied by the itemsize attribute to calculate -the buffer length in bytes. +The length should be multiplied by the itemsize attribute to +calculate the buffer length in bytes. [clinic start generated code]*/ static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=63d9ad83ba60cda8]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=c2771b9f6a8e1c86]*/ { PyObject* item1 = PyLong_FromVoidPtr(self->ob_item); if (item1 == NULL) { @@ -1572,19 +1572,18 @@ array_array_append_impl(arrayobject *self, PyObject *v) } /*[clinic input] -@permit_long_docstring_body array.array.byteswap Byteswap all items of the array. -If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError -is raised. Note, that for complex types the order of +If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, +RuntimeError is raised. Note, that for complex types the order of components (the real part, followed by imaginary part) is preserved. [clinic start generated code]*/ static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=aafda275f48191d0]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=8732f800e1b47bac]*/ { char *p; Py_ssize_t i; @@ -1967,7 +1966,6 @@ array_array_tobytes_impl(arrayobject *self) } /*[clinic input] -@permit_long_docstring_body array.array.fromunicode ustr: unicode @@ -1975,14 +1973,14 @@ array.array.fromunicode Extends this array with data from the unicode string ustr. -The array must be a unicode type array; otherwise a ValueError is raised. -Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of -some other type. +The array must be a unicode type array; otherwise a ValueError is +raised. Use array.frombytes(ustr.encode(...)) to append Unicode +data to an array of some other type. [clinic start generated code]*/ static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=158d47c302f27ca1]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01fa592ec7b948b6]*/ { const char *typecode = self->ob_descr->typecode; if (strcmp(typecode, "u") != 0 && strcmp(typecode, "w") != 0) { @@ -2030,19 +2028,19 @@ array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) } /*[clinic input] -@permit_long_docstring_body array.array.tounicode Extends this array with data from the unicode string ustr. -Convert the array to a unicode string. The array must be a unicode type array; -otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a -unicode string from an array of some other type. +Convert the array to a unicode string. The array must be a unicode +type array; otherwise a ValueError is raised. Use +array.tobytes().decode() to obtain a unicode string from an array of +some other type. [clinic start generated code]*/ static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=6690997213d219db]*/ +/*[clinic end generated code: output=08e442378336e1ef input=d4d5f398aa71a2be]*/ { const char *typecode = self->ob_descr->typecode; if (strcmp(typecode, "u") != 0 && strcmp(typecode, "w") != 0) { @@ -2678,7 +2676,8 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_TOBYTES_METHODDEF ARRAY_ARRAY_TOUNICODE_METHODDEF ARRAY_ARRAY___SIZEOF___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Arrays are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 673dca6ee134bd8..0e7af135a6f6ce4 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1057,7 +1057,8 @@ binascii.a2b_ascii85 foldspaces: bool = False Allow 'y' as a short form encoding four spaces. adobe: bool = False - Expect data to be wrapped in '<~' and '~>' as in Adobe Ascii85. + Expect data to be terminated with '~>' as in Adobe Ascii85, and + optionally accept leading '<~'. ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. canonical: bool = False @@ -1069,7 +1070,7 @@ Decode Ascii85 data. static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, int adobe, Py_buffer *ignorechars, int canonical) -/*[clinic end generated code: output=09b35f1eac531357 input=dd050604ed30199e]*/ +/*[clinic end generated code: output=09b35f1eac531357 input=08eab2e53c62f1a8]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1264,7 +1265,7 @@ binascii.b2a_ascii85 wrapcol: size_t = 0 Split result into lines of provided width. pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. adobe: bool = False Wrap result in '<~' and '~>' as in Adobe Ascii85. @@ -1274,7 +1275,7 @@ Ascii85-encode data. static PyObject * binascii_b2a_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, size_t wrapcol, int pad, int adobe) -/*[clinic end generated code: output=5ce8fdee843073f4 input=791da754508c7d17]*/ +/*[clinic end generated code: output=5ce8fdee843073f4 input=a77e31d63517bf19]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; @@ -1539,7 +1540,7 @@ binascii.b2a_base85 / * pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. wrapcol: size_t = 0 alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE85_ALPHABET @@ -1549,7 +1550,7 @@ Base85-code line of data. static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, size_t wrapcol, Py_buffer *alphabet) -/*[clinic end generated code: output=98b962ed52c776a4 input=1b20b0bd6572691b]*/ +/*[clinic end generated code: output=98b962ed52c776a4 input=54886d05128d41a8]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b3663180d726e5e..32588b0561e1acc 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -14,10 +14,11 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "\n" "Return an encoded string version of \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" -"values are \'ignore\', \'replace\' and \'xmlcharrefreplace\' as well as any other name\n" -"registered with codecs.register_error that can handle UnicodeEncodeErrors."); +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +"and \'xmlcharrefreplace\' as well as any other name registered with\n" +"codecs.register_error that can handle UnicodeEncodeErrors."); #define _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_multibytecodec_MultibyteCodec_encode), METH_FASTCALL|METH_KEYWORDS, _multibytecodec_MultibyteCodec_encode__doc__}, @@ -103,9 +104,10 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_decode__doc__, "\n" "Decodes \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeDecodeError. Other possible\n" -"values are \'ignore\' and \'replace\' as well as any other name registered with\n" +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeDecodeError. Other possible values are \'ignore\' and\n" +"\'replace\' as well as any other name registered with\n" "codecs.register_error that is able to handle UnicodeDecodeErrors.\""); #define _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF \ @@ -696,4 +698,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=014f4f6bb9d29594 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a84b1544d7d01abb input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index f1124147e2b0a78..32c96c9d2cb3cde 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -582,7 +582,6 @@ multibytecodec_encode(const MultibyteCodec *codec, } /*[clinic input] -@permit_long_docstring_body _multibytecodec.MultibyteCodec.encode input: object @@ -590,17 +589,18 @@ _multibytecodec.MultibyteCodec.encode Return an encoded string version of 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible -values are 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name -registered with codecs.register_error that can handle UnicodeEncodeErrors. +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeEncodeError. Other possible values are 'ignore', 'replace' +and 'xmlcharrefreplace' as well as any other name registered with +codecs.register_error that can handle UnicodeEncodeErrors. [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=0980aede2c564df8]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=980002ed1447697b]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -648,7 +648,6 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, } /*[clinic input] -@permit_long_docstring_body _multibytecodec.MultibyteCodec.decode input: Py_buffer @@ -656,9 +655,10 @@ _multibytecodec.MultibyteCodec.decode Decodes 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeDecodeError. Other possible -values are 'ignore' and 'replace' as well as any other name registered with +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeDecodeError. Other possible values are 'ignore' and +'replace' as well as any other name registered with codecs.register_error that is able to handle UnicodeDecodeErrors." [clinic start generated code]*/ @@ -666,7 +666,7 @@ static PyObject * _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=2c657ef914600c7c]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 04681fa2206a2a5..fa1c57dc26bf853 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -146,9 +146,9 @@ PyDoc_STRVAR(_abc_get_cache_token__doc__, "\n" "Returns the current ABC cache token.\n" "\n" -"The token is an opaque object (supporting equality testing) identifying the\n" -"current version of the ABC cache for virtual subclasses. The token changes\n" -"with every call to register() on any ABC."); +"The token is an opaque object (supporting equality testing) identifying\n" +"the current version of the ABC cache for virtual subclasses. The token\n" +"changes with every call to register() on any ABC."); #define _ABC_GET_CACHE_TOKEN_METHODDEF \ {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, @@ -161,4 +161,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=1989b6716c950e17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b05d599656aeb1e1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 66953d74213b66d..f07a09df5ac7ae0 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -90,7 +90,8 @@ PyDoc_STRVAR(_asyncio_Future_result__doc__, "\n" "If the future has been cancelled, raises CancelledError. If the\n" "future\'s result isn\'t yet available, raises InvalidStateError. If\n" -"the future is done and has an exception set, this exception is raised."); +"the future is done and has an exception set, this exception is\n" +"raised."); #define _ASYNCIO_FUTURE_RESULT_METHODDEF \ {"result", (PyCFunction)_asyncio_Future_result, METH_NOARGS, _asyncio_Future_result__doc__}, @@ -250,8 +251,8 @@ PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "\n" "Add a callback to be run when the future becomes done.\n" "\n" -"The callback is called with a single argument - the future object. If\n" -"the future is already done when this is called, the callback is\n" +"The callback is called with a single argument - the future object.\n" +"If the future is already done when this is called, the callback is\n" "scheduled with call_soon."); #define _ASYNCIO_FUTURE_ADD_DONE_CALLBACK_METHODDEF \ @@ -371,9 +372,9 @@ PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "\n" "Cancel the future and schedule callbacks.\n" "\n" -"If the future is already done or cancelled, return False. Otherwise,\n" -"change the future\'s state to cancelled, schedule the callbacks and\n" -"return True."); +"If the future is already done or cancelled, return False.\n" +"Otherwise, change the future\'s state to cancelled, schedule the\n" +"callbacks and return True."); #define _ASYNCIO_FUTURE_CANCEL_METHODDEF \ {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, @@ -465,8 +466,8 @@ PyDoc_STRVAR(_asyncio_Future_done__doc__, "\n" "Return True if the future is done.\n" "\n" -"Done means either that a result / exception are available, or that the\n" -"future was cancelled."); +"Done means either that a result / exception are available, or that\n" +"the future was cancelled."); #define _ASYNCIO_FUTURE_DONE_METHODDEF \ {"done", (PyCFunction)_asyncio_Future_done, METH_NOARGS, _asyncio_Future_done__doc__}, @@ -2232,4 +2233,4 @@ _asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=b69948ed810591d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32996fb47c48245b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 8f3492cd54b5f2f..454131faf4112b7 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(_bisect_bisect_right__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e <= x, and all e in\n" -"a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just after the rightmost x already there.\n" +"a[i:] have e > x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just after the rightmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -245,8 +245,8 @@ PyDoc_STRVAR(_bisect_bisect_left__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e < x, and all e in\n" -"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just before the leftmost x already there.\n" +"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just before the leftmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -466,4 +466,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=a3c44ed440dd6d81 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62345f14c5c01639 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 2bc6524b6a973b8..30f2c7965e73ae1 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -116,18 +116,19 @@ PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_bz2_BZ2Decompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _bz2_BZ2Decompressor_decompress__doc__}, @@ -237,4 +238,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=552ac6d4c5a101b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1dce5396d592bad7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 9e2a7950ebde64d..4a40dc660b621cc 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -14,9 +14,10 @@ PyDoc_STRVAR(_codecs_register__doc__, "\n" "Register a codec search function.\n" "\n" -"Search functions are expected to take one argument, the encoding name in\n" -"all lower case letters, and either return None, or a tuple of functions\n" -"(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object)."); +"Search functions are expected to take one argument, the encoding\n" +"name in all lower case letters, and either return None, or a tuple\n" +"of functions (encoder, decoder, stream_reader, stream_writer) (or\n" +"a CodecInfo object)."); #define _CODECS_REGISTER_METHODDEF \ {"register", (PyCFunction)_codecs_register, METH_O, _codecs_register__doc__}, @@ -76,10 +77,10 @@ PyDoc_STRVAR(_codecs_encode__doc__, "Encodes obj using the codec registered for encoding.\n" "\n" "The default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_codecs_encode), METH_FASTCALL|METH_KEYWORDS, _codecs_encode__doc__}, @@ -179,10 +180,10 @@ PyDoc_STRVAR(_codecs_decode__doc__, "Decodes obj using the codec registered for encoding.\n" "\n" "Default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(_codecs_decode), METH_FASTCALL|METH_KEYWORDS, _codecs_decode__doc__}, @@ -2649,8 +2650,9 @@ PyDoc_STRVAR(_codecs_register_error__doc__, "Register the specified error handler under the name errors.\n" "\n" "handler must be a callable object, that will be called with an exception\n" -"instance containing information about the location of the encoding/decoding\n" -"error and must return a (replacement, new position) tuple."); +"instance containing information about the location of the\n" +"encoding/decoding error and must return a (replacement, new position)\n" +"tuple."); #define _CODECS_REGISTER_ERROR_METHODDEF \ {"register_error", _PyCFunction_CAST(_codecs_register_error), METH_FASTCALL, _codecs_register_error__doc__}, @@ -2745,8 +2747,8 @@ PyDoc_STRVAR(_codecs_lookup_error__doc__, "\n" "lookup_error(errors) -> handler\n" "\n" -"Return the error handler for the specified error handling name or raise a\n" -"LookupError, if no handler exists under this name."); +"Return the error handler for the specified error handling name or raise\n" +"a LookupError, if no handler exists under this name."); #define _CODECS_LOOKUP_ERROR_METHODDEF \ {"lookup_error", (PyCFunction)_codecs_lookup_error, METH_O, _codecs_lookup_error__doc__}, @@ -2866,4 +2868,4 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=a968c493bb28be3e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=505edef891a06329 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index b5c315c680e7821..6c60678a6fbd51a 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -340,7 +340,7 @@ deque_index(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *v; Py_ssize_t start = 0; - Py_ssize_t stop = Py_SIZE(deque); + Py_ssize_t stop = PY_SSIZE_T_MAX; if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { goto exit; @@ -632,4 +632,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b9d4d647c221cb9f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5a388add99d3d15 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 75cf067c8aa822c..d8b2cba7fd3f891 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -28,7 +28,8 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "\n" "Hide the panel.\n" "\n" -"This does not delete the object, it just makes the window on screen invisible."); +"This does not delete the object, it just makes the window on screen\n" +"invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ {"hide", (PyCFunction)_curses_panel_panel_hide, METH_NOARGS, _curses_panel_panel_hide__doc__}, @@ -328,7 +329,8 @@ PyDoc_STRVAR(_curses_panel_update_panels__doc__, "\n" "Updates the virtual screen after changes in the panel stack.\n" "\n" -"This does not call curses.doupdate(), so you\'ll have to do this yourself."); +"This does not call curses.doupdate(), so you\'ll have to do this\n" +"yourself."); #define _CURSES_PANEL_UPDATE_PANELS_METHODDEF \ {"update_panels", (PyCFunction)_curses_panel_update_panels, METH_NOARGS, _curses_panel_update_panels__doc__}, @@ -341,4 +343,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=db2fe491582784aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62f20ef03eefdf44 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index e6f9798cdf12498..eec9e82739b7787 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -422,10 +422,10 @@ PyDoc_STRVAR(_curses_window_border__doc__, " br\n" " Bottom-right corner.\n" "\n" -"Each parameter specifies the character to use for a specific part of the\n" -"border. The characters can be specified as integers or as one-character\n" -"strings. A 0 value for any parameter will cause the default character to be\n" -"used for that parameter."); +"Each parameter specifies the character to use for a specific part of\n" +"the border. The characters can be specified as integers or as\n" +"one-character strings. A 0 value for any parameter will cause the\n" +"default character to be used for that parameter."); #define _CURSES_WINDOW_BORDER_METHODDEF \ {"border", _PyCFunction_CAST(_curses_window_border), METH_FASTCALL, _curses_window_border__doc__}, @@ -500,8 +500,9 @@ PyDoc_STRVAR(_curses_window_box__doc__, " horch\n" " Top and bottom side.\n" "\n" -"Similar to border(), but both ls and rs are verch and both ts and bs are\n" -"horch. The default corner characters are always used by this function."); +"Similar to border(), but both ls and rs are verch and both ts and bs\n" +"are horch. The default corner characters are always used by this\n" +"function."); #define _CURSES_WINDOW_BOX_METHODDEF \ {"box", (PyCFunction)_curses_window_box, METH_VARARGS, _curses_window_box__doc__}, @@ -593,9 +594,9 @@ PyDoc_STRVAR(_curses_window_derwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"derwin() is the same as calling subwin(), except that begin_y and begin_x\n" -"are relative to the origin of the window, rather than relative to the entire\n" -"screen."); +"derwin() is the same as calling subwin(), except that begin_y and\n" +"begin_x are relative to the origin of the window, rather than\n" +"relative to the entire screen."); #define _CURSES_WINDOW_DERWIN_METHODDEF \ {"derwin", (PyCFunction)_curses_window_derwin, METH_VARARGS, _curses_window_derwin__doc__}, @@ -751,9 +752,10 @@ PyDoc_STRVAR(_curses_window_getch__doc__, " x\n" " X-coordinate.\n" "\n" -"The integer returned does not have to be in ASCII range: function keys,\n" -"keypad keys and so on return numbers higher than 256. In no-delay mode, -1\n" -"is returned if there is no input, else getch() waits until a key is pressed."); +"The integer returned does not have to be in ASCII range: function\n" +"keys, keypad keys and so on return numbers higher than 256. In\n" +"no-delay mode, -1 is returned if there is no input, else getch()\n" +"waits until a key is pressed."); #define _CURSES_WINDOW_GETCH_METHODDEF \ {"getch", (PyCFunction)_curses_window_getch, METH_VARARGS, _curses_window_getch__doc__}, @@ -798,9 +800,10 @@ PyDoc_STRVAR(_curses_window_getkey__doc__, " x\n" " X-coordinate.\n" "\n" -"Returning a string instead of an integer, as getch() does. Function keys,\n" -"keypad keys and other special keys return a multibyte string containing the\n" -"key name. In no-delay mode, an exception is raised if there is no input."); +"Returning a string instead of an integer, as getch() does. Function\n" +"keys, keypad keys and other special keys return a multibyte string\n" +"containing the key name. In no-delay mode, an exception is raised\n" +"if there is no input."); #define _CURSES_WINDOW_GETKEY_METHODDEF \ {"getkey", (PyCFunction)_curses_window_getkey, METH_VARARGS, _curses_window_getkey__doc__}, @@ -969,8 +972,8 @@ PyDoc_STRVAR(_curses_window_insch__doc__, " attr\n" " Attributes for the character.\n" "\n" -"All characters to the right of the cursor are shifted one position right, with\n" -"the rightmost characters on the line being lost."); +"All characters to the right of the cursor are shifted one position\n" +"right, with the rightmost characters on the line being lost."); #define _CURSES_WINDOW_INSCH_METHODDEF \ {"insch", (PyCFunction)_curses_window_insch, METH_VARARGS, _curses_window_insch__doc__}, @@ -1035,7 +1038,8 @@ PyDoc_STRVAR(_curses_window_inch__doc__, " x\n" " X-coordinate.\n" "\n" -"The bottom 8 bits are the character proper, and upper bits are the attributes."); +"The bottom 8 bits are the character proper, and upper bits are the\n" +"attributes."); #define _CURSES_WINDOW_INCH_METHODDEF \ {"inch", (PyCFunction)_curses_window_inch, METH_VARARGS, _curses_window_inch__doc__}, @@ -1084,11 +1088,11 @@ PyDoc_STRVAR(_curses_window_insstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor. All characters to the right of\n" -"the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x,\n" -"if specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor. All characters to the\n" +"right of the cursor are shifted right, with the rightmost characters\n" +"on the line being lost. The cursor position does not change (after\n" +"moving to y, x, if specified)."); #define _CURSES_WINDOW_INSSTR_METHODDEF \ {"insstr", (PyCFunction)_curses_window_insstr, METH_VARARGS, _curses_window_insstr__doc__}, @@ -1159,12 +1163,12 @@ PyDoc_STRVAR(_curses_window_insnstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor, up to n characters. If n is zero\n" -"or negative, the entire string is inserted. All characters to the right\n" -"of the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x, if\n" -"specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor, up to n characters. If\n" +"n is zero or negative, the entire string is inserted. All\n" +"characters to the right of the cursor are shifted right, with the\n" +"rightmost characters on the line being lost. The cursor position\n" +"does not change (after moving to y, x, if specified)."); #define _CURSES_WINDOW_INSNSTR_METHODDEF \ {"insnstr", (PyCFunction)_curses_window_insnstr, METH_VARARGS, _curses_window_insnstr__doc__}, @@ -1230,7 +1234,8 @@ PyDoc_STRVAR(_curses_window_is_linetouched__doc__, " line\n" " Line number.\n" "\n" -"Raise a curses.error exception if line is not valid for the given window."); +"Raise a curses.error exception if line is not valid for the given\n" +"window."); #define _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF \ {"is_linetouched", (PyCFunction)_curses_window_is_linetouched, METH_O, _curses_window_is_linetouched__doc__}, @@ -1260,9 +1265,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_VARARGS, _curses_window_noutrefresh__doc__}, @@ -1314,9 +1319,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_NOARGS, _curses_window_noutrefresh__doc__}, @@ -1336,14 +1341,15 @@ PyDoc_STRVAR(_curses_window_overlay__doc__, "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\n" "Overlay the window on top of destwin.\n" "\n" -"The windows need not be the same size, only the overlapping region is copied.\n" -"This copy is non-destructive, which means that the current background\n" -"character does not overwrite the old contents of destwin.\n" +"The windows need not be the same size, only the overlapping region\n" +"is copied. This copy is non-destructive, which means that the\n" +"current background character does not overwrite the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overlay() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, and the other variables mark a rectangle in the\n" -"destination window."); +"To get fine-grained control over the copied region, the second form\n" +"of overlay() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, and the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERLAY_METHODDEF \ {"overlay", (PyCFunction)_curses_window_overlay, METH_VARARGS, _curses_window_overlay__doc__}, @@ -1394,14 +1400,15 @@ PyDoc_STRVAR(_curses_window_overwrite__doc__, " dmaxcol])\n" "Overwrite the window on top of destwin.\n" "\n" -"The windows need not be the same size, in which case only the overlapping\n" -"region is copied. This copy is destructive, which means that the current\n" -"background character overwrites the old contents of destwin.\n" +"The windows need not be the same size, in which case only the\n" +"overlapping region is copied. This copy is destructive, which means\n" +"that the current background character overwrites the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overwrite() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, the other variables mark a rectangle in the destination\n" -"window."); +"To get fine-grained control over the copied region, the second form\n" +"of overwrite() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERWRITE_METHODDEF \ {"overwrite", (PyCFunction)_curses_window_overwrite, METH_VARARGS, _curses_window_overwrite__doc__}, @@ -1520,16 +1527,17 @@ PyDoc_STRVAR(_curses_window_refresh__doc__, "Update the display immediately.\n" "\n" "Synchronize actual screen with previous drawing/deleting methods.\n" -"The 6 optional arguments can only be specified when the window is a pad\n" -"created with newpad(). The additional parameters are needed to indicate\n" -"what part of the pad and screen are involved. pminrow and pmincol specify\n" -"the upper left-hand corner of the rectangle to be displayed in the pad.\n" -"sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to\n" -"be displayed on the screen. The lower right-hand corner of the rectangle to\n" -"be displayed in the pad is calculated from the screen coordinates, since the\n" -"rectangles must be the same size. Both rectangles must be entirely contained\n" -"within their respective structures. Negative values of pminrow, pmincol,\n" -"sminrow, or smincol are treated as if they were zero."); +"The 6 optional arguments can only be specified when the window is\n" +"a pad created with newpad(). The additional parameters are needed\n" +"to indicate what part of the pad and screen are involved. pminrow\n" +"and pmincol specify the upper left-hand corner of the rectangle to\n" +"be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol\n" +"specify the edges of the rectangle to be displayed on the screen.\n" +"The lower right-hand corner of the rectangle to be displayed in the\n" +"pad is calculated from the screen coordinates, since the rectangles\n" +"must be the same size. Both rectangles must be entirely contained\n" +"within their respective structures. Negative values of pminrow,\n" +"pmincol, sminrow, or smincol are treated as if they were zero."); #define _CURSES_WINDOW_REFRESH_METHODDEF \ {"refresh", (PyCFunction)_curses_window_refresh, METH_VARARGS, _curses_window_refresh__doc__}, @@ -1627,8 +1635,8 @@ PyDoc_STRVAR(_curses_window_subwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the sub-window will extend from the specified position to the\n" -"lower right corner of the window."); +"By default, the sub-window will extend from the specified position\n" +"to the lower right corner of the window."); #define _CURSES_WINDOW_SUBWIN_METHODDEF \ {"subwin", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, @@ -1676,7 +1684,8 @@ PyDoc_STRVAR(_curses_window_scroll__doc__, " lines\n" " Number of lines to scroll.\n" "\n" -"Scroll upward if the argument is positive and downward if it is negative."); +"Scroll upward if the argument is positive and downward if it is\n" +"negative."); #define _CURSES_WINDOW_SCROLL_METHODDEF \ {"scroll", (PyCFunction)_curses_window_scroll, METH_VARARGS, _curses_window_scroll__doc__}, @@ -1715,8 +1724,9 @@ PyDoc_STRVAR(_curses_window_touchline__doc__, "touchline(start, count, [changed=True])\n" "Pretend count lines have been changed, starting with line start.\n" "\n" -"If changed is supplied, it specifies whether the affected lines are marked\n" -"as having been changed (changed=True) or unchanged (changed=False)."); +"If changed is supplied, it specifies whether the affected lines are\n" +"marked as having been changed (changed=True) or unchanged\n" +"(changed=False)."); #define _CURSES_WINDOW_TOUCHLINE_METHODDEF \ {"touchline", (PyCFunction)_curses_window_touchline, METH_VARARGS, _curses_window_touchline__doc__}, @@ -1910,11 +1920,12 @@ PyDoc_STRVAR(_curses_cbreak__doc__, " flag\n" " If false, the effect is the same as calling nocbreak().\n" "\n" -"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering is\n" -"turned off and characters are available to be read one by one. However,\n" -"unlike raw mode, special characters (interrupt, quit, suspend, and flow\n" -"control) retain their effects on the tty driver and calling program.\n" -"Calling first raw() then cbreak() leaves the terminal in cbreak mode."); +"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering\n" +"is turned off and characters are available to be read one by one.\n" +"However, unlike raw mode, special characters (interrupt, quit, suspend,\n" +"and flow control) retain their effects on the tty driver and calling\n" +"program. Calling first raw() then cbreak() leaves the terminal in\n" +"cbreak mode."); #define _CURSES_CBREAK_METHODDEF \ {"cbreak", _PyCFunction_CAST(_curses_cbreak), METH_FASTCALL, _curses_cbreak__doc__}, @@ -1954,8 +1965,9 @@ PyDoc_STRVAR(_curses_color_content__doc__, " color_number\n" " The number of the color (0 - (COLORS-1)).\n" "\n" -"A 3-tuple is returned, containing the R, G, B values for the given color,\n" -"which will be between 0 (no component) and 1000 (maximum amount of component)."); +"A 3-tuple is returned, containing the R, G, B values for the given\n" +"color, which will be between 0 (no component) and 1000 (maximum amount\n" +"of component)."); #define _CURSES_COLOR_CONTENT_METHODDEF \ {"color_content", (PyCFunction)_curses_color_content, METH_O, _curses_color_content__doc__}, @@ -1988,7 +2000,8 @@ PyDoc_STRVAR(_curses_color_pair__doc__, " The number of the color pair.\n" "\n" "This attribute value can be combined with A_STANDOUT, A_REVERSE, and the\n" -"other A_* attributes. pair_number() is the counterpart to this function."); +"other A_* attributes. pair_number() is the counterpart to this\n" +"function."); #define _CURSES_COLOR_PAIR_METHODDEF \ {"color_pair", (PyCFunction)_curses_color_pair, METH_O, _curses_color_pair__doc__}, @@ -2022,9 +2035,9 @@ PyDoc_STRVAR(_curses_curs_set__doc__, " 0 for invisible, 1 for normal visible, or 2 for very visible.\n" "\n" "If the terminal supports the visibility requested, the previous cursor\n" -"state is returned; otherwise, an exception is raised. On many terminals,\n" -"the \"visible\" mode is an underline cursor and the \"very visible\" mode is\n" -"a block cursor."); +"state is returned; otherwise, an exception is raised. On many\n" +"terminals, the \"visible\" mode is an underline cursor and the \"very\n" +"visible\" mode is a block cursor."); #define _CURSES_CURS_SET_METHODDEF \ {"curs_set", (PyCFunction)_curses_curs_set, METH_O, _curses_curs_set__doc__}, @@ -2076,7 +2089,8 @@ PyDoc_STRVAR(_curses_def_shell_mode__doc__, "\n" "Save the current terminal mode as the \"shell\" mode.\n" "\n" -"The \"shell\" mode is the mode when the running program is not using curses.\n" +"The \"shell\" mode is the mode when the running program is not using\n" +"curses.\n" "\n" "Subsequent calls to reset_shell_mode() will restore this mode."); @@ -2150,7 +2164,8 @@ PyDoc_STRVAR(_curses_echo__doc__, " flag\n" " If false, the effect is the same as calling noecho().\n" "\n" -"In echo mode, each character input is echoed to the screen as it is entered."); +"In echo mode, each character input is echoed to the screen as it is\n" +"entered."); #define _CURSES_ECHO_METHODDEF \ {"echo", _PyCFunction_CAST(_curses_echo), METH_FASTCALL, _curses_echo__doc__}, @@ -2223,7 +2238,8 @@ PyDoc_STRVAR(_curses_flash__doc__, "\n" "Flash the screen.\n" "\n" -"That is, change it to reverse-video and then change it back in a short interval."); +"That is, change it to reverse-video and then change it back in a short\n" +"interval."); #define _CURSES_FLASH_METHODDEF \ {"flash", (PyCFunction)_curses_flash, METH_NOARGS, _curses_flash__doc__}, @@ -2243,8 +2259,8 @@ PyDoc_STRVAR(_curses_flushinp__doc__, "\n" "Flush all input buffers.\n" "\n" -"This throws away any typeahead that has been typed by the user and has not\n" -"yet been processed by the program."); +"This throws away any typeahead that has been typed by the user and has\n" +"not yet been processed by the program."); #define _CURSES_FLUSHINP_METHODDEF \ {"flushinp", (PyCFunction)_curses_flushinp, METH_NOARGS, _curses_flushinp__doc__}, @@ -2614,8 +2630,9 @@ PyDoc_STRVAR(_curses_init_pair__doc__, " bg\n" " Background color number (-1 - (COLORS-1)).\n" "\n" -"If the color-pair was previously initialized, the screen is refreshed and\n" -"all occurrences of that color-pair are changed to the new definition."); +"If the color-pair was previously initialized, the screen is refreshed\n" +"and all occurrences of that color-pair are changed to the new\n" +"definition."); #define _CURSES_INIT_PAIR_METHODDEF \ {"init_pair", _PyCFunction_CAST(_curses_init_pair), METH_FASTCALL, _curses_init_pair__doc__}, @@ -2774,9 +2791,9 @@ PyDoc_STRVAR(_curses_get_escdelay__doc__, "\n" "Gets the curses ESCDELAY setting.\n" "\n" -"Gets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Gets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_GET_ESCDELAY_METHODDEF \ {"get_escdelay", (PyCFunction)_curses_get_escdelay, METH_NOARGS, _curses_get_escdelay__doc__}, @@ -2803,9 +2820,9 @@ PyDoc_STRVAR(_curses_set_escdelay__doc__, " ms\n" " length of the delay in milliseconds.\n" "\n" -"Sets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Sets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_SET_ESCDELAY_METHODDEF \ {"set_escdelay", (PyCFunction)_curses_set_escdelay, METH_O, _curses_set_escdelay__doc__}, @@ -2839,8 +2856,8 @@ PyDoc_STRVAR(_curses_get_tabsize__doc__, "\n" "Gets the curses TABSIZE setting.\n" "\n" -"Gets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Gets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_GET_TABSIZE_METHODDEF \ {"get_tabsize", (PyCFunction)_curses_get_tabsize, METH_NOARGS, _curses_get_tabsize__doc__}, @@ -2867,8 +2884,8 @@ PyDoc_STRVAR(_curses_set_tabsize__doc__, " size\n" " rendered cell width of a tab character.\n" "\n" -"Sets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Sets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_SET_TABSIZE_METHODDEF \ {"set_tabsize", (PyCFunction)_curses_set_tabsize, METH_O, _curses_set_tabsize__doc__}, @@ -3039,8 +3056,8 @@ PyDoc_STRVAR(_curses_longname__doc__, "\n" "Return the terminfo long name field describing the current terminal.\n" "\n" -"The maximum length of a verbose description is 128 characters. It is defined\n" -"only after the call to initscr()."); +"The maximum length of a verbose description is 128 characters. It is\n" +"defined only after the call to initscr()."); #define _CURSES_LONGNAME_METHODDEF \ {"longname", (PyCFunction)_curses_longname, METH_NOARGS, _curses_longname__doc__}, @@ -3097,8 +3114,8 @@ PyDoc_STRVAR(_curses_mouseinterval__doc__, " Time in milliseconds.\n" "\n" "Set the maximum time that can elapse between press and release events in\n" -"order for them to be recognized as a click, and return the previous interval\n" -"value."); +"order for them to be recognized as a click, and return the previous\n" +"interval value."); #define _CURSES_MOUSEINTERVAL_METHODDEF \ {"mouseinterval", (PyCFunction)_curses_mouseinterval, METH_O, _curses_mouseinterval__doc__}, @@ -3133,9 +3150,10 @@ PyDoc_STRVAR(_curses_mousemask__doc__, "Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n" "\n" "Return a tuple (availmask, oldmask). availmask indicates which of the\n" -"specified mouse events can be reported; on complete failure it returns 0.\n" -"oldmask is the previous value of the given window\'s mouse event mask.\n" -"If this function is never called, no mouse events are ever reported."); +"specified mouse events can be reported; on complete failure it returns\n" +"0. oldmask is the previous value of the given window\'s mouse event\n" +"mask. If this function is never called, no mouse events are ever\n" +"reported."); #define _CURSES_MOUSEMASK_METHODDEF \ {"mousemask", (PyCFunction)_curses_mousemask, METH_O, _curses_mousemask__doc__}, @@ -3267,8 +3285,8 @@ PyDoc_STRVAR(_curses_newwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the window will extend from the specified position to the lower\n" -"right corner of the screen."); +"By default, the window will extend from the specified position to the\n" +"lower right corner of the screen."); #define _CURSES_NEWWIN_METHODDEF \ {"newwin", (PyCFunction)_curses_newwin, METH_VARARGS, _curses_newwin__doc__}, @@ -3318,8 +3336,9 @@ PyDoc_STRVAR(_curses_nl__doc__, " flag\n" " If false, the effect is the same as calling nonl().\n" "\n" -"This mode translates the return key into newline on input, and translates\n" -"newline into return and line-feed on output. Newline mode is initially on."); +"This mode translates the return key into newline on input, and\n" +"translates newline into return and line-feed on output. Newline mode\n" +"is initially on."); #define _CURSES_NL_METHODDEF \ {"nl", _PyCFunction_CAST(_curses_nl), METH_FASTCALL, _curses_nl__doc__}, @@ -3396,8 +3415,8 @@ PyDoc_STRVAR(_curses_nonl__doc__, "\n" "Leave newline mode.\n" "\n" -"Disable translation of return into newline on input, and disable low-level\n" -"translation of newline into newline/return on output."); +"Disable translation of return into newline on input, and disable\n" +"low-level translation of newline into newline/return on output."); #define _CURSES_NONL_METHODDEF \ {"nonl", (PyCFunction)_curses_nonl, METH_NOARGS, _curses_nonl__doc__}, @@ -3613,8 +3632,8 @@ PyDoc_STRVAR(_curses_raw__doc__, " If false, the effect is the same as calling noraw().\n" "\n" "In raw mode, normal line buffering and processing of interrupt, quit,\n" -"suspend, and flow control keys are turned off; characters are presented to\n" -"curses input functions one by one."); +"suspend, and flow control keys are turned off; characters are presented\n" +"to curses input functions one by one."); #define _CURSES_RAW_METHODDEF \ {"raw", _PyCFunction_CAST(_curses_raw), METH_FASTCALL, _curses_raw__doc__}, @@ -3712,8 +3731,8 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, " ncols\n" " Width.\n" "\n" -"Adjusts other bookkeeping data used by the curses library that record the\n" -"window dimensions (in particular the SIGWINCH handler)."); +"Adjusts other bookkeeping data used by the curses library that record\n" +"the window dimensions (in particular the SIGWINCH handler)."); #define _CURSES_RESIZETERM_METHODDEF \ {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, @@ -3791,10 +3810,11 @@ PyDoc_STRVAR(_curses_resize_term__doc__, " Width.\n" "\n" "When resizing the windows, resize_term() blank-fills the areas that are\n" -"extended. The calling application should fill in these areas with appropriate\n" -"data. The resize_term() function attempts to resize all windows. However,\n" -"due to the calling convention of pads, it is not possible to resize these\n" -"without additional interaction with the application."); +"extended. The calling application should fill in these areas with\n" +"appropriate data. The resize_term() function attempts to resize all\n" +"windows. However, due to the calling convention of pads, it is not\n" +"possible to resize these without additional interaction with the\n" +"application."); #define _CURSES_RESIZE_TERM_METHODDEF \ {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, @@ -3929,12 +3949,12 @@ PyDoc_STRVAR(_curses_start_color__doc__, "\n" "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n" "\n" -"Must be called if the programmer wants to use colors, and before any other\n" -"color manipulation routine is called. It is good practice to call this\n" -"routine right after initscr().\n" +"Must be called if the programmer wants to use colors, and before any\n" +"other color manipulation routine is called. It is good practice to call\n" +"this routine right after initscr().\n" "\n" -"It also restores the colors on the terminal to the values they had when the\n" -"terminal was just turned on."); +"It also restores the colors on the terminal to the values they had when\n" +"the terminal was just turned on."); #define _CURSES_START_COLOR_METHODDEF \ {"start_color", (PyCFunction)_curses_start_color, METH_NOARGS, _curses_start_color__doc__}, @@ -4036,8 +4056,8 @@ PyDoc_STRVAR(_curses_tigetnum__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"The value -2 is returned if capname is not a numeric capability, or -1 if\n" -"it is canceled or absent from the terminal description."); +"The value -2 is returned if capname is not a numeric capability, or -1\n" +"if it is canceled or absent from the terminal description."); #define _CURSES_TIGETNUM_METHODDEF \ {"tigetnum", (PyCFunction)_curses_tigetnum, METH_O, _curses_tigetnum__doc__}, @@ -4079,8 +4099,8 @@ PyDoc_STRVAR(_curses_tigetstr__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"None is returned if capname is not a string capability, or is canceled or\n" -"absent from the terminal description."); +"None is returned if capname is not a string capability, or is canceled\n" +"or absent from the terminal description."); #define _CURSES_TIGETSTR_METHODDEF \ {"tigetstr", (PyCFunction)_curses_tigetstr, METH_O, _curses_tigetstr__doc__}, @@ -4234,14 +4254,14 @@ PyDoc_STRVAR(_curses_use_env__doc__, "\n" "Use environment variables LINES and COLUMNS.\n" "\n" -"If used, this function should be called before initscr() or newterm() are\n" -"called.\n" +"If used, this function should be called before initscr() or newterm()\n" +"are called.\n" "\n" -"When flag is False, the values of lines and columns specified in the terminfo\n" -"database will be used, even if environment variables LINES and COLUMNS (used\n" -"by default) are set, or if curses is running in a window (in which case\n" -"default behavior would be to use the window size if LINES and COLUMNS are\n" -"not set)."); +"When flag is False, the values of lines and columns specified in the\n" +"terminfo database will be used, even if environment variables LINES and\n" +"COLUMNS (used by default) are set, or if curses is running in a window\n" +"(in which case default behavior would be to use the window size if LINES\n" +"and COLUMNS are not set)."); #define _CURSES_USE_ENV_METHODDEF \ {"use_env", (PyCFunction)_curses_use_env, METH_O, _curses_use_env__doc__}, @@ -4450,4 +4470,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=135246e29163510c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e7c7932f4a4e9bce input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index ee621c150c31e46..fac41e7aefc7f4f 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -214,8 +214,8 @@ PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "\n" "Create a date from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATE_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", (PyCFunction)datetime_date_fromtimestamp, METH_O|METH_CLASS, datetime_date_fromtimestamp__doc__}, @@ -897,8 +897,8 @@ PyDoc_STRVAR(datetime_time_isoformat__doc__, "\n" "Return the time formatted according to ISO.\n" "\n" -"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the fractional\n" -"part is omitted if self.microsecond == 0.\n" +"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the\n" +"fractional part is omitted if self.microsecond == 0.\n" "\n" "The optional argument timespec specifies the number of additional\n" "terms of the time to include. Valid options are \'auto\', \'hours\',\n" @@ -979,7 +979,8 @@ PyDoc_STRVAR(datetime_time_strftime__doc__, "\n" "Format using strftime().\n" "\n" -"The date part of the timestamp passed to underlying strftime should not be used.\n" +"The date part of the timestamp passed to underlying strftime should\n" +"not be used.\n" "\n" "For a list of supported format codes, see the documentation:\n" " https://docs.python.org/3/library/datetime.html#format-codes"); @@ -1269,8 +1270,8 @@ PyDoc_STRVAR(datetime_datetime__doc__, "\n" "A combination of a date and a time.\n" "\n" -"The year, month and day arguments are required. tzinfo may be None, or an\n" -"instance of a tzinfo subclass. The remaining arguments may be ints."); +"The year, month and day arguments are required. tzinfo may be None, or\n" +"an instance of a tzinfo subclass. The remaining arguments may be ints."); static PyObject * datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, @@ -1491,8 +1492,8 @@ PyDoc_STRVAR(datetime_datetime_fromtimestamp__doc__, "\n" "Create a datetime from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", _PyCFunction_CAST(datetime_datetime_fromtimestamp), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_fromtimestamp__doc__}, @@ -2090,4 +2091,4 @@ datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); } -/*[clinic end generated code: output=69658acff6a43ac4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f63509398651723 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 091ce9edc43d4bd..6c979a4b0081df3 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -113,7 +113,8 @@ PyDoc_STRVAR(_dbm_dbm_setdefault__doc__, "\n" "Return the value for key if present, otherwise default.\n" "\n" -"If key is not in the database, it is inserted with default as the value."); +"If key is not in the database, it is inserted with default as the\n" +"value."); #define _DBM_DBM_SETDEFAULT_METHODDEF \ {"setdefault", _PyCFunction_CAST(_dbm_dbm_setdefault), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _dbm_dbm_setdefault__doc__}, @@ -246,4 +247,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=279511ea7cda38dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=677deecf525167a5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 23f666310850312..5f350c864f057b0 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -71,14 +71,15 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } PyDoc_STRVAR(_functools_reduce__doc__, -"reduce($module, function, iterable, /, initial=<unrepresentable>)\n" +"reduce($module, function, iterable, /,\n" +" initial=functools._initial_missing)\n" "--\n" "\n" "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" "\n" -"This effectively reduces the iterable to a single value. If initial is present,\n" -"it is placed before the items of the iterable in the calculation, and serves as\n" -"a default when the iterable is empty.\n" +"This effectively reduces the iterable to a single value. If initial is\n" +"present, it is placed before the items of the iterable in the\n" +"calculation, and serves as a default when the iterable is empty.\n" "\n" "For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" "calculates ((((1 + 2) + 3) + 4) + 5)."); @@ -192,4 +193,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=7f2abc718fcc35d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d8fdaeba4b520fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 6fd6aa3da503351..fe993cc328fbd2f 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -138,9 +138,9 @@ PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, "\n" "Return the starting key for the traversal.\n" "\n" -"It\'s possible to loop over every key in the database using this method\n" -"and the nextkey() method. The traversal is ordered by GDBM\'s internal\n" -"hash values, and won\'t be sorted by the key values."); +"It\'s possible to loop over every key in the database using this\n" +"method and the nextkey() method. The traversal is ordered by GDBM\'s\n" +"internal hash values, and won\'t be sorted by the key values."); #define _GDBM_GDBM_FIRSTKEY_METHODDEF \ {"firstkey", _PyCFunction_CAST(_gdbm_gdbm_firstkey), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_firstkey__doc__}, @@ -171,8 +171,8 @@ PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, "\n" "Returns the key that follows key in the traversal.\n" "\n" -"The following code prints every key in the database db, without having\n" -"to create a list in memory that contains them all:\n" +"The following code prints every key in the database db, without\n" +"having to create a list in memory that contains them all:\n" "\n" " k = db.firstkey()\n" " while k is not None:\n" @@ -226,9 +226,9 @@ PyDoc_STRVAR(_gdbm_gdbm_reorganize__doc__, "\n" "If you have carried out a lot of deletions and would like to shrink\n" "the space used by the GDBM file, this routine will reorganize the\n" -"database. GDBM will not shorten the length of a database file except\n" -"by using this reorganization; otherwise, deleted file space will be\n" -"kept and reused as new (key,value) pairs are added."); +"database. GDBM will not shorten the length of a database file\n" +"except by using this reorganization; otherwise, deleted file space\n" +"will be kept and reused as new (key,value) pairs are added."); #define _GDBM_GDBM_REORGANIZE_METHODDEF \ {"reorganize", _PyCFunction_CAST(_gdbm_gdbm_reorganize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_reorganize__doc__}, @@ -389,4 +389,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=8bca34ce9d4493dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=429b5db24568292e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 7ae7be185eceb58..17ed37bbd3b6191 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1865,8 +1865,8 @@ PyDoc_STRVAR(_hashlib_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments."); +"This may be used to exchange the value safely in email or other\n" +"non-binary environments."); #define _HASHLIB_HMAC_HEXDIGEST_METHODDEF \ {"hexdigest", (PyCFunction)_hashlib_HMAC_hexdigest, METH_NOARGS, _hashlib_HMAC_hexdigest__doc__}, @@ -1887,8 +1887,8 @@ PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, "Determine the OpenSSL FIPS mode of operation.\n" "\n" "For OpenSSL 3.0.0 and newer it returns the state of the default provider\n" -"in the default OSSL context. It\'s not quite the same as FIPS_mode() but good\n" -"enough for unittests.\n" +"in the default OSSL context. It\'s not quite the same as FIPS_mode() but\n" +"good enough for unittests.\n" "\n" "Effectively any non-zero return value indicates FIPS mode;\n" "values other than 1 may have additional significance."); @@ -1986,4 +1986,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #define _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #endif /* !defined(_HASHLIB_OPENSSL_SHAKE_256_METHODDEF) */ -/*[clinic end generated code: output=9ba35fcc33795b1e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cf405e652a340bb2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index b43155b6c24e3ce..f3b8256efc02216 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -326,8 +326,8 @@ PyDoc_STRVAR(_heapq_heappushpop_max__doc__, "\n" "Maxheap variant of heappushpop.\n" "\n" -"The combined action runs more efficiently than heappush_max() followed by\n" -"a separate call to heappop_max()."); +"The combined action runs more efficiently than heappush_max()\n" +"followed by a separate call to heappop_max()."); #define _HEAPQ_HEAPPUSHPOP_MAX_METHODDEF \ {"heappushpop_max", _PyCFunction_CAST(_heapq_heappushpop_max), METH_FASTCALL, _heapq_heappushpop_max__doc__}, @@ -358,4 +358,4 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=e83d50002c29a96d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=21e4f248ef6e83d6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpqueuesmodule.c.h b/Modules/clinic/_interpqueuesmodule.c.h index 3f08a0cb6d36ab1..6f27ea73b8866fe 100644 --- a/Modules/clinic/_interpqueuesmodule.c.h +++ b/Modules/clinic/_interpqueuesmodule.c.h @@ -109,7 +109,8 @@ PyDoc_STRVAR(_interpqueues_destroy__doc__, "\n" "Clear and destroy the queue.\n" "\n" -"Afterward attempts to use the queue will behave as though it never existed."); +"Afterward attempts to use the queue will behave as though it never\n" +"existed."); #define _INTERPQUEUES_DESTROY_METHODDEF \ {"destroy", _PyCFunction_CAST(_interpqueues_destroy), METH_FASTCALL|METH_KEYWORDS, _interpqueues_destroy__doc__}, @@ -762,4 +763,4 @@ _interpqueues__register_heap_types(PyObject *module, PyObject *const *args, Py_s exit: return return_value; } -/*[clinic end generated code: output=64cea8e1063429b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7e56e5b0c684d294 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index d70ffcea527895a..72792f9b583a66c 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -541,7 +541,8 @@ PyDoc_STRVAR(_interpreters_run_func__doc__, "Execute the body of the provided function in the identified interpreter.\n" "\n" "Code objects are also supported. In both cases, closures and args\n" -"are not supported. Methods and other callables are not supported either.\n" +"are not supported. Methods and other callables are not supported\n" +"either.\n" "\n" "(See _interpreters.exec().)"); @@ -1139,8 +1140,9 @@ PyDoc_STRVAR(_interpreters_capture_exception__doc__, "\n" "Return a snapshot of an exception.\n" "\n" -"If \"exc\" is None then the current exception, if any, is used (but not cleared).\n" -"The returned snapshot is the same as what _interpreters.exec() returns."); +"If \"exc\" is None then the current exception, if any, is used (but not\n" +"cleared). The returned snapshot is the same as what\n" +"_interpreters.exec() returns."); #define _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF \ {"capture_exception", _PyCFunction_CAST(_interpreters_capture_exception), METH_FASTCALL|METH_KEYWORDS, _interpreters_capture_exception__doc__}, @@ -1198,4 +1200,4 @@ _interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=c80f73761f860f6c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c3ca09c304378ad input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_json.c.h b/Modules/clinic/_json.c.h index cd37a236c7611a1..d3ed1dfe0515a0a 100644 --- a/Modules/clinic/_json.c.h +++ b/Modules/clinic/_json.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(py_scanstring__doc__, "ValueError on attempt to decode an invalid string. If strict is False\n" "then literal control characters are allowed in the string.\n" "\n" -"Returns a tuple of the decoded string and the index of the character in s\n" -"after the end quote."); +"Returns a tuple of the decoded string and the index of the character in\n" +"s after the end quote."); #define PY_SCANSTRING_METHODDEF \ {"scanstring", _PyCFunction_CAST(py_scanstring), METH_FASTCALL, py_scanstring__doc__}, @@ -125,4 +125,4 @@ py_encode_basestring(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5bdd16375c95a4d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea6e9a271d4ceaf2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index ebdc81a0dac2f05..bba107e8f806daf 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -74,18 +74,19 @@ PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_lzma_LZMADecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _lzma_LZMADecompressor_decompress__doc__}, @@ -333,4 +334,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=6386084cb43d2533 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ffc6d673d858048c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index d52629cf6eaa5b7..e7e9707f182a2ed 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -14,15 +14,15 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "\n" "Spawn a fresh new child process.\n" "\n" -"Fork a child process, close parent file descriptors as appropriate in the\n" -"child and duplicate the few that are needed before calling exec() in the\n" -"child process.\n" +"Fork a child process, close parent file descriptors as appropriate in\n" +"the child and duplicate the few that are needed before calling exec() in\n" +"the child process.\n" "\n" -"If close_fds is True, close file descriptors 3 and higher, except those listed\n" -"in the sorted tuple pass_fds.\n" +"If close_fds is True, close file descriptors 3 and higher, except those\n" +"listed in the sorted tuple pass_fds.\n" "\n" -"The preexec_fn, if supplied, will be called immediately before closing file\n" -"descriptors and exec.\n" +"The preexec_fn, if supplied, will be called immediately before closing\n" +"file descriptors and exec.\n" "\n" "WARNING: preexec_fn is NOT SAFE if your application uses threads.\n" " It may trigger infrequent, difficult to debug deadlocks.\n" @@ -150,4 +150,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=942bc2748a9c2785 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=138941c284792aa1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index c9482f40acb9d4c..b67dd23f260c9e2 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -44,8 +44,9 @@ PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, "\n" "Put the item on the queue.\n" "\n" -"The optional \'block\' and \'timeout\' arguments are ignored, as this method\n" -"never blocks. They are provided for compatibility with the Queue class."); +"The optional \'block\' and \'timeout\' arguments are ignored, as this\n" +"method never blocks. They are provided for compatibility with the\n" +"Queue class."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", _PyCFunction_CAST(_queue_SimpleQueue_put), METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, @@ -188,10 +189,11 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "\n" "Remove and return an item from the queue.\n" "\n" -"If optional args \'block\' is true and \'timeout\' is None (the default),\n" -"block if necessary until an item is available. If \'timeout\' is\n" -"a non-negative number, it blocks at most \'timeout\' seconds and raises\n" -"the Empty exception if no item was available within that time.\n" +"If optional args \'block\' is true and \'timeout\' is None (the\n" +"default), block if necessary until an item is available. If\n" +"\'timeout\' is a non-negative number, it blocks at most \'timeout\'\n" +"seconds and raises the Empty exception if no item was available\n" +"within that time.\n" "Otherwise (\'block\' is false), return an item if one is immediately\n" "available, else raise the Empty exception (\'timeout\' is ignored\n" "in that case)."); @@ -388,4 +390,4 @@ _queue_SimpleQueue___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=4af5d1b1ea31ac7d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8219fe2f2ed5f068 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 2563a16aea0b6f9..ca9cad7a572dadf 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -143,4 +143,35 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ + +static int +random_init_impl(RandomObject *self, PyObject *seed); + +static int +random_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = (PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type; + PyObject *seed = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Random", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Random", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + seed = PyTuple_GET_ITEM(args, 0); +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = random_init_impl((RandomObject *)self, seed); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec95f7df0c3f3c19 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 8c35c8443b775ae..e337ed2390a1fc4 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -334,9 +334,10 @@ _ssl__SSLSocket_compression(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This changes the context associated with the SSLSocket.\n" "\n" -"This is typically used from within a callback function set by the sni_callback\n" -"on the SSLContext to change the certificate information associated with the\n" -"SSLSocket before the cryptographic exchange handshake messages."); +"This is typically used from within a callback function set by the\n" +"sni_callback on the SSLContext to change the certificate information\n" +"associated with the SSLSocket before the cryptographic exchange\n" +"handshake messages."); #if defined(_ssl__SSLSocket_context_DOCSTR) # undef _ssl__SSLSocket_context_DOCSTR #endif @@ -571,9 +572,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, "\n" "Write size bytes from offset in the file descriptor fd to the SSL connection.\n" "\n" -"This method uses the zero-copy technique and returns the number of bytes\n" -"written. It should be called only when Kernel TLS is used for sending data in\n" -"the connection.\n" +"This method uses the zero-copy technique and returns the number of\n" +"bytes written. It should be called only when Kernel TLS is used for\n" +"sending data in the connection.\n" "\n" "The meaning of flags is platform dependent."); @@ -762,8 +763,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, "\n" "Get channel binding data for current connection.\n" "\n" -"Raise ValueError if the requested `cb_type` is not supported. Return bytes\n" -"of the data or None if the data is not available (e.g. before the handshake).\n" +"Raise ValueError if the requested `cb_type` is not supported.\n" +"Return bytes of the data or None if the data is not available (e.g.\n" +"before the handshake).\n" "Only \'tls-unique\' channel binding data from RFC 5929 is supported."); #define _SSL__SSLSOCKET_GET_CHANNEL_BINDING_METHODDEF \ @@ -2210,8 +2212,9 @@ _ssl__SSLContext_set_ecdh_curve(PyObject *self, PyObject *name) PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" "\n" -"If the argument is None then the callback is disabled. The method is called\n" -"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"If the argument is None then the callback is disabled. The method\n" +"is called with the SSLSocket, the server name as a string, and the\n" +"SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); #if defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -2275,11 +2278,11 @@ PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "\n" "Returns quantities of loaded X.509 certificates.\n" "\n" -"X.509 certificates with a CA extension and certificate revocation lists\n" -"inside the context\'s cert store.\n" +"X.509 certificates with a CA extension and certificate revocation\n" +"lists inside the context\'s cert store.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF \ {"cert_store_stats", (PyCFunction)_ssl__SSLContext_cert_store_stats, METH_NOARGS, _ssl__SSLContext_cert_store_stats__doc__}, @@ -2305,11 +2308,11 @@ PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, "\n" "Returns a list of dicts with information of loaded CA certs.\n" "\n" -"If the optional argument is True, returns a DER-encoded copy of the CA\n" -"certificate.\n" +"If the optional argument is True, returns a DER-encoded copy of the\n" +"CA certificate.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF \ {"get_ca_certs", _PyCFunction_CAST(_ssl__SSLContext_get_ca_certs), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_get_ca_certs__doc__}, @@ -2970,8 +2973,8 @@ PyDoc_STRVAR(_ssl_RAND_status__doc__, "\n" "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n" "\n" -"It is necessary to seed the PRNG with RAND_add() on some platforms before\n" -"using the ssl() function."); +"It is necessary to seed the PRNG with RAND_add() on some platforms\n" +"before using the ssl() function."); #define _SSL_RAND_STATUS_METHODDEF \ {"RAND_status", (PyCFunction)_ssl_RAND_status, METH_NOARGS, _ssl_RAND_status__doc__}, @@ -3157,11 +3160,11 @@ PyDoc_STRVAR(_ssl_enum_certificates__doc__, "\n" "Retrieve certificates from Windows\' cert store.\n" "\n" -"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may provide\n" -"more cert storages, too. The function returns a list of (bytes,\n" -"encoding_type, trust) tuples. The encoding_type flag can be interpreted\n" -"with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either\n" -"a set of OIDs or the boolean True."); +"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may\n" +"provide more cert storages, too. The function returns a list of\n" +"(bytes, encoding_type, trust) tuples. The encoding_type flag can be\n" +"interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The\n" +"trust setting is either a set of OIDs or the boolean True."); #define _SSL_ENUM_CERTIFICATES_METHODDEF \ {"enum_certificates", _PyCFunction_CAST(_ssl_enum_certificates), METH_FASTCALL|METH_KEYWORDS, _ssl_enum_certificates__doc__}, @@ -3323,4 +3326,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=e29d5ada294f97bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aef2e74b706c6106 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 311b64094767110..a0e4c66451101ed 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "\n" "Return the module of the defining class.\n" "\n" -"Also tests that result of PyType_GetModuleByDef matches defining_class\'s\n" -"module."); +"Also tests that result of PyType_GetModuleByDef matches\n" +"defining_class\'s module."); #define _TESTMULTIPHASE_STATEACCESSTYPE_GET_DEFINING_MODULE_METHODDEF \ {"get_defining_module", _PyCFunction_CAST(_testmultiphase_StateAccessType_get_defining_module), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testmultiphase_StateAccessType_get_defining_module__doc__}, @@ -165,4 +165,4 @@ _testmultiphase_StateAccessType_get_count(PyObject *self, PyTypeObject *cls, PyO } return _testmultiphase_StateAccessType_get_count_impl((StateAccessTypeObject *)self, cls); } -/*[clinic end generated code: output=8eed2f14292ec986 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aff91f6219a7baca input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h index 8455f9929babc19..926bea8e1e419a4 100644 --- a/Modules/clinic/_threadmodule.c.h +++ b/Modules/clinic/_threadmodule.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_thread_lock_acquire__doc__, "\n" "Lock the lock.\n" "\n" -"Without argument, this blocks if the lock is already\n" -"locked (even by the same thread), waiting for another thread to release\n" +"Without argument, this blocks if the lock is already locked\n" +"(even by the same thread), waiting for another thread to release\n" "the lock, and return True once the lock is acquired.\n" "With an argument, this will only block if the argument is true,\n" "and the return value reflects whether the lock is acquired.\n" @@ -445,14 +445,14 @@ PyDoc_STRVAR(_thread_RLock_release__doc__, "\n" "Release the lock.\n" "\n" -"Allows another thread that is blocked waiting for\n" -"the lock to acquire the lock. The lock must be in the locked state,\n" +"Allows another thread that is blocked waiting for the lock\n" +"to acquire the lock. The lock must be in the locked state,\n" "and must be locked by the same thread that unlocks it; otherwise a\n" "`RuntimeError` is raised.\n" "\n" -"Do note that if the lock was acquire()d several times in a row by the\n" -"current thread, release() needs to be called as many times for the lock\n" -"to be available for other threads."); +"Do note that if the lock was acquire()d several times in a row by\n" +"the current thread, release() needs to be called as many times for\n" +"the lock to be available for other threads."); #define _THREAD_RLOCK_RELEASE_METHODDEF \ {"release", (PyCFunction)_thread_RLock_release, METH_NOARGS, _thread_RLock_release__doc__}, @@ -740,4 +740,4 @@ _thread_set_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _THREAD_SET_NAME_METHODDEF #define _THREAD_SET_NAME_METHODDEF #endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ -/*[clinic end generated code: output=1255a1520f43f97a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f1707cbafc0e8f2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 352c2b9e3d410c4..3147cf7f9d2cff3 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -907,7 +907,8 @@ PyDoc_STRVAR(_tkinter_setbusywaitinterval__doc__, "\n" "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n" "\n" -"It should be set to a divisor of the maximum time between frames in an animation."); +"It should be set to a divisor of the maximum time between frames in\n" +"an animation."); #define _TKINTER_SETBUSYWAITINTERVAL_METHODDEF \ {"setbusywaitinterval", (PyCFunction)_tkinter_setbusywaitinterval, METH_O, _tkinter_setbusywaitinterval__doc__}, @@ -966,4 +967,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=052c067aa69237be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c807adb73e305725 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index eec47ab2b1f9e1d..1e9f317853d6c2e 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -292,8 +292,8 @@ PyDoc_STRVAR(array_array_buffer_info__doc__, "\n" "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array\'s contents.\n" "\n" -"The length should be multiplied by the itemsize attribute to calculate\n" -"the buffer length in bytes."); +"The length should be multiplied by the itemsize attribute to\n" +"calculate the buffer length in bytes."); #define ARRAY_ARRAY_BUFFER_INFO_METHODDEF \ {"buffer_info", (PyCFunction)array_array_buffer_info, METH_NOARGS, array_array_buffer_info__doc__}, @@ -335,8 +335,8 @@ PyDoc_STRVAR(array_array_byteswap__doc__, "\n" "Byteswap all items of the array.\n" "\n" -"If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError\n" -"is raised. Note, that for complex types the order of\n" +"If the items in the array are not 1, 2, 4, 8 or 16 bytes in size,\n" +"RuntimeError is raised. Note, that for complex types the order of\n" "components (the real part, followed by imaginary part) is preserved."); #define ARRAY_ARRAY_BYTESWAP_METHODDEF \ @@ -572,9 +572,9 @@ PyDoc_STRVAR(array_array_fromunicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"The array must be a unicode type array; otherwise a ValueError is raised.\n" -"Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of\n" -"some other type."); +"The array must be a unicode type array; otherwise a ValueError is\n" +"raised. Use array.frombytes(ustr.encode(...)) to append Unicode\n" +"data to an array of some other type."); #define ARRAY_ARRAY_FROMUNICODE_METHODDEF \ {"fromunicode", (PyCFunction)array_array_fromunicode, METH_O, array_array_fromunicode__doc__}, @@ -605,9 +605,10 @@ PyDoc_STRVAR(array_array_tounicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"Convert the array to a unicode string. The array must be a unicode type array;\n" -"otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\n" -"unicode string from an array of some other type."); +"Convert the array to a unicode string. The array must be a unicode\n" +"type array; otherwise a ValueError is raised. Use\n" +"array.tobytes().decode() to obtain a unicode string from an array of\n" +"some other type."); #define ARRAY_ARRAY_TOUNICODE_METHODDEF \ {"tounicode", (PyCFunction)array_array_tounicode, METH_NOARGS, array_array_tounicode__doc__}, @@ -780,4 +781,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=8699475b51151247 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32784678e77ac658 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index ed695758ef998c9..29fa9e87de87c7a 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -372,7 +372,8 @@ PyDoc_STRVAR(binascii_a2b_ascii85__doc__, " foldspaces\n" " Allow \'y\' as a short form encoding four spaces.\n" " adobe\n" -" Expect data to be wrapped in \'<~\' and \'~>\' as in Adobe Ascii85.\n" +" Expect data to be terminated with \'~>\' as in Adobe Ascii85, and\n" +" optionally accept leading \'<~\'.\n" " ignorechars\n" " A byte string containing characters to ignore from the input.\n" " canonical\n" @@ -492,7 +493,7 @@ PyDoc_STRVAR(binascii_b2a_ascii85__doc__, " wrapcol\n" " Split result into lines of provided width.\n" " pad\n" -" Pad input to a multiple of 4 before encoding.\n" +" Retain zero-padding bytes at end of output.\n" " adobe\n" " Wrap result in \'<~\' and \'~>\' as in Adobe Ascii85."); @@ -709,7 +710,7 @@ PyDoc_STRVAR(binascii_b2a_base85__doc__, "Base85-code line of data.\n" "\n" " pad\n" -" Pad input to a multiple of 4 before encoding."); +" Retain zero-padding bytes at end of output."); #define BINASCII_B2A_BASE85_METHODDEF \ {"b2a_base85", _PyCFunction_CAST(binascii_b2a_base85), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base85__doc__}, @@ -1684,4 +1685,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=b41544f39b0ef681 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42dd48f323cbb118 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 7f9e65baf120ea2..ecb5257cf6b2408 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,8 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base is not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e)\n" +"of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -882,11 +883,12 @@ PyDoc_STRVAR(cmath_isclose__doc__, "\n" "Return True if a is close in value to b, and False otherwise.\n" "\n" -"For the values to be considered close, the difference between them must be\n" -"smaller than at least one of the tolerances.\n" +"For the values to be considered close, the difference between them must\n" +"be smaller than at least one of the tolerances.\n" "\n" -"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is\n" -"not close to anything, even itself. inf and -inf are only close to themselves."); +"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is,\n" +"NaN is not close to anything, even itself. inf and -inf are only close\n" +"to themselves."); #define CMATH_ISCLOSE_METHODDEF \ {"isclose", _PyCFunction_CAST(cmath_isclose), METH_FASTCALL|METH_KEYWORDS, cmath_isclose__doc__}, @@ -985,4 +987,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=631db17fb1c79d66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d5ad4cf258526cd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/faulthandler.c.h b/Modules/clinic/faulthandler.c.h index e06cfdcfba29936..07a9dd7dc561e12 100644 --- a/Modules/clinic/faulthandler.c.h +++ b/Modules/clinic/faulthandler.c.h @@ -334,9 +334,9 @@ PyDoc_STRVAR(faulthandler_dump_traceback_later__doc__, "\n" "Dump the traceback of all threads in timeout seconds.\n" "\n" -"If repeat is true, the tracebacks of all threads are dumped every timeout\n" -"seconds. If exit is true, call _exit(1) which is not safe. max_threads\n" -"caps the number of threads dumped."); +"If repeat is true, the tracebacks of all threads are dumped every\n" +"timeout seconds. If exit is true, call _exit(1) which is not safe.\n" +"max_threads caps the number of threads dumped."); #define FAULTHANDLER_DUMP_TRACEBACK_LATER_METHODDEF \ {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_later__doc__}, @@ -782,4 +782,4 @@ faulthandler__raise_exception(PyObject *module, PyObject *const *args, Py_ssize_ #ifndef FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #define FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #endif /* !defined(FAULTHANDLER__RAISE_EXCEPTION_METHODDEF) */ -/*[clinic end generated code: output=2452d767c85130a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=14815a5f8afe813f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 08275e35413f667..aa743c8f40a565f 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -376,8 +376,8 @@ PyDoc_STRVAR(gc_get_objects__doc__, " generation\n" " Generation to extract the objects from.\n" "\n" -"If generation is not None, return only the objects tracked by the collector\n" -"that are in that generation."); +"If generation is not None, return only the objects tracked by the\n" +"collector that are in that generation."); #define GC_GET_OBJECTS_METHODDEF \ {"get_objects", _PyCFunction_CAST(gc_get_objects), METH_FASTCALL|METH_KEYWORDS, gc_get_objects__doc__}, @@ -520,9 +520,10 @@ PyDoc_STRVAR(gc_freeze__doc__, "\n" "Freeze all current tracked objects and ignore them for future collections.\n" "\n" -"This can be used before a POSIX fork() call to make the gc copy-on-write friendly.\n" -"Note: collection before a POSIX fork() call may free pages for future allocation\n" -"which can cause copy-on-write."); +"This can be used before a POSIX fork() call to make the gc copy-on-write\n" +"friendly.\n" +"Note: collection before a POSIX fork() call may free pages for future\n" +"allocation which can cause copy-on-write."); #define GC_FREEZE_METHODDEF \ {"freeze", (PyCFunction)gc_freeze, METH_NOARGS, gc_freeze__doc__}, @@ -583,4 +584,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=19738854607938db input=a9049054013a1b77]*/ +/*[clinic end generated code: output=756c0e7719b76971 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/hmacmodule.c.h b/Modules/clinic/hmacmodule.c.h index 1ceb2d809e830a2..a31d2ab7f04463c 100644 --- a/Modules/clinic/hmacmodule.c.h +++ b/Modules/clinic/hmacmodule.c.h @@ -187,8 +187,8 @@ PyDoc_STRVAR(_hmac_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments.\n" +"This may be used to exchange the value safely in email or other\n" +"non-binary environments.\n" "\n" "This method may raise a MemoryError."); @@ -670,4 +670,4 @@ _hmac_compute_blake2b_32(PyObject *module, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=30c0614482d963f5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6ec5948df1c5569a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 49816bfcb42feca..d9917b47dc90376 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -814,8 +814,8 @@ PyDoc_STRVAR(itertools_compress__doc__, "\n" "Return data elements corresponding to true selector elements.\n" "\n" -"Forms a shorter iterator from selected data elements using the selectors to\n" -"choose the data elements."); +"Forms a shorter iterator from selected data elements using the selectors\n" +"to choose the data elements."); static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2); @@ -980,4 +980,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=7f385837b13edbeb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a34a31f60100e0ff input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index b023299dd9cb2dc..a5e583c180defee 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -198,8 +198,9 @@ PyDoc_STRVAR(math_frexp__doc__, "\n" "Return the mantissa and exponent of x, as pair (m, e).\n" "\n" -"m is a float and e is an int, such that x = m * 2.**e.\n" -"If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0."); +"If x is a finite nonzero number, then m is a float with\n" +"0.5 <= abs(m) < 1.0 and an integer e is such that\n" +"x == m * 2**e exactly. Else, return (x, 0)."); #define MATH_FREXP_METHODDEF \ {"frexp", (PyCFunction)math_frexp, METH_O, math_frexp__doc__}, @@ -1043,8 +1044,8 @@ PyDoc_STRVAR(math_nextafter__doc__, "\n" "If steps is not specified or is None, it defaults to 1.\n" "\n" -"Raises a TypeError, if x or y is not a double, or if steps is not an integer.\n" -"Raises ValueError if steps is negative."); +"Raises a TypeError, if x or y is not a double, or if steps is not\n" +"an integer. Raises ValueError if steps is negative."); #define MATH_NEXTAFTER_METHODDEF \ {"nextafter", _PyCFunction_CAST(math_nextafter), METH_FASTCALL|METH_KEYWORDS, math_nextafter__doc__}, @@ -1163,4 +1164,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=23b2453ba77453e5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3452ce8caa2d1bd7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 7e2480bdace38d2..ba41eab15650e8f 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -529,8 +529,9 @@ PyDoc_STRVAR(_overlapped_Overlapped_getresult__doc__, "\n" "Retrieve result of operation.\n" "\n" -"If wait is true then it blocks until the operation is finished. If wait\n" -"is false and the operation is still pending then an error is raised."); +"If wait is true then it blocks until the operation is finished. If\n" +"wait is false and the operation is still pending then an error is\n" +"raised."); #define _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF \ {"getresult", _PyCFunction_CAST(_overlapped_Overlapped_getresult), METH_FASTCALL, _overlapped_Overlapped_getresult__doc__}, @@ -1242,4 +1243,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=3e4cb2b55342cd96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ecaf45a09539599 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ad4c5dd1c9bc084..45e509d003085be 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -512,9 +512,9 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "Change the current working directory to the specified path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -"If this functionality is unavailable, using it raises an exception."); +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -649,14 +649,15 @@ PyDoc_STRVAR(os_chmod__doc__, "Change the access permissions of a file.\n" "\n" " path\n" -" Path to be modified. May always be specified as a str, bytes, or a path-like object.\n" -" On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +" Path to be modified. May always be specified as a str, bytes, or\n" +" a path-like object. On some platforms, path may also be specified\n" +" as an open file descriptor. If this functionality is unavailable,\n" +" using it raises an exception.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -765,9 +766,9 @@ PyDoc_STRVAR(os_fchmod__doc__, " The file descriptor of the file to be modified.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" "\n" "Equivalent to os.chmod(fd, mode)."); @@ -841,8 +842,8 @@ PyDoc_STRVAR(os_lchmod__doc__, "\n" "Change the access permissions of a file, without following symbolic links.\n" "\n" -"If path is a symlink, this affects the link itself rather than the target.\n" -"Equivalent to chmod(path, mode, follow_symlinks=False).\""); +"If path is a symlink, this affects the link itself rather than the\n" +"target. Equivalent to chmod(path, mode, follow_symlinks=False)."); #define OS_LCHMOD_METHODDEF \ {"lchmod", _PyCFunction_CAST(os_lchmod), METH_FASTCALL|METH_KEYWORDS, os_lchmod__doc__}, @@ -916,9 +917,9 @@ PyDoc_STRVAR(os_chflags__doc__, "\n" "Set file flags.\n" "\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chflags will change flags on the symbolic link itself instead of the\n" -" file the link points to.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chflags() will change flags on the symbolic link itself\n" +"instead of the file the link points to.\n" "follow_symlinks may not be implemented on your platform. If it is\n" "unavailable, using it will raise a NotImplementedError."); @@ -1329,10 +1330,11 @@ PyDoc_STRVAR(os_chown__doc__, "chown($module, /, path, uid, gid, *, dir_fd=None, follow_symlinks=True)\n" "--\n" "\n" -"Change the owner and group id of path to the numeric uid and gid.\\\n" +"Change the owner and group id of path to the numeric uid and gid.\n" "\n" " path\n" -" Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int.\n" +" Path to be examined; can be string, bytes, a path-like object, or\n" +" open-file-descriptor int.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -1342,18 +1344,19 @@ PyDoc_STRVAR(os_chown__doc__, " stat will examine the symbolic link itself instead of the file\n" " the link points to.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chown will modify the symbolic link itself instead of the file the\n" -" link points to.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chown will modify the symbolic link itself instead of\n" +"the file the link points to.\n" "It is an error to use dir_fd or follow_symlinks when specifying path as\n" -" an open file descriptor.\n" -"dir_fd and follow_symlinks may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"an open file descriptor.\n" +"dir_fd and follow_symlinks may not be implemented on your platform. If\n" +"they are unavailable, using them will raise a NotImplementedError."); #define OS_CHOWN_METHODDEF \ {"chown", _PyCFunction_CAST(os_chown), METH_FASTCALL|METH_KEYWORDS, os_chown__doc__}, @@ -1641,14 +1644,15 @@ PyDoc_STRVAR(os_link__doc__, "Create a hard link to a file.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "If follow_symlinks is False, and the last element of src is a symbolic\n" -" link, link will create a link to the symbolic link itself instead of the\n" -" file the link points to.\n" -"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n" -" platform. If they are unavailable, using them will raise a\n" -" NotImplementedError."); +"link, link will create a link to the symbolic link itself instead of the\n" +"file the link points to.\n" +"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on\n" +"your platform. If they are unavailable, using them will raise\n" +"a NotImplementedError."); #define OS_LINK_METHODDEF \ {"link", _PyCFunction_CAST(os_link), METH_FASTCALL|METH_KEYWORDS, os_link__doc__}, @@ -1750,13 +1754,13 @@ PyDoc_STRVAR(os_listdir__doc__, "\n" "Return a list containing the names of the files in the directory.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path is bytes,\n" -" the filenames returned will also be bytes; in all other circumstances\n" -" the filenames returned will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the filenames returned will also be bytes; in all other\n" +"circumstances the filenames returned will be str.\n" "If path is None, uses the path=\'.\'.\n" -"On some platforms, path may also be specified as an open file descriptor;\\\n" -" the file descriptor must refer to a directory.\n" -" If this functionality is unavailable, using it raises NotImplementedError.\n" +"On some platforms, path may also be specified as an open file\n" +"descriptor; the file descriptor must refer to a directory. If this\n" +"functionality is unavailable, using it raises NotImplementedError.\n" "\n" "The list is in arbitrary order. It does not include the special\n" "entries \'.\' and \'..\' even if they are present in the directory."); @@ -2709,13 +2713,14 @@ PyDoc_STRVAR(os_mkdir__doc__, "\n" "Create a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError.\n" "\n" -"The mode argument is ignored on Windows. Where it is used, the current umask\n" -"value is first masked out."); +"The mode argument is ignored on Windows. Where it is used, the current\n" +"umask value is first masked out."); #define OS_MKDIR_METHODDEF \ {"mkdir", _PyCFunction_CAST(os_mkdir), METH_FASTCALL|METH_KEYWORDS, os_mkdir__doc__}, @@ -2981,10 +2986,11 @@ PyDoc_STRVAR(os_rename__doc__, "Rename a file or directory.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_RENAME_METHODDEF \ {"rename", _PyCFunction_CAST(os_rename), METH_FASTCALL|METH_KEYWORDS, os_rename__doc__}, @@ -3075,10 +3081,11 @@ PyDoc_STRVAR(os_replace__doc__, "Rename a file or directory, overwriting the destination.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_REPLACE_METHODDEF \ {"replace", _PyCFunction_CAST(os_replace), METH_FASTCALL|METH_KEYWORDS, os_replace__doc__}, @@ -3168,10 +3175,11 @@ PyDoc_STRVAR(os_rmdir__doc__, "\n" "Remove a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_RMDIR_METHODDEF \ {"rmdir", _PyCFunction_CAST(os_rmdir), METH_FASTCALL|METH_KEYWORDS, os_rmdir__doc__}, @@ -3426,10 +3434,11 @@ PyDoc_STRVAR(os_unlink__doc__, "\n" "Remove a file (same as remove()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_UNLINK_METHODDEF \ {"unlink", _PyCFunction_CAST(os_unlink), METH_FASTCALL|METH_KEYWORDS, os_unlink__doc__}, @@ -3503,10 +3512,11 @@ PyDoc_STRVAR(os_remove__doc__, "\n" "Remove a file (same as unlink()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_REMOVE_METHODDEF \ {"remove", _PyCFunction_CAST(os_remove), METH_FASTCALL|METH_KEYWORDS, os_remove__doc__}, @@ -3606,27 +3616,28 @@ PyDoc_STRVAR(os_utime__doc__, "\n" "Set the access and modified time of path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" "\n" "If times is not None, it must be a tuple (atime, mtime);\n" -" atime and mtime should be expressed as float seconds since the epoch.\n" +"atime and mtime should be expressed as float seconds since the epoch.\n" "If ns is specified, it must be a tuple (atime_ns, mtime_ns);\n" -" atime_ns and mtime_ns should be expressed as integer nanoseconds\n" -" since the epoch.\n" +"atime_ns and mtime_ns should be expressed as integer nanoseconds\n" +"since the epoch.\n" "If times is None and ns is unspecified, utime uses the current time.\n" "Specifying tuples for both times and ns is an error.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, utime will modify the symbolic link itself instead of the file the\n" -" link points to.\n" -"It is an error to use dir_fd or follow_symlinks when specifying path\n" -" as an open file descriptor.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, utime will modify the symbolic link itself instead of\n" +"the file the link points to.\n" +"It is an error to use dir_fd or follow_symlinks when specifying path as\n" +"an open file descriptor.\n" "dir_fd and follow_symlinks may not be available on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_UTIME_METHODDEF \ {"utime", _PyCFunction_CAST(os_utime), METH_FASTCALL|METH_KEYWORDS, os_utime__doc__}, @@ -3929,7 +3940,8 @@ PyDoc_STRVAR(os_posix_spawn__doc__, " resetids\n" " If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -4082,7 +4094,8 @@ PyDoc_STRVAR(os_posix_spawnp__doc__, " resetids\n" " If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -4953,8 +4966,8 @@ PyDoc_STRVAR(os_posix_openpt__doc__, "Open and return a file descriptor for a master pseudo-terminal device.\n" "\n" "Performs a posix_openpt() C function call. The oflag argument is used to\n" -"set file status flags and file access modes as specified in the manual page\n" -"of posix_openpt() of your system."); +"set file status flags and file access modes as specified in the manual\n" +"page of posix_openpt() of your system."); #define OS_POSIX_OPENPT_METHODDEF \ {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, @@ -5411,9 +5424,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5457,9 +5470,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5612,7 +5625,8 @@ PyDoc_STRVAR(os_getppid__doc__, "Return the parent\'s process id.\n" "\n" "If the parent process has already exited, Windows machines will still\n" -"return its id; others systems will return the id of the \'init\' process (1)."); +"return its id; others systems will return the id of the \'init\' proces\n" +"(1)."); #define OS_GETPPID_METHODDEF \ {"getppid", (PyCFunction)os_getppid, METH_NOARGS, os_getppid__doc__}, @@ -6162,8 +6176,8 @@ PyDoc_STRVAR(os_waitid__doc__, " Constructed from the ORing of one or more of WEXITED, WSTOPPED\n" " or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n" "\n" -"Returns either waitid_result or None if WNOHANG is specified and there are\n" -"no children in a waitable state."); +"Returns either waitid_result or None if WNOHANG is specified and there\n" +"are no children in a waitable state."); #define OS_WAITID_METHODDEF \ {"waitid", _PyCFunction_CAST(os_waitid), METH_FASTCALL, os_waitid__doc__}, @@ -6324,8 +6338,8 @@ PyDoc_STRVAR(os_pidfd_open__doc__, "\n" "Return a file descriptor referring to the process *pid*.\n" "\n" -"The descriptor can be used to perform process management without races and\n" -"signals."); +"The descriptor can be used to perform process management without races\n" +"and signals."); #define OS_PIDFD_OPEN_METHODDEF \ {"pidfd_open", _PyCFunction_CAST(os_pidfd_open), METH_FASTCALL|METH_KEYWORDS, os_pidfd_open__doc__}, @@ -6549,8 +6563,9 @@ PyDoc_STRVAR(os_readlink__doc__, "\n" "Return a string representing the path to which the symbolic link points.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -"and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "\n" "dir_fd may not be implemented on your platform. If it is unavailable,\n" "using it will raise a NotImplementedError."); @@ -6632,14 +6647,15 @@ PyDoc_STRVAR(os_symlink__doc__, "Create a symbolic link pointing to src named dst.\n" "\n" "target_is_directory is required on Windows if the target is to be\n" -" interpreted as a directory. (On Windows, symlink requires\n" -" Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n" -" target_is_directory is ignored on non-Windows platforms.\n" +"interpreted as a directory. (On Windows, symlink requires Windows 6.0\n" +"or greater, and raises a NotImplementedError otherwise.)\n" +"target_is_directory is ignored on non-Windows platforms.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_SYMLINK_METHODDEF \ {"symlink", _PyCFunction_CAST(os_symlink), METH_FASTCALL|METH_KEYWORDS, os_symlink__doc__}, @@ -7307,10 +7323,11 @@ PyDoc_STRVAR(os_open__doc__, "\n" "Open a file for low level IO. Returns a file descriptor (integer).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_OPEN_METHODDEF \ {"open", _PyCFunction_CAST(os_open), METH_FASTCALL|METH_KEYWORDS, os_open__doc__}, @@ -7684,7 +7701,8 @@ PyDoc_STRVAR(os_lseek__doc__, " - SEEK_CUR: seek from the current file position.\n" " - SEEK_END: seek from the end of the file.\n" "\n" -"The return value is the number of bytes relative to the beginning of the file."); +"The return value is the number of bytes relative to the beginning of\n" +"the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -7775,15 +7793,15 @@ PyDoc_STRVAR(os_readinto__doc__, "\n" "Read into a buffer object from a file descriptor.\n" "\n" -"The buffer should be mutable and bytes-like. On success, returns the number of\n" -"bytes read. Less bytes may be read than the size of the buffer. The underlying\n" -"system call will be retried when interrupted by a signal, unless the signal\n" -"handler raises an exception. Other errors will not be retried and an error will\n" -"be raised.\n" +"The buffer should be mutable and bytes-like. On success, returns the\n" +"number of bytes read. Less bytes may be read than the size of the\n" +"buffer. The underlying system call will be retried when interrupted by\n" +"a signal, unless the signal handler raises an exception. Other errors\n" +"will not be retried and an error will be raised.\n" "\n" -"Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n" -"(which can be used to check for errors without reading data). Never returns\n" -"negative."); +"Returns 0 if *fd* is at end of file or if the provided *buffer* has\n" +"length 0 (which can be used to check for errors without reading data).\n" +"Never returns negative."); #define OS_READINTO_METHODDEF \ {"readinto", _PyCFunction_CAST(os_readinto), METH_FASTCALL, os_readinto__doc__}, @@ -7938,14 +7956,15 @@ PyDoc_STRVAR(os_preadv__doc__, "\n" "Reads from a file descriptor into a number of mutable bytes-like objects.\n" "\n" -"Combines the functionality of readv() and pread(). As readv(), it will\n" -"transfer data into each buffer until it is full and then move on to the next\n" -"buffer in the sequence to hold the rest of the data. Its fourth argument,\n" -"specifies the file offset at which the input operation is to be performed. It\n" -"will return the total number of bytes read (which can be less than the total\n" -"capacity of all the objects).\n" +"Combines the functionality of readv() and pread(). As readv(), it will\n" +"transfer data into each buffer until it is full and then move on to the\n" +"next buffer in the sequence to hold the rest of the data. Its fourth\n" +"argument, specifies the file offset at which the input operation is to\n" +"be performed. It will return the total number of bytes read (which can\n" +"be less than the total capacity of all the objects).\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_HIPRI\n" "- RWF_NOWAIT\n" @@ -8679,14 +8698,16 @@ PyDoc_STRVAR(os_pwritev__doc__, "\n" "Writes the contents of bytes-like objects to a file descriptor at a given offset.\n" "\n" -"Combines the functionality of writev() and pwrite(). All buffers must be a sequence\n" -"of bytes-like objects. Buffers are processed in array order. Entire contents of first\n" -"buffer is written before proceeding to second, and so on. The operating system may\n" -"set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\n" -"This function writes the contents of each object to the file descriptor and returns\n" -"the total number of bytes written.\n" +"Combines the functionality of writev() and pwrite(). All buffers must be\n" +"a sequence of bytes-like objects. Buffers are processed in array order.\n" +"Entire contents of first buffer is written before proceeding to second,\n" +"and so on. The operating system may set a limit (sysconf() value\n" +"SC_IOV_MAX) on the number of buffers that can be used.\n" +"This function writes the contents of each object to the file descriptor\n" +"and returns the total number of bytes written.\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_DSYNC\n" "- RWF_SYNC\n" @@ -9001,10 +9022,11 @@ PyDoc_STRVAR(os_mkfifo__doc__, "\n" "Create a \"fifo\" (a POSIX named pipe).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKFIFO_METHODDEF \ {"mkfifo", _PyCFunction_CAST(os_mkfifo), METH_FASTCALL|METH_KEYWORDS, os_mkfifo__doc__}, @@ -9096,17 +9118,18 @@ PyDoc_STRVAR(os_mknod__doc__, "\n" "Create a node in the file system.\n" "\n" -"Create a node in the file system (file, device special file or named pipe)\n" -"at path. mode specifies both the permissions to use and the\n" +"Create a node in the file system (file, device special file or named\n" +"pipe) at path. mode specifies both the permissions to use and the\n" "type of node to be created, being combined (bitwise OR) with one of\n" -"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode,\n" -"device defines the newly created device special file (probably using\n" -"os.makedev()). Otherwise device is ignored.\n" +"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set\n" +"on mode, device defines the newly created device special file (probably\n" +"using os.makedev()). Otherwise device is ignored.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKNOD_METHODDEF \ {"mknod", _PyCFunction_CAST(os_mknod), METH_FASTCALL|METH_KEYWORDS, os_mknod__doc__}, @@ -9352,8 +9375,9 @@ PyDoc_STRVAR(os_truncate__doc__, "\n" "Truncate a file, specified by path, to a specific length.\n" "\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(os_truncate), METH_FASTCALL|METH_KEYWORDS, os_truncate__doc__}, @@ -9427,7 +9451,8 @@ PyDoc_STRVAR(os_posix_fallocate__doc__, "Ensure a file has allocated at least a particular number of bytes on disk.\n" "\n" "Ensure that the file specified by fd encompasses a range of bytes\n" -"starting at offset bytes from the beginning and continuing for length bytes."); +"starting at offset bytes from the beginning and continuing for length\n" +"bytes."); #define OS_POSIX_FALLOCATE_METHODDEF \ {"posix_fallocate", _PyCFunction_CAST(os_posix_fallocate), METH_FASTCALL, os_posix_fallocate__doc__}, @@ -9473,8 +9498,8 @@ PyDoc_STRVAR(os_posix_fadvise__doc__, "\n" "Announce an intention to access data in a specific pattern.\n" "\n" -"Announce an intention to access data in a specific pattern, thus allowing\n" -"the kernel to make optimizations.\n" +"Announce an intention to access data in a specific pattern, thus\n" +"allowing the kernel to make optimizations.\n" "The advice applies to the region of the file specified by fd starting at\n" "offset and continuing for length bytes.\n" "advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\n" @@ -10296,8 +10321,9 @@ PyDoc_STRVAR(os_statvfs__doc__, "Perform a statvfs system call on the given path.\n" "\n" "path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_STATVFS_METHODDEF \ {"statvfs", _PyCFunction_CAST(os_statvfs), METH_FASTCALL|METH_KEYWORDS, os_statvfs__doc__}, @@ -10480,8 +10506,9 @@ PyDoc_STRVAR(os_pathconf__doc__, "Return the configuration limit name for the file or directory path.\n" "\n" "If there is no limit, return -1.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_PATHCONF_METHODDEF \ {"pathconf", _PyCFunction_CAST(os_pathconf), METH_FASTCALL|METH_KEYWORDS, os_pathconf__doc__}, @@ -10624,8 +10651,8 @@ PyDoc_STRVAR(os_abort__doc__, "\n" "Abort the interpreter immediately.\n" "\n" -"This function \'dumps core\' or otherwise fails in the hardest way possible\n" -"on the hosting operating system. This function never returns."); +"This function \'dumps core\' or otherwise fails in the hardest way\n" +"possible on the hosting operating system. This function never returns."); #define OS_ABORT_METHODDEF \ {"abort", (PyCFunction)os_abort, METH_NOARGS, os_abort__doc__}, @@ -11013,10 +11040,11 @@ PyDoc_STRVAR(os_getxattr__doc__, "\n" "Return the value of extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, getxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, getxattr will examine the symbolic link itself\n" +"instead of the file the link points to."); #define OS_GETXATTR_METHODDEF \ {"getxattr", _PyCFunction_CAST(os_getxattr), METH_FASTCALL|METH_KEYWORDS, os_getxattr__doc__}, @@ -11103,10 +11131,11 @@ PyDoc_STRVAR(os_setxattr__doc__, "\n" "Set extended attribute attribute on path to value.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, setxattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, setxattr will modify the symbolic link itself instead\n" +"of the file the link points to."); #define OS_SETXATTR_METHODDEF \ {"setxattr", _PyCFunction_CAST(os_setxattr), METH_FASTCALL|METH_KEYWORDS, os_setxattr__doc__}, @@ -11214,10 +11243,11 @@ PyDoc_STRVAR(os_removexattr__doc__, "\n" "Remove extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, removexattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, removexattr will modify the symbolic link itself\n" +"instead of the file the link points to."); #define OS_REMOVEXATTR_METHODDEF \ {"removexattr", _PyCFunction_CAST(os_removexattr), METH_FASTCALL|METH_KEYWORDS, os_removexattr__doc__}, @@ -11303,11 +11333,12 @@ PyDoc_STRVAR(os_listxattr__doc__, "\n" "Return a list of extended attributes on path.\n" "\n" -"path may be either None, a string, a path-like object, or an open file descriptor.\n" -"if path is None, listxattr will examine the current directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, listxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either None, a string, a path-like object, or an open file\n" +"descriptor. If path is None, listxattr will examine the current\n" +"directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, listxattr will examine the symbolic link itself instead\n" +"of the file the link points to."); #define OS_LISTXATTR_METHODDEF \ {"listxattr", _PyCFunction_CAST(os_listxattr), METH_FASTCALL|METH_KEYWORDS, os_listxattr__doc__}, @@ -12329,9 +12360,9 @@ PyDoc_STRVAR(os_scandir__doc__, "\n" "Return an iterator of DirEntry objects for given path.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path\n" -"is bytes, the names of yielded DirEntry objects will also be bytes; in\n" -"all other circumstances they will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the names of yielded DirEntry objects will also be bytes;\n" +"in all other circumstances they will be str.\n" "\n" "If path is None, uses the path=\'.\'."); @@ -12403,9 +12434,9 @@ PyDoc_STRVAR(os_fspath__doc__, "\n" "Return the file system path representation of the object.\n" "\n" -"If the object is str or bytes, then allow it to pass through as-is. If the\n" -"object defines __fspath__(), then return the result of that method. All other\n" -"types raise a TypeError."); +"If the object is str or bytes, then allow it to pass through as-is. If\n" +"the object defines __fspath__(), then return the result of that method.\n" +"All other types raise a TypeError."); #define OS_FSPATH_METHODDEF \ {"fspath", _PyCFunction_CAST(os_fspath), METH_FASTCALL|METH_KEYWORDS, os_fspath__doc__}, @@ -12699,8 +12730,8 @@ PyDoc_STRVAR(os_waitstatus_to_exitcode__doc__, "On Windows, return status shifted right by 8 bits.\n" "\n" "On Unix, if the process is being traced or if waitpid() was called with\n" -"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true.\n" -"This function must not be called if WIFSTOPPED(status) is true."); +"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is\n" +"true. This function must not be called if WIFSTOPPED(status) is true."); #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF \ {"waitstatus_to_exitcode", _PyCFunction_CAST(os_waitstatus_to_exitcode), METH_FASTCALL|METH_KEYWORDS, os_waitstatus_to_exitcode__doc__}, @@ -13611,4 +13642,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=e709b8b783fbc261 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d6889ab281d7676f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index ff2e28269dc927e..1a07726d303ecad 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -218,8 +218,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, "\n" "Return the untranslated text of the input that caused the current event.\n" "\n" -"If the event was generated by a large amount of text (such as a start tag\n" -"for an element with many attributes), not all of the text may be available."); +"If the event was generated by a large amount of text (such as\n" +"a start tag for an element with many attributes), not all of the\n" +"text may be available."); #define PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF \ {"GetInputContext", (PyCFunction)pyexpat_xmlparser_GetInputContext, METH_NOARGS, pyexpat_xmlparser_GetInputContext__doc__}, @@ -357,9 +358,10 @@ PyDoc_STRVAR(pyexpat_xmlparser_UseForeignDTD__doc__, "\n" "Allows the application to provide an artificial external subset if one is not specified as part of the document instance.\n" "\n" -"This readily allows the use of a \'default\' document type controlled by the\n" -"application, while still getting the advantage of providing document type\n" -"information to the parser. \'flag\' defaults to True if not provided."); +"This readily allows the use of a \'default\' document type controlled\n" +"by the application, while still getting the advantage of providing\n" +"document type information to the parser. \'flag\' defaults to True if\n" +"not provided."); #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF \ {"UseForeignDTD", _PyCFunction_CAST(pyexpat_xmlparser_UseForeignDTD), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_UseForeignDTD__doc__}, @@ -417,14 +419,15 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThresho "\n" "Sets the number of output bytes needed to activate protection against billion laughs attacks.\n" "\n" -"The number of output bytes includes amplification from entity expansion\n" -"and reading DTD files.\n" +"The number of output bytes includes amplification from entity\n" +"expansion and reading DTD files.\n" "\n" -"Parser objects usually have a protection activation threshold of 8 MiB,\n" -"but the actual default value depends on the underlying Expat library.\n" +"Parser objects usually have a protection activation threshold of\n" +"8 MiB, but the actual default value depends on the underlying Expat\n" +"library.\n" "\n" -"Activation thresholds below 4 MiB are known to break support for DITA 1.3\n" -"payload and are hence not recommended."); +"Activation thresholds below 4 MiB are known to break support for\n" +"DITA 1.3 payload and are hence not recommended."); #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF \ {"SetBillionLaughsAttackProtectionActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__}, @@ -479,18 +482,21 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplificat "\n" "Sets the maximum tolerated amplification factor for protection against billion laughs attacks.\n" "\n" -"The amplification factor is calculated as \"(direct + indirect) / direct\"\n" -"while parsing, where \"direct\" is the number of bytes read from the primary\n" -"document in parsing and \"indirect\" is the number of bytes added by expanding\n" -"entities and reading external DTD files, combined.\n" +"The amplification factor is calculated as \"(direct + indirect) /\n" +"direct\" while parsing, where \"direct\" is the number of bytes read\n" +"from the primary document in parsing and \"indirect\" is the number of\n" +"bytes added by expanding entities and reading external DTD files,\n" +"combined.\n" "\n" -"The \'max_factor\' value must be a non-NaN floating point value greater than\n" -"or equal to 1.0. Amplification factors greater than 30,000 can be observed\n" -"in the middle of parsing even with benign files in practice. In particular,\n" -"the activation threshold should be carefully chosen to avoid false positives.\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"30,000 can be observed in the middle of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" "\n" "Parser objects usually have a maximum amplification factor of 100,\n" -"but the actual default value depends on the underlying Expat library."); +"but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF \ {"SetBillionLaughsAttackProtectionMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__}, @@ -551,8 +557,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, "\n" "Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n" "\n" -"Parser objects usually have an allocation activation threshold of 64 MiB,\n" -"but the actual default value depends on the underlying Expat library."); +"Parser objects usually have an allocation activation threshold of\n" +"64 MiB, but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF \ {"SetAllocTrackerActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__}, @@ -606,18 +613,20 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__, "\n" "Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n" "\n" -"The amplification factor is calculated as \"allocated / direct\" while parsing,\n" -"where \"direct\" is the number of bytes read from the primary document in parsing\n" -"and \"allocated\" is the number of bytes of dynamic memory allocated in the parser\n" -"hierarchy.\n" +"The amplification factor is calculated as \"allocated / direct\" while\n" +"parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"allocated\" is the number of bytes of\n" +"dynamic memory allocated in the parser hierarchy.\n" "\n" -"The \'max_factor\' value must be a non-NaN floating point value greater than\n" -"or equal to 1.0. Amplification factors greater than 100.0 can be observed\n" -"near the start of parsing even with benign files in practice. In particular,\n" -"the activation threshold should be carefully chosen to avoid false positives.\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"100.0 can be observed near the start of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" "\n" "Parser objects usually have a maximum amplification factor of 100,\n" -"but the actual default value depends on the underlying Expat library."); +"but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF \ {"SetAllocTrackerMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__}, @@ -830,4 +839,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ -/*[clinic end generated code: output=81101a16a409daf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=270a0bfe3300e8a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 3952054e9e32bfb..c1c8ad40e724f53 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(select_select__doc__, "\n" "Wait until one or more file descriptors are ready for some kind of I/O.\n" "\n" -"The first three arguments are iterables of file descriptors to be waited for:\n" +"The first three arguments are iterables of file descriptors to be waited\n" +"for:\n" "rlist -- wait until ready for reading\n" "wlist -- wait until ready for writing\n" "xlist -- wait for an \"exceptional condition\"\n" @@ -29,9 +30,9 @@ PyDoc_STRVAR(select_select__doc__, "a non-integer to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" -"The return value is a tuple of three lists corresponding to the first three\n" -"arguments; each contains the subset of the corresponding file descriptors\n" -"that are ready.\n" +"The return value is a tuple of three lists corresponding to the first\n" +"three arguments; each contains the subset of the corresponding file\n" +"descriptors that are ready.\n" "\n" "*** IMPORTANT NOTICE ***\n" "On Windows, only sockets are supported; on Unix, all file\n" @@ -214,8 +215,8 @@ PyDoc_STRVAR(select_poll_poll__doc__, " The maximum time to wait in milliseconds, or else None (or a negative\n" " value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_POLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_poll_poll), METH_FASTCALL, select_poll_poll__doc__}, @@ -396,11 +397,11 @@ PyDoc_STRVAR(select_devpoll_poll__doc__, "Polls the set of registered file descriptors.\n" "\n" " timeout\n" -" The maximum time to wait in milliseconds, or else None (or a negative\n" -" value) to wait indefinitely.\n" +" The maximum time to wait in milliseconds, or else None (or\n" +" a negative value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_DEVPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_devpoll_poll), METH_FASTCALL, select_devpoll_poll__doc__}, @@ -498,8 +499,8 @@ PyDoc_STRVAR(select_poll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_POLL_METHODDEF \ {"poll", (PyCFunction)select_poll, METH_NOARGS, select_poll__doc__}, @@ -523,8 +524,8 @@ PyDoc_STRVAR(select_devpoll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_DEVPOLL_METHODDEF \ {"devpoll", (PyCFunction)select_devpoll, METH_NOARGS, select_devpoll__doc__}, @@ -978,8 +979,8 @@ PyDoc_STRVAR(select_epoll_poll__doc__, " maxevents\n" " the maximum number of events returned; -1 means no limit\n" "\n" -"Returns a list containing any descriptors that have events to report,\n" -"as a list of (fd, events) 2-tuples."); +"Returns a list containing any descriptors that have events to\n" +"report, as a list of (fd, events) 2-tuples."); #define SELECT_EPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_epoll_poll), METH_FASTCALL|METH_KEYWORDS, select_epoll_poll__doc__}, @@ -1399,4 +1400,4 @@ select_kqueue_control(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=52e3be5cc66cf1b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a1ac666294fd14bd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 9fd24d15bf25004..ca47033446074cd 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -138,11 +138,12 @@ PyDoc_STRVAR(signal_signal__doc__, "Set the action for the given signal.\n" "\n" "The action can be SIG_DFL, SIG_IGN, or a callable Python object.\n" -"The previous action is returned. See getsignal() for possible return values.\n" +"The previous action is returned. See getsignal() for possible return\n" +"values.\n" "\n" "*** IMPORTANT NOTICE ***\n" -"A signal handler function is called with two arguments:\n" -"the first is the signal number, the second is the interrupted stack frame."); +"A signal handler function is called with two arguments: the first is\n" +"the signal number, the second is the interrupted stack frame."); #define SIGNAL_SIGNAL_METHODDEF \ {"signal", _PyCFunction_CAST(signal_signal), METH_FASTCALL, signal_signal__doc__}, @@ -362,8 +363,8 @@ PyDoc_STRVAR(signal_setitimer__doc__, "\n" "Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF).\n" "\n" -"The timer will fire after value seconds and after that every interval seconds.\n" -"The itimer can be cleared by setting seconds to zero.\n" +"The timer will fire after value seconds and after that every interval\n" +"seconds. The itimer can be cleared by setting seconds to zero.\n" "\n" "Returns old values as a tuple: (delay, interval)."); @@ -508,8 +509,8 @@ PyDoc_STRVAR(signal_sigwait__doc__, "Wait for a signal.\n" "\n" "Suspend execution of the calling thread until the delivery of one of the\n" -"signals specified in the signal set sigset. The function accepts the signal\n" -"and returns the signal number."); +"signals specified in the signal set sigset. The function accepts the\n" +"signal and returns the signal number."); #define SIGNAL_SIGWAIT_METHODDEF \ {"sigwait", (PyCFunction)signal_sigwait, METH_O, signal_sigwait__doc__}, @@ -794,4 +795,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=42e20d118435d7fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0731d6f05c42c09a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index e0cc1c50dcbac30..b565e7516d50f33 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -36,7 +36,8 @@ PyDoc_STRVAR(_socket_socket_send__doc__, "Send a data string to the socket.\n" "\n" "For the optional flags argument, see the Unix manual.\n" -"Return the number of bytes sent; this may be less than len(data) if the network is busy."); +"Return the number of bytes sent; this may be less than len(data) if\n" +"the network is busy."); #define _SOCKET_SOCKET_SEND_METHODDEF \ {"send", _PyCFunction_CAST(_socket_socket_send), METH_FASTCALL, _socket_socket_send__doc__}, @@ -84,7 +85,8 @@ PyDoc_STRVAR(_socket_socket_sendall__doc__, "\n" "For the optional flags argument, see the Unix manual.\n" "This calls send() repeatedly until all data is sent.\n" -"If an error occurs, it\'s impossible to tell how much data has been sent."); +"If an error occurs, it\'s impossible to tell how much data has been\n" +"sent."); #define _SOCKET_SOCKET_SENDALL_METHODDEF \ {"sendall", _PyCFunction_CAST(_socket_socket_sendall), METH_FASTCALL, _socket_socket_sendall__doc__}, @@ -140,13 +142,13 @@ PyDoc_STRVAR(_socket_socket_sendmsg__doc__, "data as an iterable of bytes-like objects (e.g. bytes objects).\n" "The ancdata argument specifies the ancillary data (control messages)\n" "as an iterable of zero or more tuples (cmsg_level, cmsg_type,\n" -"cmsg_data), where cmsg_level and cmsg_type are integers specifying the\n" -"protocol level and protocol-specific type respectively, and cmsg_data\n" -"is a bytes-like object holding the associated data. The flags\n" -"argument defaults to 0 and has the same meaning as for send(). If\n" -"address is supplied and not None, it sets a destination address for\n" -"the message. The return value is the number of bytes of non-ancillary\n" -"data sent."); +"cmsg_data), where cmsg_level and cmsg_type are integers specifying\n" +"the protocol level and protocol-specific type respectively, and\n" +"cmsg_data is a bytes-like object holding the associated data. The\n" +"flags argument defaults to 0 and has the same meaning as for send().\n" +"If address is supplied and not None, it sets a destination address\n" +"for the message. The return value is the number of bytes of\n" +"non-ancillary data sent."); #define _SOCKET_SOCKET_SENDMSG_METHODDEF \ {"sendmsg", _PyCFunction_CAST(_socket_socket_sendmsg), METH_FASTCALL, _socket_socket_sendmsg__doc__}, @@ -541,4 +543,4 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=36051ebf6ad1e6f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0b1fa78ac6589353 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 83f5a4f6e9f8820..35522bef1dcae9d 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -270,7 +270,8 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "Set the tty winsize for file descriptor fd.\n" "\n" "The winsize to be set is taken from the winsize argument, which\n" -"is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); +"is a two-item tuple (ws_row, ws_col) like the one returned by\n" +"tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, @@ -299,4 +300,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2176c4d9043d3cc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index f8fd111754a894a..620e483d5a759a7 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -503,8 +503,8 @@ PyDoc_STRVAR(zlib_Decompress_decompress__doc__, " Unconsumed input data will be stored in\n" " the unconsumed_tail attribute.\n" "\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" +"After calling this function, some of the input data may still be\n" +"stored in internal buffers for later processing.\n" "Call the flush() method to clear these buffers."); #define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ @@ -914,18 +914,19 @@ PyDoc_STRVAR(zlib__ZlibDecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define ZLIB__ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(zlib__ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib__ZlibDecompressor_decompress__doc__}, @@ -1402,4 +1403,4 @@ zlib_crc32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=13627e14206d3552 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c9a60fe6600a2e4d input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 1e9f9ae051a0b12..7c736f4610bb988 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -948,12 +948,13 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base is not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) +of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=eb25de0757baf4a0]*/ { Py_complex y; @@ -1162,7 +1163,6 @@ cmath_isinf_impl(PyObject *module, Py_complex z) } /*[clinic input] -@permit_long_docstring_body cmath.isclose -> bool a: Py_complex @@ -1179,17 +1179,18 @@ Determine whether two complex numbers are close in value. Return True if a is close in value to b, and False otherwise. -For the values to be considered close, the difference between them must be -smaller than at least one of the tolerances. +For the values to be considered close, the difference between them must +be smaller than at least one of the tolerances. --inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is -not close to anything, even itself. inf and -inf are only close to themselves. +-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, +NaN is not close to anything, even itself. inf and -inf are only close +to themselves. [clinic start generated code]*/ static int cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=8a2486cc6e0014d1 input=0d45feea7c626f47]*/ +/*[clinic end generated code: output=8a2486cc6e0014d1 input=301b56c90d9a79de]*/ { double diff; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 79c609f19aa4cff..ec3f58544cb00d5 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 0 +# define XML_MICRO_VERSION 1 # ifdef __cplusplus } diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 774e0b89d94c0ec..fa3692f9379510e 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_0" -expected_libexpat_version="2.8.0" -expected_libexpat_sha256="c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" +expected_libexpat_tag="R_2_8_1" +expected_libexpat_version="2.8.1" +expected_libexpat_sha256="a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e6842f3f0bf750b..95d346758563ab7 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* a5d18f6a50f536615ac1c70304f87d94f99cc85a86b502188952440610ccf0f8 (2.8.0+) +/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -387,6 +387,7 @@ typedef struct { int nDefaultAtts; int allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -3769,6 +3770,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -7102,10 +7105,10 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } @@ -7152,6 +7155,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -7477,6 +7486,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7518,6 +7528,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7611,6 +7622,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. * The preprocessor guard addresses the "always false" warning @@ -7635,8 +7650,9 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); for (i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7645,6 +7661,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -8391,6 +8413,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 1b4f0c2302daae2..7c727d8c2d4ff0e 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -795,9 +795,9 @@ faulthandler.dump_traceback_later Dump the traceback of all threads in timeout seconds. -If repeat is true, the tracebacks of all threads are dumped every timeout -seconds. If exit is true, call _exit(1) which is not safe. max_threads -caps the number of threads dumped. +If repeat is true, the tracebacks of all threads are dumped every +timeout seconds. If exit is true, call _exit(1) which is not safe. +max_threads caps the number of threads dumped. [clinic start generated code]*/ static PyObject * @@ -805,7 +805,7 @@ faulthandler_dump_traceback_later_impl(PyObject *module, PyObject *timeout_obj, int repeat, PyObject *file, int exit, Py_ssize_t max_threads) -/*[clinic end generated code: output=543a0f3807113394 input=6836555ee157ddb4]*/ +/*[clinic end generated code: output=543a0f3807113394 input=32aaf7437d0928db]*/ { PyTime_t timeout, timeout_us; int fd; @@ -1349,21 +1349,6 @@ faulthandler__stack_overflow_impl(PyObject *module) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - - #ifdef MS_WINDOWS /*[clinic input] faulthandler._raise_exception @@ -1459,7 +1444,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 18bddf46a7466bf..8762e592b258104 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -167,6 +167,8 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, gcstate->generations[2].threshold = threshold2; } #else + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); gcstate->young.threshold = threshold0; if (group_right_1) { gcstate->old[0].threshold = threshold1; @@ -174,6 +176,7 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, if (group_right_2) { gcstate->old[1].threshold = threshold2; } + _PyEval_StartTheWorld(interp); #endif Py_RETURN_NONE; } @@ -326,13 +329,13 @@ gc.get_objects Return a list of objects tracked by the collector (excluding the list returned). -If generation is not None, return only the objects tracked by the collector -that are in that generation. +If generation is not None, return only the objects tracked by the +collector that are in that generation. [clinic start generated code]*/ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) -/*[clinic end generated code: output=48b35fea4ba6cb0e input=a887f1d9924be7cf]*/ +/*[clinic end generated code: output=48b35fea4ba6cb0e input=89bca0d4a64e0135]*/ { if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; @@ -440,19 +443,20 @@ gc_is_finalized_impl(PyObject *module, PyObject *obj) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary gc.freeze Freeze all current tracked objects and ignore them for future collections. -This can be used before a POSIX fork() call to make the gc copy-on-write friendly. -Note: collection before a POSIX fork() call may free pages for future allocation -which can cause copy-on-write. +This can be used before a POSIX fork() call to make the gc copy-on-write +friendly. +Note: collection before a POSIX fork() call may free pages for future +allocation which can cause copy-on-write. [clinic start generated code]*/ static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=11fb59b0a75dcf3d]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=02674706fc9c0de6]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Freeze(interp); diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b39a8f99ed91e82..0f9eca2f73bd0c5 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -942,20 +942,19 @@ _hmac_HMAC_digest_impl(HMACObject *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _hmac.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. This method may raise a MemoryError. [clinic start generated code]*/ static PyObject * _hmac_HMAC_hexdigest_impl(HMACObject *self) -/*[clinic end generated code: output=6659807a09ae14ec input=6e0e796e38d82fc8]*/ +/*[clinic end generated code: output=6659807a09ae14ec input=9097dce732ed808f]*/ { assert(self->digest_size <= Py_hmac_hash_max_digest_size); uint8_t digest[Py_hmac_hash_max_digest_size]; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index a6bfa78a461bb05..0dd31dfbc5a3469 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -107,6 +107,7 @@ typedef struct { #define batchedobject_CAST(op) ((batchedobject *)(op)) /*[clinic input] +@permit_long_summary @classmethod itertools.batched.__new__ as batched_new iterable: object @@ -136,7 +137,7 @@ than n. static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, int strict) -/*[clinic end generated code: output=c6de11b061529d3e input=7814b47e222f5467]*/ +/*[clinic end generated code: output=c6de11b061529d3e input=b31d8be8e8577a34]*/ { PyObject *it; batchedobject *bo; @@ -437,6 +438,7 @@ typedef struct { static PyObject *_grouper_create(groupbyobject *, PyObject *); /*[clinic input] +@permit_long_summary @classmethod itertools.groupby.__new__ @@ -452,7 +454,7 @@ make an iterator that returns consecutive keys and groups from the iterable static PyObject * itertools_groupby_impl(PyTypeObject *type, PyObject *it, PyObject *keyfunc) -/*[clinic end generated code: output=cbb1ae3a90fd4141 input=6b3d123e87ff65a1]*/ +/*[clinic end generated code: output=cbb1ae3a90fd4141 input=9f89fe625b20ef1a]*/ { groupbyobject *gbo; @@ -1950,10 +1952,14 @@ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ iterable, until all of the iterables are exhausted."); +PyDoc_STRVAR(chain_class_getitem_doc, +"chain is generic over the type of its contents.\n\ +This is the union of the types of the input iterable contents."); + static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, chain_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; @@ -3163,13 +3169,13 @@ itertools.compress.__new__ selectors as seq2: object Return data elements corresponding to true selector elements. -Forms a shorter iterator from selected data elements using the selectors to -choose the data elements. +Forms a shorter iterator from selected data elements using the selectors +to choose the data elements. [clinic start generated code]*/ static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2) -/*[clinic end generated code: output=7e67157212ed09e0 input=79596d7cd20c77e5]*/ +/*[clinic end generated code: output=7e67157212ed09e0 input=32ca4347dbc46749]*/ { PyObject *data=NULL, *selectors=NULL; compressobject *lz; @@ -3427,6 +3433,7 @@ slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. */ /*[clinic input] +@permit_long_summary @classmethod itertools.count.__new__ start as long_cnt: object(c_default="NULL") = 0 @@ -3444,7 +3451,7 @@ Equivalent to: static PyObject * itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, PyObject *long_step) -/*[clinic end generated code: output=09a9250aebd00b1c input=d7a85eec18bfcd94]*/ +/*[clinic end generated code: output=09a9250aebd00b1c input=91e4b12c0e88b9f4]*/ { countobject *lz; int fast_mode; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 6b7fc004d0d858f..5636a00afe10c4b 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1438,13 +1438,14 @@ math.frexp Return the mantissa and exponent of x, as pair (m, e). -m is a float and e is an int, such that x = m * 2.**e. -If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0. +If x is a finite nonzero number, then m is a float with +0.5 <= abs(m) < 1.0 and an integer e is such that +x == m * 2**e exactly. Else, return (x, 0). [clinic start generated code]*/ static PyObject * math_frexp_impl(PyObject *module, double x) -/*[clinic end generated code: output=03e30d252a15ad4a input=96251c9e208bc6e9]*/ +/*[clinic end generated code: output=03e30d252a15ad4a input=215cf8ea28a0959b]*/ { int i; /* deal with special cases directly, to sidestep platform @@ -2541,6 +2542,7 @@ math_isnan_impl(PyObject *module, double x) /*[clinic input] +@permit_long_summary math.isinf x: double @@ -2551,7 +2553,7 @@ Return True if x is a positive or negative infinity, and False otherwise. static PyObject * math_isinf_impl(PyObject *module, double x) -/*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ +/*[clinic end generated code: output=9f00cbec4de7b06b input=8584152a71a3aea9]*/ { return PyBool_FromLong((long)isinf(x)); } @@ -2831,7 +2833,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary math.nextafter x: double @@ -2844,13 +2846,13 @@ Return the floating-point value the given number of steps after x towards y. If steps is not specified or is None, it defaults to 1. -Raises a TypeError, if x or y is not a double, or if steps is not an integer. -Raises ValueError if steps is negative. +Raises a TypeError, if x or y is not a double, or if steps is not +an integer. Raises ValueError if steps is negative. [clinic start generated code]*/ static PyObject * math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) -/*[clinic end generated code: output=cc6511f02afc099e input=cc8f0dad1b27a8a4]*/ +/*[clinic end generated code: output=cc6511f02afc099e input=3a9151e6b1e9f346]*/ { #if defined(_AIX) if (x == y) { diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index a30afe91f8fa171..6fb04ba7bd47c67 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -620,8 +620,6 @@ mmap_gfind_lock_held(mmap_object *self, Py_buffer *view, PyObject *start_obj, start += self->size; if (start < 0) start = 0; - else if (start > self->size) - start = self->size; if (end < 0) end += self->size; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 51aee5afd35b6da..255576cc057cdd4 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -885,13 +885,14 @@ _overlapped.Overlapped.getresult Retrieve result of operation. -If wait is true then it blocks until the operation is finished. If wait -is false and the operation is still pending then an error is raised. +If wait is true then it blocks until the operation is finished. If +wait is false and the operation is still pending then an error is +raised. [clinic start generated code]*/ static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ { DWORD transferred = 0; BOOL ret; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5bd53c2146a822f..4b6b51961731695 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2365,7 +2365,7 @@ PyDoc_STRVAR(stat_result__doc__, "stat_result: Result from stat, fstat, or lstat.\n\n\ This object may be accessed either as a tuple of\n\ (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\n\ -or via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\ +or via the attributes st_mode, st_ino, st_dev, st_nlink, and so on.\n\ \n\ Posix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\n\ or st_flags, they are available as attributes only.\n\ @@ -3123,7 +3123,7 @@ class path_t_converter(CConverter): impl_by_reference = True parse_by_reference = True default_type = () - c_init_default = "<placeholder>" # overridden in pre_render(() + c_init_default = "<placeholder>" # overridden in pre_render() converter = 'path_converter' @@ -3266,7 +3266,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=d58f18bdf3bd3565]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ddbf3ac90a981122]*/ /*[clinic input] @@ -3925,14 +3925,14 @@ os.chdir Change the current working directory to the specified path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. -If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=a74ceab5d72adf74]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=64673c342e4369f1]*/ { int result; @@ -4045,15 +4045,16 @@ win32_fchmod(int fd, int mode) os.chmod path: path_t(allow_fd='PATH_HAVE_FCHMOD') - Path to be modified. May always be specified as a str, bytes, or a path-like object. - On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. + Path to be modified. May always be specified as a str, bytes, or + a path-like object. On some platforms, path may also be specified + as an open file descriptor. If this functionality is unavailable, + using it raises an exception. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. * @@ -4080,7 +4081,7 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=7b6e2eeadd8bf199]*/ { int result; @@ -4216,9 +4217,9 @@ os.fchmod The file descriptor of the file to be modified. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. Change the access permissions of the file given by file descriptor fd. @@ -4227,7 +4228,7 @@ Equivalent to os.chmod(fd, mode). static PyObject * os_fchmod_impl(PyObject *module, int fd, int mode) -/*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ +/*[clinic end generated code: output=afd9bc05b4e426b3 input=d24331f9fdc17f49]*/ { int res; @@ -4261,6 +4262,7 @@ os_fchmod_impl(PyObject *module, int fd, int mode) #if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) /*[clinic input] +@permit_long_summary os.lchmod path: path_t @@ -4268,13 +4270,13 @@ os.lchmod Change the access permissions of a file, without following symbolic links. -If path is a symlink, this affects the link itself rather than the target. -Equivalent to chmod(path, mode, follow_symlinks=False)." +If path is a symlink, this affects the link itself rather than the +target. Equivalent to chmod(path, mode, follow_symlinks=False). [clinic start generated code]*/ static PyObject * os_lchmod_impl(PyObject *module, path_t *path, int mode) -/*[clinic end generated code: output=082344022b51a1d5 input=90c5663c7465d24f]*/ +/*[clinic end generated code: output=082344022b51a1d5 input=13110fb62911b015]*/ { int res; if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { @@ -4312,9 +4314,9 @@ os.chflags Set file flags. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chflags will change flags on the symbolic link itself instead of the - file the link points to. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chflags() will change flags on the symbolic link itself +instead of the file the link points to. follow_symlinks may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -4323,7 +4325,7 @@ unavailable, using it will raise a NotImplementedError. static PyObject * os_chflags_impl(PyObject *module, path_t *path, unsigned long flags, int follow_symlinks) -/*[clinic end generated code: output=85571c6737661ce9 input=0327e29feb876236]*/ +/*[clinic end generated code: output=85571c6737661ce9 input=31391927707be1de]*/ { int result; @@ -4473,7 +4475,8 @@ os_fdatasync_impl(PyObject *module, int fd) os.chown path : path_t(allow_fd='PATH_HAVE_FCHOWN') - Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int. + Path to be examined; can be string, bytes, a path-like object, or + open-file-descriptor int. uid: uid_t @@ -4481,7 +4484,7 @@ os.chown * - dir_fd : dir_fd(requires='fchownat') = None + dir_fd: dir_fd(requires='fchownat') = None If not None, it should be a file descriptor open to a directory, and path should be relative; path will then be relative to that directory. @@ -4491,27 +4494,28 @@ os.chown stat will examine the symbolic link itself instead of the file the link points to. -Change the owner and group id of path to the numeric uid and gid.\ +Change the owner and group id of path to the numeric uid and gid. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chown will modify the symbolic link itself instead of the file the - link points to. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chown will modify the symbolic link itself instead of +the file the link points to. It is an error to use dir_fd or follow_symlinks when specifying path as - an open file descriptor. -dir_fd and follow_symlinks may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +an open file descriptor. +dir_fd and follow_symlinks may not be implemented on your platform. If +they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=4beadab0db5f70cd input=b08c5ec67996a97d]*/ +/*[clinic end generated code: output=4beadab0db5f70cd input=509c91b7a0e72f52]*/ { int result; @@ -4795,7 +4799,6 @@ os_getcwdb_impl(PyObject *module) #ifdef HAVE_LINK /*[clinic input] -@permit_long_docstring_body os.link src : path_t @@ -4808,20 +4811,21 @@ os.link Create a hard link to a file. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. If follow_symlinks is False, and the last element of src is a symbolic - link, link will create a link to the symbolic link itself instead of the - file the link points to. -src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your - platform. If they are unavailable, using them will raise a - NotImplementedError. +link, link will create a link to the symbolic link itself instead of the +file the link points to. +src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on +your platform. If they are unavailable, using them will raise +a NotImplementedError. [clinic start generated code]*/ static PyObject * os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int follow_symlinks) -/*[clinic end generated code: output=7f00f6007fd5269a input=e2a50a6497050e44]*/ +/*[clinic end generated code: output=7f00f6007fd5269a input=a28e6866fbd20a01]*/ { #ifdef MS_WINDOWS BOOL result = FALSE; @@ -5115,30 +5119,28 @@ _posix_listdir(path_t *path, PyObject *list) /*[clinic input] -@permit_long_docstring_body os.listdir path : path_t(nullable=True, allow_fd='PATH_HAVE_FDOPENDIR') = None Return a list containing the names of the files in the directory. -path can be specified as either str, bytes, or a path-like object. If path is bytes, - the filenames returned will also be bytes; in all other circumstances - the filenames returned will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the filenames returned will also be bytes; in all other +circumstances the filenames returned will be str. If path is None, uses the path='.'. -On some platforms, path may also be specified as an open file descriptor;\ - the file descriptor must refer to a directory. - If this functionality is unavailable, using it raises NotImplementedError. +On some platforms, path may also be specified as an open file +descriptor; the file descriptor must refer to a directory. If this +functionality is unavailable, using it raises NotImplementedError. The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory. - [clinic start generated code]*/ static PyObject * os_listdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=293045673fcd1a75 input=0bd1728387391b9a]*/ +/*[clinic end generated code: output=293045673fcd1a75 input=4eefe7c6a42ec9b2]*/ { if (PySys_Audit("os.listdir", "O", path->object ? path->object : Py_None) < 0) { @@ -5597,6 +5599,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } /*[clinic input] +@permit_long_summary os._findfirstfile path: path_t / @@ -5605,7 +5608,7 @@ A function to get the real file name without accessing the file in Windows. static PyObject * os__findfirstfile_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=106dd3f0779c83dd input=0734dff70f60e1a8]*/ +/*[clinic end generated code: output=106dd3f0779c83dd input=48c319aaa48d05d4]*/ { PyObject *result; HANDLE hFindFile; @@ -6159,22 +6162,21 @@ os.mkdir dir_fd : dir_fd(requires='mkdirat') = None -# "mkdir(path, mode=0o777, *, dir_fd=None)\n\n\ - Create a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. -The mode argument is ignored on Windows. Where it is used, the current umask -value is first masked out. +The mode argument is ignored on Windows. Where it is used, the current +umask value is first masked out. [clinic start generated code]*/ static PyObject * os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ +/*[clinic end generated code: output=a70446903abe821f input=30270d369599634b]*/ { int result; #ifdef MS_WINDOWS @@ -6428,7 +6430,6 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is /*[clinic input] -@permit_long_docstring_body os.rename src : path_t @@ -6440,38 +6441,39 @@ os.rename Rename a file or directory. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rename_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=59e803072cf41230 input=11aae8c091162766]*/ +/*[clinic end generated code: output=59e803072cf41230 input=7d320d687c715fd6]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 0); } /*[clinic input] -@permit_long_docstring_body os.replace = os.rename Rename a file or directory, overwriting the destination. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_replace_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=1968c02e7857422b input=78d6c8087e90994c]*/ +/*[clinic end generated code: output=1968c02e7857422b input=44ed6b762d5953fc]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 1); } @@ -6486,15 +6488,16 @@ os.rmdir Remove a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ +/*[clinic end generated code: output=080eb54f506e8301 input=84325211e33a98e0]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6665,16 +6668,17 @@ os.unlink Remove a file (same as remove()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_unlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ +/*[clinic end generated code: output=621797807b9963b1 input=1a2ef2579207eab1]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6726,15 +6730,16 @@ os.remove = os.unlink Remove a file (same as unlink()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_remove_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=a8535b28f0068883 input=e05c5ab55cd30983]*/ +/*[clinic end generated code: output=a8535b28f0068883 input=9f6e66912126bd56]*/ { return os_unlink_impl(module, path, dir_fd); } @@ -7056,38 +7061,37 @@ os.utime dir_fd: dir_fd(requires='futimensat') = None follow_symlinks: bool=True -# "utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True)\n\ - Set the access and modified time of path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. If times is not None, it must be a tuple (atime, mtime); - atime and mtime should be expressed as float seconds since the epoch. +atime and mtime should be expressed as float seconds since the epoch. If ns is specified, it must be a tuple (atime_ns, mtime_ns); - atime_ns and mtime_ns should be expressed as integer nanoseconds - since the epoch. +atime_ns and mtime_ns should be expressed as integer nanoseconds +since the epoch. If times is None and ns is unspecified, utime uses the current time. Specifying tuples for both times and ns is an error. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, utime will modify the symbolic link itself instead of the file the - link points to. -It is an error to use dir_fd or follow_symlinks when specifying path - as an open file descriptor. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, utime will modify the symbolic link itself instead of +the file the link points to. +It is an error to use dir_fd or follow_symlinks when specifying path as +an open file descriptor. dir_fd and follow_symlinks may not be available on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ +/*[clinic end generated code: output=cfcac69d027b82cf input=5ab470b2bc250788]*/ { #ifdef MS_WINDOWS HANDLE hFile; @@ -7244,6 +7248,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, /*[clinic input] +@permit_long_summary os._exit status: int @@ -7253,7 +7258,7 @@ Exit to the system with specified status, without normal exit processing. static PyObject * os__exit_impl(PyObject *module, int status) -/*[clinic end generated code: output=116e52d9c2260d54 input=5e6d57556b0c4a62]*/ +/*[clinic end generated code: output=116e52d9c2260d54 input=c35d282acfebe8fd]*/ { _exit(status); return NULL; /* Make gcc -Wall happy */ @@ -8057,7 +8062,8 @@ os.posix_spawn resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () @@ -8074,7 +8080,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=69e7c9ebbdcf94a5]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=c7592dcbc96e8114]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -8103,7 +8109,8 @@ os.posix_spawnp resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () @@ -8120,7 +8127,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=a5c057527c6881a5]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=43ccc1452cae2be3]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -9156,13 +9163,13 @@ os.posix_openpt -> int Open and return a file descriptor for a master pseudo-terminal device. Performs a posix_openpt() C function call. The oflag argument is used to -set file status flags and file access modes as specified in the manual page -of posix_openpt() of your system. +set file status flags and file access modes as specified in the manual +page of posix_openpt() of your system. [clinic start generated code]*/ static int os_posix_openpt_impl(PyObject *module, int oflag) -/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +/*[clinic end generated code: output=ee0bc2624305fc79 input=3ce4eb297fa64307]*/ { int fd; @@ -9437,6 +9444,7 @@ os_openpty_impl(PyObject *module) #if defined(HAVE_LOGIN_TTY) || defined(HAVE_FALLBACK_LOGIN_TTY) /*[clinic input] +@permit_long_summary os.login_tty fd: fildes @@ -9451,7 +9459,7 @@ calling process; close fd. static PyObject * os_login_tty_impl(PyObject *module, int fd) -/*[clinic end generated code: output=495a79911b4cc1bc input=5f298565099903a2]*/ +/*[clinic end generated code: output=495a79911b4cc1bc input=b102a7c36e8baf00]*/ { #ifdef HAVE_LOGIN_TTY if (login_tty(fd) == -1) { @@ -9806,14 +9814,14 @@ os.initgroups Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=984e60c7fed88cb4]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=35f2d4fb7fcc0bdf]*/ #else /*[clinic input] os.initgroups @@ -9824,14 +9832,14 @@ os.initgroups Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=17d8fbe2dea42ca4]*/ +/*[clinic end generated code: output=59341244521a9e3f input=7e4514dff4526a95]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -10028,12 +10036,13 @@ os.getppid Return the parent's process id. If the parent process has already exited, Windows machines will still -return its id; others systems will return the id of the 'init' process (1). +return its id; others systems will return the id of the 'init' proces +(1). [clinic start generated code]*/ static PyObject * os_getppid_impl(PyObject *module) -/*[clinic end generated code: output=43b2a946a8c603b4 input=e637cb87539c030e]*/ +/*[clinic end generated code: output=43b2a946a8c603b4 input=e17c1de18f41316b]*/ { #ifdef MS_WINDOWS return win32_getppid(); @@ -10587,13 +10596,13 @@ os.waitid Returns the result of waiting for a process or processes. -Returns either waitid_result or None if WNOHANG is specified and there are -no children in a waitable state. +Returns either waitid_result or None if WNOHANG is specified and there +are no children in a waitable state. [clinic start generated code]*/ static PyObject * os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) -/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=d8e7f76e052b7920]*/ +/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=14956bc8d102b5db]*/ { PyObject *result; int res; @@ -10760,13 +10769,13 @@ os.pidfd_open Return a file descriptor referring to the process *pid*. -The descriptor can be used to perform process management without races and -signals. +The descriptor can be used to perform process management without races +and signals. [clinic start generated code]*/ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) -/*[clinic end generated code: output=5c7252698947dc41 input=c3fd99ce947ccfef]*/ +/*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { int fd = syscall(__NR_pidfd_open, pid, flags); if (fd < 0) { @@ -10845,8 +10854,9 @@ os.readlink Return a string representing the path to which the symbolic link points. -If dir_fd is not None, it should be a file descriptor open to a directory, -and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -10854,7 +10864,7 @@ using it will raise a NotImplementedError. static PyObject * os_readlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=d21b732a2e814030 input=113c87e0db1ecaf2]*/ +/*[clinic end generated code: output=d21b732a2e814030 input=03d10130870dbca8]*/ { #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; @@ -11049,26 +11059,25 @@ os.symlink * dir_fd: dir_fd(requires='symlinkat')=None -# "symlink(src, dst, target_is_directory=False, *, dir_fd=None)\n\n\ - Create a symbolic link pointing to src named dst. target_is_directory is required on Windows if the target is to be - interpreted as a directory. (On Windows, symlink requires - Windows 6.0 or greater, and raises a NotImplementedError otherwise.) - target_is_directory is ignored on non-Windows platforms. +interpreted as a directory. (On Windows, symlink requires Windows 6.0 +or greater, and raises a NotImplementedError otherwise.) +target_is_directory is ignored on non-Windows platforms. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_symlink_impl(PyObject *module, path_t *src, path_t *dst, int target_is_directory, int dir_fd) -/*[clinic end generated code: output=08ca9f3f3cf960f6 input=e820ec4472547bc3]*/ +/*[clinic end generated code: output=08ca9f3f3cf960f6 input=71b75467b31c45f7]*/ { #ifdef MS_WINDOWS DWORD result; @@ -11627,19 +11636,18 @@ os.open -> int * dir_fd: dir_fd(requires='openat') = None -# "open(path, flags, mode=0o777, *, dir_fd=None)\n\n\ - Open a file for low level IO. Returns a file descriptor (integer). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static int os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) -/*[clinic end generated code: output=abc7227888c8bc73 input=ad8623b29acd2934]*/ +/*[clinic end generated code: output=abc7227888c8bc73 input=75f7b4eaf92f2225]*/ { int fd; int async_err = 0; @@ -11913,7 +11921,6 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) /*[clinic input] -@permit_long_docstring_body os.lseek -> Py_off_t fd: int @@ -11929,12 +11936,13 @@ os.lseek -> Py_off_t Set the position of a file descriptor. Return the new position. -The return value is the number of bytes relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of +the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=4a3de549f07e1c40]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=32ea0788da7cb44b]*/ { Py_off_t result; @@ -11998,7 +12006,6 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) } /*[clinic input] -@permit_long_docstring_body os.readinto -> Py_ssize_t fd: int buffer: Py_buffer(accept={rwbuffer}) @@ -12006,20 +12013,20 @@ os.readinto -> Py_ssize_t Read into a buffer object from a file descriptor. -The buffer should be mutable and bytes-like. On success, returns the number of -bytes read. Less bytes may be read than the size of the buffer. The underlying -system call will be retried when interrupted by a signal, unless the signal -handler raises an exception. Other errors will not be retried and an error will -be raised. +The buffer should be mutable and bytes-like. On success, returns the +number of bytes read. Less bytes may be read than the size of the +buffer. The underlying system call will be retried when interrupted by +a signal, unless the signal handler raises an exception. Other errors +will not be retried and an error will be raised. -Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0 -(which can be used to check for errors without reading data). Never returns -negative. +Returns 0 if *fd* is at end of file or if the provided *buffer* has +length 0 (which can be used to check for errors without reading data). +Never returns negative. [clinic start generated code]*/ static Py_ssize_t os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer) -/*[clinic end generated code: output=8091a3513c683a80 input=a770382bd3d32f9a]*/ +/*[clinic end generated code: output=8091a3513c683a80 input=2a5f8b212cb5730c]*/ { assert(buffer->len >= 0); Py_ssize_t result = _Py_read(fd, buffer->buf, buffer->len); @@ -12203,7 +12210,7 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) #if defined(HAVE_PREADV) || defined (HAVE_PREADV2) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary os.preadv -> Py_ssize_t fd: int @@ -12214,14 +12221,15 @@ os.preadv -> Py_ssize_t Reads from a file descriptor into a number of mutable bytes-like objects. -Combines the functionality of readv() and pread(). As readv(), it will -transfer data into each buffer until it is full and then move on to the next -buffer in the sequence to hold the rest of the data. Its fourth argument, -specifies the file offset at which the input operation is to be performed. It -will return the total number of bytes read (which can be less than the total -capacity of all the objects). +Combines the functionality of readv() and pread(). As readv(), it will +transfer data into each buffer until it is full and then move on to the +next buffer in the sequence to hold the rest of the data. Its fourth +argument, specifies the file offset at which the input operation is to +be performed. It will return the total number of bytes read (which can +be less than the total capacity of all the objects). -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_HIPRI - RWF_NOWAIT @@ -12233,7 +12241,7 @@ Using non-zero flags requires Linux 4.6 or newer. static Py_ssize_t os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=26fc9c6e58e7ada5 input=34fb3b9ca06f7ba7]*/ +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=bbc70c63b4f4e877]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -12570,6 +12578,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, #if defined(__APPLE__) /*[clinic input] +@permit_long_summary os._fcopyfile in_fd: int @@ -12582,7 +12591,7 @@ Efficiently copy content or metadata of 2 regular file descriptors (macOS). static PyObject * os__fcopyfile_impl(PyObject *module, int in_fd, int out_fd, int flags) -/*[clinic end generated code: output=c9d1a35a992e401b input=1e34638a86948795]*/ +/*[clinic end generated code: output=c9d1a35a992e401b input=80b53ad8863c9101]*/ { int ret; @@ -12771,6 +12780,7 @@ os_pipe2_impl(PyObject *module, int flags) #ifdef HAVE_WRITEV /*[clinic input] +@permit_long_summary os.writev -> Py_ssize_t fd: int buffers: object @@ -12784,7 +12794,7 @@ buffers must be a sequence of bytes-like objects. static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) -/*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ +/*[clinic end generated code: output=56565cfac3aac15b input=5771a0f0c2b326f2]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -12860,7 +12870,6 @@ os_pwrite_impl(PyObject *module, int fd, Py_buffer *buffer, Py_off_t offset) #if defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) /*[clinic input] @permit_long_summary -@permit_long_docstring_body os.pwritev -> Py_ssize_t fd: int @@ -12871,14 +12880,16 @@ os.pwritev -> Py_ssize_t Writes the contents of bytes-like objects to a file descriptor at a given offset. -Combines the functionality of writev() and pwrite(). All buffers must be a sequence -of bytes-like objects. Buffers are processed in array order. Entire contents of first -buffer is written before proceeding to second, and so on. The operating system may -set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. -This function writes the contents of each object to the file descriptor and returns -the total number of bytes written. +Combines the functionality of writev() and pwrite(). All buffers must be +a sequence of bytes-like objects. Buffers are processed in array order. +Entire contents of first buffer is written before proceeding to second, +and so on. The operating system may set a limit (sysconf() value +SC_IOV_MAX) on the number of buffers that can be used. +This function writes the contents of each object to the file descriptor +and returns the total number of bytes written. -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_DSYNC - RWF_SYNC @@ -12892,7 +12903,7 @@ Using non-zero flags requires Linux 4.7 or newer. static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=7de72245873f56bf]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=b2e352a22f030e9a]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -13105,15 +13116,16 @@ os.mkfifo Create a "fifo" (a POSIX named pipe). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=ce41cfad0e68c940 input=73032e98a36e0e19]*/ +/*[clinic end generated code: output=ce41cfad0e68c940 input=d2fb917c01e888d6]*/ { int result; int async_err = 0; @@ -13156,7 +13168,6 @@ os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) #if defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV) /*[clinic input] -@permit_long_docstring_body os.mknod path: path_t @@ -13167,23 +13178,24 @@ os.mknod Create a node in the file system. -Create a node in the file system (file, device special file or named pipe) -at path. mode specifies both the permissions to use and the +Create a node in the file system (file, device special file or named +pipe) at path. mode specifies both the permissions to use and the type of node to be created, being combined (bitwise OR) with one of -S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode, -device defines the newly created device special file (probably using -os.makedev()). Otherwise device is ignored. +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set +on mode, device defines the newly created device special file (probably +using os.makedev()). Otherwise device is ignored. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, int dir_fd) -/*[clinic end generated code: output=92e55d3ca8917461 input=7121c4723d22545b]*/ +/*[clinic end generated code: output=92e55d3ca8917461 input=7d0099e85c6b4cba]*/ { int result; int async_err = 0; @@ -13352,13 +13364,14 @@ os.truncate Truncate a file, specified by path, to a specific length. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) -/*[clinic end generated code: output=43009c8df5c0a12b input=77229cf0b50a9b77]*/ +/*[clinic end generated code: output=43009c8df5c0a12b input=ce33fd7808a511c4]*/ { int result; #ifdef MS_WINDOWS @@ -13401,6 +13414,7 @@ os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) OSs, support was dropped in WASI preview2. */ #if defined(HAVE_POSIX_FALLOCATE) && !defined(__wasi__) /*[clinic input] +@permit_long_summary os.posix_fallocate fd: int @@ -13411,13 +13425,14 @@ os.posix_fallocate Ensure a file has allocated at least a particular number of bytes on disk. Ensure that the file specified by fd encompasses a range of bytes -starting at offset bytes from the beginning and continuing for length bytes. +starting at offset bytes from the beginning and continuing for length +bytes. [clinic start generated code]*/ static PyObject * os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length) -/*[clinic end generated code: output=73f107139564aa9d input=d7a2ef0ab2ca52fb]*/ +/*[clinic end generated code: output=73f107139564aa9d input=c718971d18b96896]*/ { int result; int async_err = 0; @@ -13452,8 +13467,8 @@ os.posix_fadvise Announce an intention to access data in a specific pattern. -Announce an intention to access data in a specific pattern, thus allowing -the kernel to make optimizations. +Announce an intention to access data in a specific pattern, thus +allowing the kernel to make optimizations. The advice applies to the region of the file specified by fd starting at offset and continuing for length bytes. advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL, @@ -13464,7 +13479,7 @@ POSIX_FADV_DONTNEED. static PyObject * os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length, int advice) -/*[clinic end generated code: output=412ef4aa70c98642 input=0fbe554edc2f04b5]*/ +/*[clinic end generated code: output=412ef4aa70c98642 input=961b01a4518ef727]*/ { int result; int async_err = 0; @@ -13663,6 +13678,10 @@ static PyObject * os__clearenv_impl(PyObject *module) /*[clinic end generated code: output=2d6705d62c014b51 input=47d2fa7f323c43ca]*/ { + if (PySys_Audit("os._clearenv", NULL) < 0) { + return NULL; + } + errno = 0; int err = clearenv(); if (err) { @@ -13720,6 +13739,7 @@ os_WCOREDUMP_impl(PyObject *module, int status) #ifdef WIFCONTINUED /*[clinic input] +@permit_long_summary os.WIFCONTINUED -> bool status: int @@ -13732,7 +13752,7 @@ job control stop. static int os_WIFCONTINUED_impl(PyObject *module, int status) -/*[clinic end generated code: output=1e35295d844364bd input=e777e7d38eb25bd9]*/ +/*[clinic end generated code: output=1e35295d844364bd input=7b577845a0f8b12f]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -13845,6 +13865,7 @@ os_WTERMSIG_impl(PyObject *module, int status) #ifdef WSTOPSIG /*[clinic input] +@permit_long_summary os.WSTOPSIG -> int status: int @@ -13854,7 +13875,7 @@ Return the signal that stopped the process that provided the status value. static int os_WSTOPSIG_impl(PyObject *module, int status) -/*[clinic end generated code: output=0ab7586396f5d82b input=46ebf1d1b293c5c1]*/ +/*[clinic end generated code: output=0ab7586396f5d82b input=4698db1a6a320433]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -14045,13 +14066,14 @@ os.statvfs Perform a statvfs system call on the given path. path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_statvfs_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ +/*[clinic end generated code: output=87106dd1beb8556e input=1cfd9a4fd36f7425]*/ { int result; @@ -14100,6 +14122,7 @@ os_statvfs_impl(PyObject *module, path_t *path) #ifdef MS_WINDOWS /*[clinic input] +@permit_long_summary os._getdiskusage path: path_t @@ -14109,7 +14132,7 @@ Return disk usage statistics about the given path as a (total, free) tuple. static PyObject * os__getdiskusage_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3bd3991f5e5c5dfb input=6af8d1b7781cc042]*/ +/*[clinic end generated code: output=3bd3991f5e5c5dfb input=aee7e38bc3e7fb08]*/ { BOOL retval; ULARGE_INTEGER _, total, free; @@ -14330,13 +14353,14 @@ os.pathconf -> long Return the configuration limit name for the file or directory path. If there is no limit, return -1. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static long os_pathconf_impl(PyObject *module, path_t *path, int name) -/*[clinic end generated code: output=5bedee35b293a089 input=6f6072f57b10c787]*/ +/*[clinic end generated code: output=5bedee35b293a089 input=e86f6eacfa006426]*/ { long limit; @@ -15141,13 +15165,13 @@ os.abort Abort the interpreter immediately. -This function 'dumps core' or otherwise fails in the hardest way possible -on the hosting operating system. This function never returns. +This function 'dumps core' or otherwise fails in the hardest way +possible on the hosting operating system. This function never returns. [clinic start generated code]*/ static PyObject * os_abort_impl(PyObject *module) -/*[clinic end generated code: output=dcf52586dad2467c input=cf2c7d98bc504047]*/ +/*[clinic end generated code: output=dcf52586dad2467c input=ee8bd0ed690440ab]*/ { abort(); /*NOTREACHED*/ @@ -15356,6 +15380,7 @@ os_setresgid_impl(PyObject *module, gid_t rgid, gid_t egid, gid_t sgid) #ifdef HAVE_GETRESUID /*[clinic input] +@permit_long_summary os.getresuid Return a tuple of the current process's real, effective, and saved user ids. @@ -15363,7 +15388,7 @@ Return a tuple of the current process's real, effective, and saved user ids. static PyObject * os_getresuid_impl(PyObject *module) -/*[clinic end generated code: output=8e0becff5dece5bf input=41ccfa8e1f6517ad]*/ +/*[clinic end generated code: output=8e0becff5dece5bf input=ddf95881f492cb97]*/ { uid_t ruid, euid, suid; if (getresuid(&ruid, &euid, &suid) < 0) @@ -15408,17 +15433,18 @@ os.getxattr Return the value of extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, getxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, getxattr will examine the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=5f2f44200a43cff2 input=025789491708f7eb]*/ +/*[clinic end generated code: output=5f2f44200a43cff2 input=db1021ed738d9754]*/ { if (fd_and_follow_symlinks_invalid("getxattr", path->is_fd, follow_symlinks)) return NULL; @@ -15465,7 +15491,6 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, /*[clinic input] -@permit_long_docstring_body os.setxattr path: path_t(allow_fd=True) @@ -15477,17 +15502,18 @@ os.setxattr Set extended attribute attribute on path to value. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, setxattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, setxattr will modify the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_buffer *value, int flags, int follow_symlinks) -/*[clinic end generated code: output=98b83f63fdde26bb input=4098e6f68699f3d7]*/ +/*[clinic end generated code: output=98b83f63fdde26bb input=6c4ee6724e8947a4]*/ { ssize_t result; @@ -15530,17 +15556,18 @@ os.removexattr Remove extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, removexattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, removexattr will modify the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=521a51817980cda6 input=3d9a7d36fe2f7c4e]*/ +/*[clinic end generated code: output=521a51817980cda6 input=a7ec62a86aa83f01]*/ { ssize_t result; @@ -15569,7 +15596,6 @@ os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, /*[clinic input] -@permit_long_docstring_body os.listxattr path: path_t(allow_fd=True, nullable=True) = None @@ -15578,16 +15604,17 @@ os.listxattr Return a list of extended attributes on path. -path may be either None, a string, a path-like object, or an open file descriptor. -if path is None, listxattr will examine the current directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, listxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either None, a string, a path-like object, or an open file +descriptor. If path is None, listxattr will examine the current +directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, listxattr will examine the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) -/*[clinic end generated code: output=bebdb4e2ad0ce435 input=48aa9ac8be47dea1]*/ +/*[clinic end generated code: output=bebdb4e2ad0ce435 input=cb4a6414afaa99bd]*/ { Py_ssize_t i; PyObject *result = NULL; @@ -16490,7 +16517,7 @@ static PyMemberDef DirEntry_members[] = { {"name", Py_T_OBJECT_EX, offsetof(DirEntry, name), Py_READONLY, "the entry's base filename, relative to scandir() \"path\" argument"}, {"path", Py_T_OBJECT_EX, offsetof(DirEntry, path), Py_READONLY, - "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)"}, + "the entry's full path name; equivalent to\nos.path.join(scandir_path, entry.name)"}, {NULL} }; @@ -16505,7 +16532,7 @@ static PyMethodDef DirEntry_methods[] = { OS_DIRENTRY_INODE_METHODDEF OS_DIRENTRY___FSPATH___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("DirEntry is generic over the type of the path (str or bytes)")}, {NULL} }; @@ -16984,16 +17011,16 @@ os.scandir Return an iterator of DirEntry objects for given path. -path can be specified as either str, bytes, or a path-like object. If path -is bytes, the names of yielded DirEntry objects will also be bytes; in -all other circumstances they will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the names of yielded DirEntry objects will also be bytes; +in all other circumstances they will be str. If path is None, uses the path='.'. [clinic start generated code]*/ static PyObject * os_scandir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=6eb2668b675ca89e input=6bdd312708fc3bb0]*/ +/*[clinic end generated code: output=6eb2668b675ca89e input=6ab9600993f51577]*/ { ScandirIterator *iterator; #ifdef MS_WINDOWS @@ -17140,21 +17167,20 @@ PyOS_FSPath(PyObject *path) } /*[clinic input] -@permit_long_docstring_body os.fspath path: object Return the file system path representation of the object. -If the object is str or bytes, then allow it to pass through as-is. If the -object defines __fspath__(), then return the result of that method. All other -types raise a TypeError. +If the object is str or bytes, then allow it to pass through as-is. If +the object defines __fspath__(), then return the result of that method. +All other types raise a TypeError. [clinic start generated code]*/ static PyObject * os_fspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=c3c3b78ecff2914f input=f608743e60a3211e]*/ +/*[clinic end generated code: output=c3c3b78ecff2914f input=d3c54404240d5da0]*/ { return PyOS_FSPath(path); } @@ -17331,13 +17357,13 @@ On Unix: On Windows, return status shifted right by 8 bits. On Unix, if the process is being traced or if waitpid() was called with -WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true. -This function must not be called if WIFSTOPPED(status) is true. +WUNTRACED option, the caller must first check if WIFSTOPPED(status) is +true. This function must not be called if WIFSTOPPED(status) is true. [clinic start generated code]*/ static PyObject * os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) -/*[clinic end generated code: output=db50b1b0ba3c7153 input=7fe2d7fdaea3db42]*/ +/*[clinic end generated code: output=db50b1b0ba3c7153 input=3b44a23f5090006c]*/ { #ifndef MS_WINDOWS int status = PyLong_AsInt(status_obj); diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0f0afe17513ef1c..d204b6f27d99082 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -393,7 +393,7 @@ my_CharacterDataHandler(void *userData, const XML_Char *data, int len) if (self->buffer == NULL) call_character_handler(self, data, len); else { - if ((self->buffer_used + len) > self->buffer_size) { + if (len > (self->buffer_size - self->buffer_used)) { if (flush_character_buffer(self) < 0) return; /* handler might have changed; drop the rest on the floor @@ -830,6 +830,7 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) /*[clinic input] +@permit_long_summary pyexpat.xmlparser.SetReparseDeferralEnabled enabled: bool @@ -841,7 +842,7 @@ Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. static PyObject * pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, int enabled) -/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +/*[clinic end generated code: output=5ec539e3b63c8c49 input=6d3743500dcee799]*/ { #if XML_COMBINED_VERSION >= 20600 XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); @@ -1053,18 +1054,19 @@ pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary pyexpat.xmlparser.GetInputContext Return the untranslated text of the input that caused the current event. -If the event was generated by a large amount of text (such as a start tag -for an element with many attributes), not all of the text may be available. +If the event was generated by a large amount of text (such as +a start tag for an element with many attributes), not all of the +text may be available. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=925cea010fdfa682]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=a672f48f09bb73d2]*/ { if (self->in_callback) { int offset, size; @@ -1191,7 +1193,6 @@ pyexpat_xmlparser_SetParamEntityParsing_impl(xmlparseobject *self, int flag) #if XML_COMBINED_VERSION >= 19505 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.UseForeignDTD cls: defining_class @@ -1200,15 +1201,16 @@ pyexpat.xmlparser.UseForeignDTD Allows the application to provide an artificial external subset if one is not specified as part of the document instance. -This readily allows the use of a 'default' document type controlled by the -application, while still getting the advantage of providing document type -information to the parser. 'flag' defaults to True if not provided. +This readily allows the use of a 'default' document type controlled +by the application, while still getting the advantage of providing +document type information to the parser. 'flag' defaults to True if +not provided. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag) -/*[clinic end generated code: output=d7d98252bd25a20f input=c2264845d8c0029c]*/ +/*[clinic end generated code: output=d7d98252bd25a20f input=2920baa5bf24714d]*/ { pyexpat_state *state = PyType_GetModuleState(cls); enum XML_Error rc; @@ -1268,7 +1270,6 @@ set_maximum_amplification(xmlparseobject *self, #if XML_COMBINED_VERSION >= 20400 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold cls: defining_class @@ -1277,21 +1278,22 @@ pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold Sets the number of output bytes needed to activate protection against billion laughs attacks. -The number of output bytes includes amplification from entity expansion -and reading DTD files. +The number of output bytes includes amplification from entity +expansion and reading DTD files. -Parser objects usually have a protection activation threshold of 8 MiB, -but the actual default value depends on the underlying Expat library. +Parser objects usually have a protection activation threshold of +8 MiB, but the actual default value depends on the underlying Expat +library. -Activation thresholds below 4 MiB are known to break support for DITA 1.3 -payload and are hence not recommended. +Activation thresholds below 4 MiB are known to break support for +DITA 1.3 payload and are hence not recommended. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, PyTypeObject *cls, unsigned long long threshold) -/*[clinic end generated code: output=0c082342f1c78114 input=fa2f91f26b62a42a]*/ +/*[clinic end generated code: output=0c082342f1c78114 input=8d84b0e3a873cdba]*/ { return set_activation_threshold( self, cls, threshold, @@ -1303,7 +1305,6 @@ pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlpa #if XML_COMBINED_VERSION >= 20400 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification cls: defining_class @@ -1312,25 +1313,28 @@ pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification Sets the maximum tolerated amplification factor for protection against billion laughs attacks. -The amplification factor is calculated as "(direct + indirect) / direct" -while parsing, where "direct" is the number of bytes read from the primary -document in parsing and "indirect" is the number of bytes added by expanding -entities and reading external DTD files, combined. +The amplification factor is calculated as "(direct + indirect) / +direct" while parsing, where "direct" is the number of bytes read +from the primary document in parsing and "indirect" is the number of +bytes added by expanding entities and reading external DTD files, +combined. -The 'max_factor' value must be a non-NaN floating point value greater than -or equal to 1.0. Amplification factors greater than 30,000 can be observed -in the middle of parsing even with benign files in practice. In particular, -the activation threshold should be carefully chosen to avoid false positives. +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +30,000 can be observed in the middle of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. Parser objects usually have a maximum amplification factor of 100, -but the actual default value depends on the underlying Expat library. +but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, PyTypeObject *cls, float max_factor) -/*[clinic end generated code: output=c590439eadf463fa input=cc1e97c1fd2bd950]*/ +/*[clinic end generated code: output=c590439eadf463fa input=d0f11971c5b9e98b]*/ { return set_maximum_amplification( self, cls, max_factor, @@ -1342,7 +1346,6 @@ pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlp #if XML_COMBINED_VERSION >= 20702 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetAllocTrackerActivationThreshold cls: defining_class @@ -1351,15 +1354,16 @@ pyexpat.xmlparser.SetAllocTrackerActivationThreshold Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. -Parser objects usually have an allocation activation threshold of 64 MiB, -but the actual default value depends on the underlying Expat library. +Parser objects usually have an allocation activation threshold of +64 MiB, but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, PyTypeObject *cls, unsigned long long threshold) -/*[clinic end generated code: output=bed7e93207ba08c5 input=b7a7a3e3d054286a]*/ +/*[clinic end generated code: output=bed7e93207ba08c5 input=4728360b545de87a]*/ { return set_activation_threshold( self, cls, threshold, @@ -1371,7 +1375,6 @@ pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, #if XML_COMBINED_VERSION >= 20702 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetAllocTrackerMaximumAmplification cls: defining_class @@ -1380,25 +1383,27 @@ pyexpat.xmlparser.SetAllocTrackerMaximumAmplification Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated. -The amplification factor is calculated as "allocated / direct" while parsing, -where "direct" is the number of bytes read from the primary document in parsing -and "allocated" is the number of bytes of dynamic memory allocated in the parser -hierarchy. +The amplification factor is calculated as "allocated / direct" while +parsing, where "direct" is the number of bytes read from the primary +document in parsing and "allocated" is the number of bytes of +dynamic memory allocated in the parser hierarchy. -The 'max_factor' value must be a non-NaN floating point value greater than -or equal to 1.0. Amplification factors greater than 100.0 can be observed -near the start of parsing even with benign files in practice. In particular, -the activation threshold should be carefully chosen to avoid false positives. +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +100.0 can be observed near the start of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. Parser objects usually have a maximum amplification factor of 100, -but the actual default value depends on the underlying Expat library. +but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, PyTypeObject *cls, float max_factor) -/*[clinic end generated code: output=6e44bd48c9b112a0 input=c6af7ccb76ae5c6b]*/ +/*[clinic end generated code: output=6e44bd48c9b112a0 input=dd23ea3ef2069b69]*/ { return set_maximum_amplification( self, cls, max_factor, @@ -1533,7 +1538,10 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* This feature was added upstream in libexpat 2.8.0. */ + XML_SetHashSalt16Bytes(self->itself, _Py_HashSecret.expat.hashsalt16); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); @@ -2427,6 +2435,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi->SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi->SetHashSalt16Bytes = NULL; +#endif #if XML_COMBINED_VERSION >= 20600 capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; #else diff --git a/Modules/readline.c b/Modules/readline.c index 488332f548e5fe3..c580d2022fccf3d 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -432,6 +432,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, /* Set history length */ /*[clinic input] +@permit_long_summary readline.set_history_length length: int @@ -444,7 +445,7 @@ A negative length is used to inhibit history truncation. static PyObject * readline_set_history_length_impl(PyObject *module, int length) -/*[clinic end generated code: output=e161a53e45987dc7 input=b8901bf16488b760]*/ +/*[clinic end generated code: output=e161a53e45987dc7 input=8d02c81b38ef81ec]*/ { FT_ATOMIC_STORE_INT_RELAXED(_history_length, length); Py_RETURN_NONE; @@ -453,6 +454,7 @@ readline_set_history_length_impl(PyObject *module, int length) /* Get history length */ /*[clinic input] +@permit_long_summary readline.get_history_length Return the maximum number of lines that will be written to the history file. @@ -460,7 +462,7 @@ Return the maximum number of lines that will be written to the history file. static PyObject * readline_get_history_length_impl(PyObject *module) -/*[clinic end generated code: output=83a2eeae35b6d2b9 input=5dce2eeba4327817]*/ +/*[clinic end generated code: output=83a2eeae35b6d2b9 input=a65823e732ebfa9d]*/ { int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length); return PyLong_FromLong(history_length); @@ -1404,6 +1406,10 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ + + if (!completer_word_break_characters) { + goto error; + } #ifdef WITH_EDITLINE // libedit uses rl_basic_word_break_characters instead of // rl_completer_word_break_characters as complete delimiter @@ -1447,6 +1453,10 @@ setup_readline(readlinestate *mod_state) RESTORE_LOCALE(saved_locale) return 0; + +error: + RESTORE_LOCALE(saved_locale) + return -1; } /* Wrapper around GNU readline that handles signals differently. */ diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index eb3148ef24631bb..2c56dbc6a541f7a 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -242,7 +242,6 @@ set2list(fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) #endif /* FD_SETSIZE > 1024 */ /*[clinic input] -@permit_long_docstring_body select.select rlist: object @@ -253,7 +252,8 @@ select.select Wait until one or more file descriptors are ready for some kind of I/O. -The first three arguments are iterables of file descriptors to be waited for: +The first three arguments are iterables of file descriptors to be waited +for: rlist -- wait until ready for reading wlist -- wait until ready for writing xlist -- wait for an "exceptional condition" @@ -266,9 +266,9 @@ The optional 4th argument specifies a timeout in seconds; it may be a non-integer to specify fractions of seconds. If it is absent or None, the call will never time out. -The return value is a tuple of three lists corresponding to the first three -arguments; each contains the subset of the corresponding file descriptors -that are ready. +The return value is a tuple of three lists corresponding to the first +three arguments; each contains the subset of the corresponding file +descriptors that are ready. *** IMPORTANT NOTICE *** On Windows, only sockets are supported; on Unix, all file @@ -278,7 +278,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=b0403de75cd11cc1]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=cc93e9bb9ffacbaf]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -616,13 +616,13 @@ select.poll.poll Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=e0a9c0aa283de8c8]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -975,19 +975,19 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) @critical_section select.devpoll.poll timeout as timeout_obj: object = None - The maximum time to wait in milliseconds, or else None (or a negative - value) to wait indefinitely. + The maximum time to wait in milliseconds, or else None (or + a negative value) to wait indefinitely. / Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=9e1672658d728539]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1233,18 +1233,17 @@ static PyType_Spec devpoll_Type_spec = { /*[clinic input] -@permit_long_docstring_body select.poll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_poll_impl(PyObject *module) -/*[clinic end generated code: output=16a665a4e1d228c5 input=5e07eea8ad564e7f]*/ +/*[clinic end generated code: output=16a665a4e1d228c5 input=0aefd4527e99e0aa]*/ { return (PyObject *)newPollObject(module); } @@ -1252,18 +1251,17 @@ select_poll_impl(PyObject *module) #ifdef HAVE_SYS_DEVPOLL_H /*[clinic input] -@permit_long_docstring_body select.devpoll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_devpoll_impl(PyObject *module) -/*[clinic end generated code: output=ea9213cc87fd9581 input=048506faef19d947]*/ +/*[clinic end generated code: output=ea9213cc87fd9581 input=4c2ac27d10248526]*/ { return (PyObject *)newDevPollObject(module); } @@ -1540,6 +1538,7 @@ pyepoll_internal_ctl(int epfd, int op, int fd, unsigned int events) } /*[clinic input] +@permit_long_summary select.epoll.register fd: fildes @@ -1555,7 +1554,7 @@ The epoll interface supports all file descriptors that support poll. static PyObject * select_epoll_register_impl(pyEpoll_Object *self, int fd, unsigned int eventmask) -/*[clinic end generated code: output=318e5e6386520599 input=a5071b71edfe3578]*/ +/*[clinic end generated code: output=318e5e6386520599 input=9f0c9ebb25a4fc8f]*/ { return pyepoll_internal_ctl(self->epfd, EPOLL_CTL_ADD, fd, eventmask); } @@ -1606,14 +1605,14 @@ select.epoll.poll Wait for events on the epoll file descriptor. -Returns a list containing any descriptors that have events to report, -as a list of (fd, events) 2-tuples. +Returns a list containing any descriptors that have events to +report, as a list of (fd, events) 2-tuples. [clinic start generated code]*/ static PyObject * select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents) -/*[clinic end generated code: output=e02d121a20246c6c input=deafa7f04a60ebe0]*/ +/*[clinic end generated code: output=e02d121a20246c6c input=911ddc16978a9159]*/ { int nfds, i; PyObject *elist = NULL, *etuple = NULL; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fb548b8ca00f24e..8456239dee202d3 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -450,7 +450,6 @@ signal_raise_signal_impl(PyObject *module, int signalnum) } /*[clinic input] -@permit_long_docstring_body signal.signal signalnum: int @@ -460,16 +459,17 @@ signal.signal Set the action for the given signal. The action can be SIG_DFL, SIG_IGN, or a callable Python object. -The previous action is returned. See getsignal() for possible return values. +The previous action is returned. See getsignal() for possible return +values. *** IMPORTANT NOTICE *** -A signal handler function is called with two arguments: -the first is the signal number, the second is the interrupted stack frame. +A signal handler function is called with two arguments: the first is +the signal number, the second is the interrupted stack frame. [clinic start generated code]*/ static PyObject * signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) -/*[clinic end generated code: output=b44cfda43780f3a1 input=7608656f34fa378b]*/ +/*[clinic end generated code: output=b44cfda43780f3a1 input=99ce4035ec56ffc1]*/ { _signal_module_state *modstate = get_signal_state(module); PyObject *old_handler; @@ -839,7 +839,6 @@ PySignal_SetWakeupFd(int fd) #ifdef HAVE_SETITIMER /*[clinic input] -@permit_long_docstring_body signal.setitimer which: int @@ -849,8 +848,8 @@ signal.setitimer Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF). -The timer will fire after value seconds and after that every interval seconds. -The itimer can be cleared by setting seconds to zero. +The timer will fire after value seconds and after that every interval +seconds. The itimer can be cleared by setting seconds to zero. Returns old values as a tuple: (delay, interval). [clinic start generated code]*/ @@ -858,7 +857,7 @@ Returns old values as a tuple: (delay, interval). static PyObject * signal_setitimer_impl(PyObject *module, int which, PyObject *seconds, PyObject *interval) -/*[clinic end generated code: output=65f9dcbddc35527b input=ab5bf2b8f5cff3f4]*/ +/*[clinic end generated code: output=65f9dcbddc35527b input=bd9f0d2ed8614193]*/ { _signal_module_state *modstate = get_signal_state(module); @@ -1019,13 +1018,13 @@ signal.sigwait Wait for a signal. Suspend execution of the calling thread until the delivery of one of the -signals specified in the signal set sigset. The function accepts the signal -and returns the signal number. +signals specified in the signal set sigset. The function accepts the +signal and returns the signal number. [clinic start generated code]*/ static PyObject * signal_sigwait_impl(PyObject *module, sigset_t sigset) -/*[clinic end generated code: output=f43770699d682f96 input=a6fbd47b1086d119]*/ +/*[clinic end generated code: output=f43770699d682f96 input=91773742dd416a3e]*/ { int err, signum; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f5993fc8fdaab28..cf7aadfe95a721f 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4620,7 +4620,6 @@ sock_send_impl(PySocketSockObject *s, void *data) } /*[clinic input] -@permit_long_docstring_body _socket.socket.send self as s: self(type="PySocketSockObject *") data as pbuf: Py_buffer @@ -4630,12 +4629,13 @@ _socket.socket.send Send a data string to the socket. For the optional flags argument, see the Unix manual. -Return the number of bytes sent; this may be less than len(data) if the network is busy. +Return the number of bytes sent; this may be less than len(data) if +the network is busy. [clinic start generated code]*/ static PyObject * _socket_socket_send_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags) -/*[clinic end generated code: output=3ddf83f17d0c875b input=e776a48af2e3d615]*/ +/*[clinic end generated code: output=3ddf83f17d0c875b input=d2b8af9bf99cfafb]*/ { struct sock_send ctx; @@ -4665,13 +4665,14 @@ Send a data string to the socket. For the optional flags argument, see the Unix manual. This calls send() repeatedly until all data is sent. -If an error occurs, it's impossible to tell how much data has been sent. +If an error occurs, it's impossible to tell how much data has been +sent. [clinic start generated code]*/ static PyObject * _socket_socket_sendall_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags) -/*[clinic end generated code: output=ec92861424d3faa8 input=732b15b9ca64dce6]*/ +/*[clinic end generated code: output=ec92861424d3faa8 input=2600de13b4614893]*/ { char *buf; @@ -4921,20 +4922,20 @@ The buffers argument specifies the non-ancillary data as an iterable of bytes-like objects (e.g. bytes objects). The ancdata argument specifies the ancillary data (control messages) as an iterable of zero or more tuples (cmsg_level, cmsg_type, -cmsg_data), where cmsg_level and cmsg_type are integers specifying the -protocol level and protocol-specific type respectively, and cmsg_data -is a bytes-like object holding the associated data. The flags -argument defaults to 0 and has the same meaning as for send(). If -address is supplied and not None, it sets a destination address for -the message. The return value is the number of bytes of non-ancillary -data sent. +cmsg_data), where cmsg_level and cmsg_type are integers specifying +the protocol level and protocol-specific type respectively, and +cmsg_data is a bytes-like object holding the associated data. The +flags argument defaults to 0 and has the same meaning as for send(). +If address is supplied and not None, it sets a destination address +for the message. The return value is the number of bytes of +non-ancillary data sent. [clinic start generated code]*/ static PyObject * _socket_socket_sendmsg_impl(PySocketSockObject *s, PyObject *data_arg, PyObject *cmsg_arg, int flags, PyObject *addr_arg) -/*[clinic end generated code: output=3b4cb1110644ce39 input=479c13d90bd2f88b]*/ +/*[clinic end generated code: output=3b4cb1110644ce39 input=8ae408971a3aa329]*/ { Py_ssize_t i, ndatabufs = 0, ncmsgs, ncmsgbufs = 0; @@ -7310,6 +7311,7 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) /*[clinic input] +@permit_long_summary _socket.if_indextoname if_index as index: NET_IFINDEX / @@ -7319,7 +7321,7 @@ Returns the interface name corresponding to the interface index if_index. static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) -/*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ +/*[clinic end generated code: output=e48bc324993052e0 input=2a0026b271cd43ae]*/ { errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; @@ -9283,6 +9285,9 @@ socket_exec(PyObject *m) /* Initialize gethostbyname lock */ #if defined(USE_GETHOSTBYNAME_LOCK) netdb_lock = PyThread_allocate_lock(); + if (netdb_lock == NULL) { + goto error; + } #endif #ifdef MS_WINDOWS diff --git a/Modules/termios.c b/Modules/termios.c index 95b9c920f39c126..38743e176f0bf6c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -312,6 +312,7 @@ termios_tcsendbreak_impl(PyObject *module, int fd, int duration) } /*[clinic input] +@permit_long_summary termios.tcdrain fd: fildes @@ -322,7 +323,7 @@ Wait until all output written to file descriptor fd has been transmitted. static PyObject * termios_tcdrain_impl(PyObject *module, int fd) -/*[clinic end generated code: output=5fd86944c6255955 input=c99241b140b32447]*/ +/*[clinic end generated code: output=5fd86944c6255955 input=d1557e60b5ec66c5]*/ { termiosmodulestate *state = PyModule_GetState(module); int r; @@ -474,7 +475,6 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) } /*[clinic input] -@permit_long_docstring_body termios.tcsetwinsize fd: fildes @@ -484,12 +484,13 @@ termios.tcsetwinsize Set the tty winsize for file descriptor fd. The winsize to be set is taken from the winsize argument, which -is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize(). +is a two-item tuple (ws_row, ws_col) like the one returned by +tcgetwinsize(). [clinic start generated code]*/ static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) -/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=9a163c4e06fc4a41]*/ +/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=efc9beb16d06382a]*/ { if (!PySequence_Check(winsz) || PySequence_Size(winsz) != 2) { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 25e744d7da25c72..d90bf1f2ef90ed9 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -820,12 +820,15 @@ time_strftime1(time_char **outbuf, size_t *bufsize, PyErr_NoMemory(); return NULL; } - *outbuf = (time_char *)PyMem_Realloc(*outbuf, - *bufsize*sizeof(time_char)); - if (*outbuf == NULL) { + time_char *tmp = (time_char *)PyMem_Realloc(*outbuf, + *bufsize*sizeof(time_char)); + if (tmp == NULL) { + PyMem_Free(*outbuf); + *outbuf = NULL; PyErr_NoMemory(); return NULL; } + *outbuf = tmp; #if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) errno = 0; #endif diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 55b33a76e7af8a3..60df68216938134 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -291,6 +291,7 @@ unicodedata_UCD_numeric_impl(PyObject *self, int chr, } /*[clinic input] +@permit_long_summary unicodedata.UCD.category self: self @@ -302,7 +303,7 @@ Returns the general category assigned to the character chr as string. static PyObject * unicodedata_UCD_category_impl(PyObject *self, int chr) -/*[clinic end generated code: output=8571539ee2e6783a input=27d6f3d85050bc06]*/ +/*[clinic end generated code: output=8571539ee2e6783a input=1d729c67299e8a31]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -316,6 +317,7 @@ unicodedata_UCD_category_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.bidirectional self: self @@ -329,7 +331,7 @@ If no such value is defined, an empty string is returned. static PyObject * unicodedata_UCD_bidirectional_impl(PyObject *self, int chr) -/*[clinic end generated code: output=d36310ce2039bb92 input=b3d8f42cebfcf475]*/ +/*[clinic end generated code: output=d36310ce2039bb92 input=838f8a2203bd2990]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -373,6 +375,7 @@ unicodedata_UCD_combining_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.mirrored -> int self: self @@ -387,7 +390,7 @@ character in bidirectional text, 0 otherwise. static int unicodedata_UCD_mirrored_impl(PyObject *self, int chr) -/*[clinic end generated code: output=2532dbf8121b50e6 input=5dd400d351ae6f3b]*/ +/*[clinic end generated code: output=2532dbf8121b50e6 input=6db28989e49cd9c8]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -403,6 +406,7 @@ unicodedata_UCD_mirrored_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.east_asian_width self: self @@ -414,7 +418,7 @@ Returns the east asian width assigned to the character chr as string. static PyObject * unicodedata_UCD_east_asian_width_impl(PyObject *self, int chr) -/*[clinic end generated code: output=484e8537d9ee8197 input=c4854798aab026e0]*/ +/*[clinic end generated code: output=484e8537d9ee8197 input=207c5f68fa475516]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -552,19 +556,80 @@ get_decomp_record(PyObject *self, Py_UCS4 code, (*index)++; } +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(int kind, void *data, + Py_ssize_t start, Py_ssize_t end) +{ + for (Py_ssize_t i = start + 1; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + Py_ssize_t j = i; + + while (j > start) { + Py_UCS4 previous = PyUnicode_READ(kind, data, j - 1); + if (_getrecord_ex(previous)->combining <= combining) { + break; + } + PyUnicode_WRITE(kind, data, j, previous); + j--; + } + if (j != i) { + PyUnicode_WRITE(kind, data, j, code); + } + } +} + +static void +canonical_ordering_sort_counting(int kind, void *data, + Py_ssize_t start, Py_ssize_t end, + Py_UCS4 *sortbuf) +{ + Py_ssize_t counts[256] = {0}; + Py_ssize_t run_length = end - start; + Py_ssize_t total = 0; + + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + counts[combining]++; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) { + Py_ssize_t count = counts[i]; + counts[i] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC. */ + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + sortbuf[counts[combining]++] = code; + } + for (Py_ssize_t i = 0; i < run_length; i++) { + PyUnicode_WRITE(kind, data, start + i, sortbuf[i]); + } +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; Py_UCS4 *output; Py_ssize_t i, o, osize; - int kind; - const void *data; + int input_kind, result_kind; + const void *input_data; + void *result_data; /* Longest decomposition in Unicode 3.2: U+FDFA */ Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UCS4 *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_LENGTH(input); @@ -584,11 +649,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) return NULL; } i = o = 0; - kind = PyUnicode_KIND(input); - data = PyUnicode_DATA(input); + input_kind = PyUnicode_KIND(input); + input_data = PyUnicode_DATA(input); while (i < isize) { - stack[stackptr++] = PyUnicode_READ(kind, data, i++); + stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++); while(stackptr) { Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in @@ -656,34 +721,64 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) if (!result) return NULL; - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); + result_kind = PyUnicode_KIND(result); + result_data = PyUnicode_DATA(result); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. */ i = 0; - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - for (i++; i < PyUnicode_GET_LENGTH(result); i++) { - cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < o) { + Py_ssize_t run_length, run_start; + int needs_sort = 0; + + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + prev = _getrecord_ex(ch)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); - PyUnicode_WRITE(kind, data, o+1, - PyUnicode_READ(kind, data, o)); - PyUnicode_WRITE(kind, data, o, tmp); - o--; - if (o < 0) - break; - prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < o) { + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + cur = _getrecord_ex(ch)->combining; + if (cur == 0) { break; + } + if (prev > cur) { + needs_sort = 1; + } + prev = cur; + i++; } - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + if (!needs_sort) { + continue; + } + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(result_kind, result_data, + run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf, + Py_UCS4, + run_length); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + + canonical_ordering_sort_counting(result_kind, result_data, + run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } @@ -911,6 +1006,7 @@ is_normalized_quickcheck(PyObject *self, PyObject *input, bool nfc, bool k, } /*[clinic input] +@permit_long_summary unicodedata.UCD.is_normalized self: self @@ -926,7 +1022,7 @@ Valid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'. static PyObject * unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form, PyObject *input) -/*[clinic end generated code: output=11e5a3694e723ca5 input=a544f14cea79e508]*/ +/*[clinic end generated code: output=11e5a3694e723ca5 input=de66aa679265300b]*/ { if (PyUnicode_GET_LENGTH(input) == 0) { /* special case empty input strings. */ @@ -1543,32 +1639,17 @@ capi_getcode(const char* name, int namelen, Py_UCS4* code, return _check_alias_and_seq(code, with_named_seq); } -static void -unicodedata_destroy_capi(PyObject *capsule) -{ - void *capi = PyCapsule_GetPointer(capsule, PyUnicodeData_CAPSULE_NAME); - PyMem_Free(capi); -} - static PyObject * unicodedata_create_capi(void) { - _PyUnicode_Name_CAPI *capi = PyMem_Malloc(sizeof(_PyUnicode_Name_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->getname = capi_getucname; - capi->getcode = capi_getcode; - - PyObject *capsule = PyCapsule_New(capi, - PyUnicodeData_CAPSULE_NAME, - unicodedata_destroy_capi); - if (capsule == NULL) { - PyMem_Free(capi); - } - return capsule; -}; + // Statically allocated so that any cached pointers stay valid after unicodedata + // is removed from sys.modules and the capsule is gc'd (gh-149449). + static _PyUnicode_Name_CAPI capi = { + .getname = capi_getucname, + .getcode = capi_getcode, + }; + return PyCapsule_New(&capi, PyUnicodeData_CAPSULE_NAME, NULL); +} /* -------------------------------------------------------------------- */ diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 09c8d9487f54266..96454ee5e83eab7 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -11,7 +11,13 @@ other files, you'll have to create a file "foobarobject.h"; see floatobject.h for an example. - This module roughly corresponds to:: + This module uses Limited API 3.15. + See ``xxlimited_3_13.c`` if you want to support older CPython versions. + + This module roughly corresponds to the following. + (All underscore-prefixed attributes are not accessible from Python.) + + :: class Xxo: """A class that explicitly stores attributes in an internal dict @@ -27,6 +33,8 @@ return self._x_attr[name] def __setattr__(self, name, value): + if name == "reserved": + raise AttributeError("cannot set 'reserved'") self._x_attr[name] = value def __delattr__(self, name): @@ -64,11 +72,13 @@ pass */ -// Need limited C API version 3.13 for Py_mod_gil -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif +// Target both flavors of the Stable ABI. +// Both are set to version 3.15, which adds PyModExport +// (When using a build tool, check if it has an option to set these +// so they do not need to be defined in the source.) +#define Py_LIMITED_API 0x030f0000 // abi3 (GIL-enabled builds) +#define Py_TARGET_ABI3T 0x030f0000 // abi3t (free-threaded builds) + #include "Python.h" #include <string.h> @@ -77,43 +87,135 @@ // Module state typedef struct { - PyObject *Xxo_Type; // Xxo class + PyTypeObject *Xxo_Type; // Xxo class PyObject *Error_Type; // Error class } xx_state; -/* Xxo objects */ +/* Xxo objects. + * + * A non-trivial extension type, intentionally showing a number of features + * that aren't easy to implement in the Limited API. + */ + +// Forward declaration +static PyType_Spec Xxo_Type_spec; + +// Get the module state (xx_state*) from a given type object 'type', which +// must be a subclass of Xxo (the type we're defining). +// This is complicated by the fact that the Xxo type is dynamically allocated, +// and there may be several such types in a given Python process -- for +// example, in different subinterpreters, or through loading this +// extension module several times. +// So, we don't have a "global" pointer to the type, or to the module, etc.; +// instead we search based on `Xxo_Type_spec` (which is static, immutable, +// and process-global). +// +// When possible, it's better to avoid `PyType_GetBaseByToken` -- for an +// example, see the `demo` method (Xxo_demo C function), which uses a +// "defining class". But, in many cases it's the best solution. +static xx_state * +Xxo_state_from_type(PyTypeObject *type) +{ + PyTypeObject *base; + // Search all superclasses of 'type' for one that was defined using + // "Xxo_Type_spec". That must be our 'Xxo' class. + if (PyType_GetBaseByToken(type, &Xxo_Type_spec, &base) < 0) { + return NULL; + } + if (base == NULL) { + PyErr_SetString(PyExc_TypeError, "need Xxo subclass"); + return NULL; + } + // From this type, get the associated module. That must be the + // relevant `xxlimited` module. + xx_state *state = PyType_GetModuleState(base); + Py_DECREF(base); + return state; +} -// Instance state +// Structure for data needed by the XxoObject type. +// Since the object may be shared across threads, access to the fields +// usually needs to be synchronized (using Py_BEGIN_CRITICAL_SECTION). typedef struct { - PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary. - * May be NULL, which acts as an - * empty dict. - */ - char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ - Py_ssize_t x_exports; /* how many buffer are exported */ -} XxoObject; - -#define XxoObject_CAST(op) ((XxoObject *)(op)) -// TODO: full support for type-checking was added in 3.14 (Py_tp_token) -// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) - -static XxoObject * -newXxoObject(PyObject *module) + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + Py_ssize_t x_exports; /* how many buffers are exported */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer (for simplicity, + * this is constant, so does not need + * synchronization) + */ +} XxoObject_Data; + +// Get the `XxoObject_Data` structure for a given instance of our type. +static XxoObject_Data * +Xxo_get_data(PyObject *self) { - xx_state *state = PyModule_GetState(module); + xx_state *state = Xxo_state_from_type(Py_TYPE(self)); + if (!state) { + return NULL; + } + XxoObject_Data *data = PyObject_GetTypeData(self, state->Xxo_Type); + return data; +} + +// A variant of Xxo_get_data to be used in the tp_traverse handler. +// This function cannot have side effects (including reference count +// manipulation, creating objects, and raising exceptions), and must not +// call API functions that might have side effects. +// See: https://docs.python.org/3.15/c-api/gcsupport.html#traversal +static XxoObject_Data * +Xxo_get_data_DuringGC(PyObject *self) +{ + PyTypeObject *base; + PyType_GetBaseByToken_DuringGC(Py_TYPE(self), &Xxo_Type_spec, &base); + if (base == NULL) { + return NULL; + } + xx_state *state = PyType_GetModuleState_DuringGC(base); if (state == NULL) { return NULL; } - XxoObject *self; - self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + XxoObject_Data *data = PyObject_GetTypeData_DuringGC(self, state->Xxo_Type); + return data; +} + +// Xxo initialization +// This is the implementation of Xxo.__new__ +static PyObject * +Xxo_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // Validate that we did not get any arguments. + if ((args != NULL && PyObject_Length(args)) + || (kwargs != NULL && PyObject_Length(kwargs))) + { + PyErr_SetString(PyExc_TypeError, "Xxo.__new__() takes no arguments"); + return NULL; + } + // Create an instance of *type* (which may be a subclass) + allocfunc alloc = PyType_GetSlot(type, Py_tp_alloc); + PyObject *self = alloc(type, 0); if (self == NULL) { return NULL; } - self->x_attr = NULL; - memset(self->x_buffer, 0, BUFSIZE); - self->x_exports = 0; + + // Initialize the C members on the instance. + // This is only included for the sake of example. The default alloc + // function zeroes instance memory; we don't need to do it again. + // Note that we during initialization (and finalization), we hold the only + // reference to the object, so we don't need to synchronize with + // other threads. + XxoObject_Data *xxo_data = Xxo_get_data(self); + if (xxo_data == NULL) { + Py_DECREF(self); + return NULL; + } + + xxo_data->x_attr = NULL; + memset(xxo_data->x_buffer, 0, BUFSIZE); + xxo_data->x_exports = 0; return self; } @@ -125,45 +227,63 @@ newXxoObject(PyObject *module) // traverse: Visit all references from an object, including its type static int -Xxo_traverse(PyObject *op, visitproc visit, void *arg) +Xxo_traverse(PyObject *self, visitproc visit, void *arg) { // Visit the type - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(self)); // Visit the attribute dict - XxoObject *self = XxoObject_CAST(op); - Py_VISIT(self->x_attr); + XxoObject_Data *data = Xxo_get_data_DuringGC(self); + if (data == NULL) { + return 0; + } + Py_VISIT(data->x_attr); return 0; } // clear: drop references in order to break all reference cycles static int -Xxo_clear(PyObject *op) +Xxo_clear(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + Py_CLEAR(data->x_attr); return 0; } // finalize: like clear, but should leave the object in a consistent state. // Equivalent to `__del__` in Python. static void -Xxo_finalize(PyObject *op) +Xxo_finalize(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_CLEAR(data->x_attr); } // dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { + // This function must preserve currently raised exception, if any. + PyObject *exc = PyErr_GetRaisedException(); + PyObject_GC_UnTrack(self); Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); freefunc free = PyType_GetSlot(tp, Py_tp_free); free(self); Py_DECREF(tp); + + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); } @@ -171,11 +291,20 @@ Xxo_dealloc(PyObject *self) // Get an attribute. static PyObject * -Xxo_getattro(PyObject *op, PyObject *name) +Xxo_getattro(PyObject *self, PyObject *name) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr != NULL) { - PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + Py_END_CRITICAL_SECTION(); + + if (x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(x_attr, name); if (v != NULL) { return Py_NewRef(v); } @@ -185,24 +314,42 @@ Xxo_getattro(PyObject *op, PyObject *name) } // Fall back to generic implementation (this handles special attributes, // raising AttributeError, etc.) - return PyObject_GenericGetAttr(op, name); + return PyObject_GenericGetAttr(self, name); } // Set or delete an attribute. static int -Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +Xxo_setattro(PyObject *self, PyObject *name, PyObject *v) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr == NULL) { + // filter a specific attribute name + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "reserved")) { + PyErr_Format(PyExc_AttributeError, "cannot set %R", name); + return -1; + } + + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + + // If the attribute dict is not created yet, make one. + // This needs to be protected by a critical section to avoid another thread + // creating a duplicate dict. + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + if (x_attr == NULL) { // prepare the attribute dict - self->x_attr = PyDict_New(); - if (self->x_attr == NULL) { - return -1; - } + data->x_attr = x_attr = PyDict_New(); } + Py_END_CRITICAL_SECTION(); + if (x_attr == NULL) { + return -1; + } + if (v == NULL) { // delete an attribute - int rv = PyDict_DelItem(self->x_attr, name); + int rv = PyDict_DelItem(x_attr, name); if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetString(PyExc_AttributeError, "delete non-existing Xxo attribute"); @@ -212,7 +359,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } else { // set an attribute - return PyDict_SetItem(self->x_attr, name, v); + return PyDict_SetItem(x_attr, name, v); } } @@ -221,7 +368,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) */ static PyObject * -Xxo_demo(PyObject *op, PyTypeObject *defining_class, +Xxo_demo(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (kwnames != NULL && PyObject_Length(kwnames)) { @@ -260,30 +407,49 @@ static PyMethodDef Xxo_methods[] = { */ static int -Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +Xxo_getbuffer(PyObject *self, Py_buffer *view, int flags) { - XxoObject *self = XxoObject_CAST(op); - int res = PyBuffer_FillInfo(view, op, - (void *)self->x_buffer, BUFSIZE, + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + int res = PyBuffer_FillInfo(view, self, + (void *)data->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - self->x_exports++; + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports++; + Py_END_CRITICAL_SECTION(); } return res; } static void -Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +Xxo_releasebuffer(PyObject *self, Py_buffer *Py_UNUSED(view)) { - XxoObject *self = XxoObject_CAST(op); - self->x_exports--; + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports--; + Py_END_CRITICAL_SECTION(); } static PyObject * -Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +Xxo_get_x_exports(PyObject *self, void *Py_UNUSED(closure)) { - XxoObject *self = XxoObject_CAST(op); - return PyLong_FromSsize_t(self->x_exports); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return NULL; + } + Py_ssize_t result; + + Py_BEGIN_CRITICAL_SECTION(self); + result = data->x_exports; + Py_END_CRITICAL_SECTION(); + + return PyLong_FromSsize_t(result); } /* Xxo type definition */ @@ -299,6 +465,7 @@ static PyGetSetDef Xxo_getsetlist[] = { static PyType_Slot Xxo_Type_slots[] = { {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_new, Xxo_new}, {Py_tp_traverse, Xxo_traverse}, {Py_tp_clear, Xxo_clear}, {Py_tp_finalize, Xxo_finalize}, @@ -309,13 +476,14 @@ static PyType_Slot Xxo_Type_slots[] = { {Py_bf_getbuffer, Xxo_getbuffer}, {Py_bf_releasebuffer, Xxo_releasebuffer}, {Py_tp_getset, Xxo_getsetlist}, + {Py_tp_token, Py_TP_USE_SPEC}, {0, 0}, /* sentinel */ }; static PyType_Spec Xxo_Type_spec = { .name = "xxlimited.Xxo", - .basicsize = sizeof(XxoObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .basicsize = -(Py_ssize_t)sizeof(XxoObject_Data), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, .slots = Xxo_Type_slots, }; @@ -354,17 +522,17 @@ xx_foo(PyObject *module, PyObject *args) } -/* Function of no arguments returning new Xxo object */ +/* Function of no arguments returning new Xxo object. + * Note that a function exposed to Python with METH_NOARGS requires an unused + * second argument, so we cannot use newXxoObject directly. + */ static PyObject * xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) { - XxoObject *rv; + xx_state *state = PyModule_GetState(module); - rv = newXxoObject(module); - if (rv == NULL) - return NULL; - return (PyObject *)rv; + return Xxo_new(state->Xxo_Type, NULL, NULL); } @@ -398,11 +566,12 @@ xx_modexec(PyObject *m) return -1; } - state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + state->Xxo_Type = (PyTypeObject*)PyType_FromModuleAndSpec( + m, &Xxo_Type_spec, NULL); if (state->Xxo_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + if (PyModule_AddType(m, state->Xxo_Type) < 0) { return -1; } @@ -410,12 +579,13 @@ xx_modexec(PyObject *m) // added to the module dict. // It does not inherit from "object" (PyObject_Type), but from "str" // (PyUnincode_Type). - PyObject *Str_Type = PyType_FromModuleAndSpec( + PyTypeObject *Str_Type = (PyTypeObject*)PyType_FromModuleAndSpec( m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); if (Str_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + if (PyModule_AddType(m, Str_Type) < 0) { + Py_DECREF(Str_Type); return -1; } Py_DECREF(Str_Type); @@ -423,29 +593,6 @@ xx_modexec(PyObject *m) return 0; } -static PyModuleDef_Slot xx_slots[] = { - - /* exec function to initialize the module (called as part of import - * after the object was added to sys.modules) - */ - {Py_mod_exec, xx_modexec}, - - /* Signal that this module supports being loaded in multiple interpreters - * with separate GILs (global interpreter locks). - * See "Isolating Extension Modules" on how to prepare a module for this: - * https://docs.python.org/3/howto/isolating-extensions.html - */ - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - - /* Signal that this module does not rely on the GIL for its own needs. - * Without this slot, free-threaded builds of CPython will enable - * the GIL when this module is loaded. - */ - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0, NULL} -}; - // Module finalization: modules that hold references in their module state // need to implement the fullowing GC hooks. They're similar to the ones for // types (see "Xxo finalization"). @@ -453,7 +600,10 @@ static PyModuleDef_Slot xx_slots[] = { static int xx_traverse(PyObject *module, visitproc visit, void *arg) { - xx_state *state = PyModule_GetState(module); + xx_state *state = PyModule_GetState_DuringGC(module); + if (state == NULL) { + return 0; + } Py_VISIT(state->Xxo_Type); Py_VISIT(state->Error_Type); return 0; @@ -463,6 +613,9 @@ static int xx_clear(PyObject *module) { xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return 0; + } Py_CLEAR(state->Xxo_Type); Py_CLEAR(state->Error_Type); return 0; @@ -473,27 +626,59 @@ xx_free(void *module) { // allow xx_modexec to omit calling xx_clear on error (void)xx_clear((PyObject *)module); + + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return; + } } -static struct PyModuleDef xxmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "xxlimited", - .m_doc = module_doc, - .m_size = sizeof(xx_state), - .m_methods = xx_methods, - .m_slots = xx_slots, - .m_traverse = xx_traverse, - .m_clear = xx_clear, - .m_free = xx_free, +// Information that CPython uses to prevent loading incompatible extenstions +PyABIInfo_VAR(abi_info); + +static PySlot xx_slots[] = { + /* Basic metadata */ + PySlot_STATIC_DATA(Py_mod_name, "xxlimited"), + PySlot_STATIC_DATA(Py_mod_doc, (void*)module_doc), + PySlot_DATA(Py_mod_abi, &abi_info), + + /* The method table */ + PySlot_STATIC_DATA(Py_mod_methods, xx_methods), + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + PySlot_FUNC(Py_mod_exec, xx_modexec), + + /* Module state and associated functions */ + PySlot_SIZE(Py_mod_state_size, sizeof(xx_state)), + PySlot_FUNC(Py_mod_state_traverse, xx_traverse), + PySlot_FUNC(Py_mod_state_clear, xx_clear), + PySlot_FUNC(Py_mod_state_free, xx_free), + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + + PySlot_END }; -/* Export function for the module. *Must* be called PyInit_xx; usually it is - * the only non-`static` object in a module definition. +/* Export function for the module. *Must* be called PyModExport_xx; usually + * it is the only non-`static` object in a module definition. */ -PyMODINIT_FUNC -PyInit_xxlimited(void) +PyMODEXPORT_FUNC +PyModExport_xxlimited(void) { - return PyModuleDef_Init(&xxmodule); + return xx_slots; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index b0a682ac4e6bb69..9ef0eac9a924e6c 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -305,7 +305,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, #ifdef Py_GIL_DISABLED - // These definitions are in the limited API, but not until 3.13. + // In a free-threaded build, we don't use Limited API. {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} diff --git a/Modules/xxlimited_3_13.c b/Modules/xxlimited_3_13.c new file mode 100644 index 000000000000000..4f100f9150fc2a3 --- /dev/null +++ b/Modules/xxlimited_3_13.c @@ -0,0 +1,499 @@ +/* Use this file as a template to start implementing a module that + also declares object types. All occurrences of 'Xxo' should be changed + to something reasonable for your objects. After that, all other + occurrences of 'xx' should be changed to something reasonable for your + module. If your module is named foo your source file should be named + foo.c or foomodule.c. + + You will probably want to delete all references to 'x_attr' and add + your own types of attributes instead. Maybe you want to name your + local variables other than 'self'. If your object type is needed in + other files, you'll have to create a file "foobarobject.h"; see + floatobject.h for an example. + + This module roughly corresponds to:: + + class Xxo: + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ + + def __init__(self): + # In the C class, "_x_attr" is not accessible from Python code + self._x_attr = {} + self._x_exports = 0 + + def __getattr__(self, name): + return self._x_attr[name] + + def __setattr__(self, name, value): + self._x_attr[name] = value + + def __delattr__(self, name): + del self._x_attr[name] + + @property + def x_exports(self): + """Return the number of times an internal buffer is exported.""" + # Each Xxo instance has a 10-byte buffer that can be + # accessed via the buffer interface (e.g. `memoryview`). + return self._x_exports + + def demo(o, /): + if isinstance(o, str): + return o + elif isinstance(o, Xxo): + return o + else: + raise Error('argument must be str or Xxo') + + class Error(Exception): + """Exception raised by the xxlimited module""" + + def foo(i: int, j: int, /): + """Return the sum of i and j.""" + # Unlike this pseudocode, the C function will *only* work with + # integers and perform C long int arithmetic + return i + j + + def new(): + return Xxo() + + def Str(str): + # A trivial subclass of a built-in type + pass + */ + +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "Python.h" +#include <string.h> + +#define BUFSIZE 10 + +// Module state +typedef struct { + PyObject *Xxo_Type; // Xxo class + PyObject *Error_Type; // Error class +} xx_state; + + +/* Xxo objects */ + +// Instance state +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ + Py_ssize_t x_exports; /* how many buffer are exported */ +} XxoObject; + +#define XxoObject_CAST(op) ((XxoObject *)(op)) +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) +// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) + +static XxoObject * +newXxoObject(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return NULL; + } + XxoObject *self; + self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + memset(self->x_buffer, 0, BUFSIZE); + self->x_exports = 0; + return self; +} + +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ + +// traverse: Visit all references from an object, including its type +static int +Xxo_traverse(PyObject *op, visitproc visit, void *arg) +{ + // Visit the type + Py_VISIT(Py_TYPE(op)); + + // Visit the attribute dict + XxoObject *self = XxoObject_CAST(op); + Py_VISIT(self->x_attr); + return 0; +} + +// clear: drop references in order to break all reference cycles +static int +Xxo_clear(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); + return 0; +} + +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. +static void +Xxo_finalize(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); +} + +// dealloc: drop all remaining references and free memory +static void +Xxo_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); + freefunc free = PyType_GetSlot(tp, Py_tp_free); + free(self); + Py_DECREF(tp); +} + + +/* Xxo attribute handling */ + +// Get an attribute. +static PyObject * +Xxo_getattro(PyObject *op, PyObject *name) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + if (v != NULL) { + return Py_NewRef(v); + } + else if (PyErr_Occurred()) { + return NULL; + } + } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) + return PyObject_GenericGetAttr(op, name); +} + +// Set or delete an attribute. +static int +Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr == NULL) { + // prepare the attribute dict + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) { + return -1; + } + } + if (v == NULL) { + // delete an attribute + int rv = PyDict_DelItem(self->x_attr, name); + if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, + "delete non-existing Xxo attribute"); + return -1; + } + return rv; + } + else { + // set an attribute + return PyDict_SetItem(self->x_attr, name, v); + } +} + +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ + +static PyObject * +Xxo_demo(PyObject *op, PyTypeObject *defining_class, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (kwnames != NULL && PyObject_Length(kwnames)) { + PyErr_SetString(PyExc_TypeError, "demo() takes no keyword arguments"); + return NULL; + } + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "demo() takes exactly 1 argument"); + return NULL; + } + + PyObject *o = args[0]; + + /* Test if the argument is "str" */ + if (PyUnicode_Check(o)) { + return Py_NewRef(o); + } + + /* test if the argument is of the Xxo class */ + if (PyObject_TypeCheck(o, defining_class)) { + return Py_NewRef(o); + } + + return Py_NewRef(Py_None); +} + +static PyMethodDef Xxo_methods[] = { + {"demo", _PyCFunction_CAST(Xxo_demo), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("demo(o) -> o")}, + {NULL, NULL} /* sentinel */ +}; + +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ + +static int +Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + XxoObject *self = XxoObject_CAST(op); + int res = PyBuffer_FillInfo(view, op, + (void *)self->x_buffer, BUFSIZE, + 0, flags); + if (res == 0) { + self->x_exports++; + } + return res; +} + +static void +Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +{ + XxoObject *self = XxoObject_CAST(op); + self->x_exports--; +} + +static PyObject * +Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +{ + XxoObject *self = XxoObject_CAST(op); + return PyLong_FromSsize_t(self->x_exports); +} + +/* Xxo type definition */ + +PyDoc_STRVAR(Xxo_doc, + "A class that explicitly stores attributes in an internal dict"); + +static PyGetSetDef Xxo_getsetlist[] = { + {"x_exports", Xxo_get_x_exports, NULL, NULL}, + {NULL}, +}; + + +static PyType_Slot Xxo_Type_slots[] = { + {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_traverse, Xxo_traverse}, + {Py_tp_clear, Xxo_clear}, + {Py_tp_finalize, Xxo_finalize}, + {Py_tp_dealloc, Xxo_dealloc}, + {Py_tp_getattro, Xxo_getattro}, + {Py_tp_setattro, Xxo_setattro}, + {Py_tp_methods, Xxo_methods}, + {Py_bf_getbuffer, Xxo_getbuffer}, + {Py_bf_releasebuffer, Xxo_releasebuffer}, + {Py_tp_getset, Xxo_getsetlist}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Xxo_Type_spec = { + .name = "xxlimited_3_13.Xxo", + .basicsize = sizeof(XxoObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = Xxo_Type_slots, +}; + + +/* Str type definition*/ + +static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Str_Type_spec = { + .name = "xxlimited_3_13.Str", + .basicsize = 0, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = Str_Type_slots, +}; + + +/* Function of two integers returning integer (with C "long int" arithmetic) */ + +PyDoc_STRVAR(xx_foo_doc, +"foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +xx_foo(PyObject *module, PyObject *args) +{ + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + return NULL; + res = i+j; /* XXX Do something here */ + return PyLong_FromLong(res); +} + + +/* Function of no arguments returning new Xxo object */ + +static PyObject * +xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) +{ + XxoObject *rv; + + rv = newXxoObject(module); + if (rv == NULL) + return NULL; + return (PyObject *)rv; +} + + + +/* List of functions defined in the module */ + +static PyMethodDef xx_methods[] = { + {"foo", xx_foo, METH_VARARGS, + xx_foo_doc}, + {"new", xx_new, METH_NOARGS, + PyDoc_STR("new() -> new Xx object")}, + {NULL, NULL} /* sentinel */ +}; + + +/* The module itself */ + +PyDoc_STRVAR(module_doc, +"This is a template module just for instruction."); + +static int +xx_modexec(PyObject *m) +{ + xx_state *state = PyModule_GetState(m); + + state->Error_Type = PyErr_NewException("xxlimited_3_13.Error", NULL, NULL); + if (state->Error_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Error_Type) < 0) { + return -1; + } + + state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + if (state->Xxo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + return -1; + } + + // Add the Str type. It is not needed from C code, so it is only + // added to the module dict. + // It does not inherit from "object" (PyObject_Type), but from "str" + // (PyUnincode_Type). + PyObject *Str_Type = PyType_FromModuleAndSpec( + m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); + if (Str_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + return -1; + } + Py_DECREF(Str_Type); + + return 0; +} + +static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + + {0, NULL} +}; + +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + +static int +xx_traverse(PyObject *module, visitproc visit, void *arg) +{ + xx_state *state = PyModule_GetState(module); + Py_VISIT(state->Xxo_Type); + Py_VISIT(state->Error_Type); + return 0; +} + +static int +xx_clear(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + Py_CLEAR(state->Xxo_Type); + Py_CLEAR(state->Error_Type); + return 0; +} + +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + +static struct PyModuleDef xxmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "xxlimited_3_13", + .m_doc = module_doc, + .m_size = sizeof(xx_state), + .m_methods = xx_methods, + .m_slots = xx_slots, + .m_traverse = xx_traverse, + .m_clear = xx_clear, + .m_free = xx_free, +}; + + +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ + +PyMODINIT_FUNC +PyInit_xxlimited_3_13(void) +{ + return PyModuleDef_Init(&xxmodule); +} diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 9c5820fbe97a6b0..0a6732835eb51f5 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -858,7 +858,7 @@ save_unconsumed_input(compobject *self, Py_buffer *data, int err) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary zlib.Decompress.decompress cls: defining_class @@ -872,15 +872,15 @@ zlib.Decompress.decompress Return a bytes object containing the decompressed version of the data. -After calling this function, some of the input data may still be stored in -internal buffers for later processing. +After calling this function, some of the input data may still be +stored in internal buffers for later processing. Call the flush() method to clear these buffers. [clinic start generated code]*/ static PyObject * zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=b024a93c2c922d57 input=77de124bd2a2ecc0]*/ +/*[clinic end generated code: output=b024a93c2c922d57 input=9035027c9e4be7fd]*/ { int err = Z_OK; Py_ssize_t ibuflen; @@ -1675,7 +1675,6 @@ decompress(ZlibDecompressor *self, uint8_t *data, } /*[clinic input] -@permit_long_docstring_body zlib._ZlibDecompressor.decompress data: Py_buffer @@ -1683,25 +1682,26 @@ zlib._ZlibDecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * zlib__ZlibDecompressor_decompress_impl(ZlibDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ac00dcf73e843e99 input=c9278e791be1152b]*/ +/*[clinic end generated code: output=ac00dcf73e843e99 input=d7862eade3f29d56]*/ { PyObject *result = NULL; diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index c583193b5a252ca..ca7956579e80bb6 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1620,6 +1620,7 @@ bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) /*[clinic input] +@permit_long_summary @critical_section bytearray.translate @@ -1630,14 +1631,15 @@ bytearray.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=b6a8f01c2a74e446 input=cd6fa93ca04e05bc]*/ +/*[clinic end generated code: output=b6a8f01c2a74e446 input=e30d2ae004365ed9]*/ { char *input, *output; const char *table_chars; @@ -1727,7 +1729,6 @@ bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, /*[clinic input] @permit_long_summary -@permit_long_docstring_body @staticmethod bytearray.maketrans @@ -1737,15 +1738,15 @@ bytearray.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=1df267d99f56b15e input=1146b43a592eca13]*/ +/*[clinic end generated code: output=1df267d99f56b15e input=c2f5f6e7e6b0221d]*/ { return _Py_bytes_maketrans(frm, to); } @@ -1785,8 +1786,8 @@ bytearray.split sep: object = None The delimiter according which to split the bytearray. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1797,7 +1798,7 @@ Return a list of the sections in the bytearray, using sep as the delimiter. static PyObject * bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=833e2cf385d9a04d input=dd9f6e2910cc3a34]*/ +/*[clinic end generated code: output=833e2cf385d9a04d input=45605178023b52ac]*/ { PyObject *list = NULL; @@ -1829,7 +1830,6 @@ bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.partition @@ -1838,17 +1838,18 @@ bytearray.partition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray. If the separator is -found, returns a 3-tuple containing the part before the separator, the -separator itself, and the part after it as new bytearray objects. +This will search for the separator sep in the bytearray. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it as new +bytearray objects. -If the separator is not found, returns a 3-tuple containing the copy of the -original bytearray object and two empty bytearray objects. +If the separator is not found, returns a 3-tuple containing the copy +of the original bytearray object and two empty bytearray objects. [clinic start generated code]*/ static PyObject * bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=b5fa1e03f10cfccb input=b87276af883f39d9]*/ +/*[clinic end generated code: output=b5fa1e03f10cfccb input=d76673ed03acf5dd]*/ { PyObject *bytesep, *result; @@ -1868,7 +1869,6 @@ bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.rpartition @@ -1877,18 +1877,19 @@ bytearray.rpartition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray, starting at the end. -If the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it as new bytearray -objects. +This will search for the separator sep in the bytearray, starting at +the end. If the separator is found, returns a 3-tuple containing +the part before the separator, the separator itself, and the part +after it as new bytearray objects. -If the separator is not found, returns a 3-tuple containing two empty bytearray -objects and the copy of the original bytearray object. +If the separator is not found, returns a 3-tuple containing two +empty bytearray objects and the copy of the original bytearray +object. [clinic start generated code]*/ static PyObject * bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=0186ce7b1ef61289 input=5bdcfc4c333bcfab]*/ +/*[clinic end generated code: output=0186ce7b1ef61289 input=b9216a2074174a36]*/ { PyObject *bytesep, *result; @@ -1909,19 +1910,19 @@ bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section bytearray.rsplit = bytearray.split Return a list of the sections in the bytearray, using sep as the delimiter. -Splitting is done starting at the end of the bytearray and working to the front. +Splitting is done starting at the end of the bytearray and working +to the front. [clinic start generated code]*/ static PyObject * bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=a55e0b5a03cb6190 input=60e9abf305128ff4]*/ +/*[clinic end generated code: output=a55e0b5a03cb6190 input=e201671c9a0c19ee]*/ { PyObject *list = NULL; @@ -2392,7 +2393,6 @@ bytearray_strip_impl_helper(PyByteArrayObject* self, PyObject* bytes, int stript } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.strip @@ -2401,12 +2401,13 @@ bytearray.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) -/*[clinic end generated code: output=760412661a34ad5a input=6acaf88b2ec9daa7]*/ +/*[clinic end generated code: output=760412661a34ad5a input=f4ec5fa609df7d14]*/ { return bytearray_strip_impl_helper(self, bytes, BOTHSTRIP); } @@ -2506,11 +2507,11 @@ bytearray.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytearray. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytearray using the codec registered for encoding. [clinic start generated code]*/ @@ -2518,7 +2519,7 @@ Decode the bytearray using the codec registered for encoding. static PyObject * bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=f57d43f4a00b42c5 input=86c303ee376b8453]*/ +/*[clinic end generated code: output=f57d43f4a00b42c5 input=e51ce9b82b51e2ca]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -2550,14 +2551,15 @@ bytearray.join Concatenate any number of bytes/bytearray objects. -The bytearray whose method is called is inserted in between each pair. +The bytearray whose method is called is inserted in between each +pair. The result is returned as a new bytearray object. [clinic start generated code]*/ static PyObject * bytearray_join_impl(PyByteArrayObject *self, PyObject *iterable_of_bytes) -/*[clinic end generated code: output=0ced382b5846a7ee input=49627e07ca31ca26]*/ +/*[clinic end generated code: output=0ced382b5846a7ee input=0a31db349efcd7fa]*/ { PyObject *ret; self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator @@ -2588,7 +2590,6 @@ bytearray_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section bytearray.splitlines @@ -2596,13 +2597,13 @@ bytearray.splitlines Return a list of the lines in the bytearray, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=21bc3f02bf1be832]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=cc2bb740eed19f27]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2620,12 +2621,13 @@ bytearray.fromhex Create a bytearray object from a string of hexadecimal numbers. Spaces between two numbers are accepted. -Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') +Example: + bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') [clinic start generated code]*/ static PyObject * bytearray_fromhex_impl(PyTypeObject *type, PyObject *string) -/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=7e314e5b2d7ab484]*/ +/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=2243a8b0b9e66cd5]*/ { PyObject *result = _PyBytes_FromHex(string, type == &PyByteArray_Type); if (type != &PyByteArray_Type && result != NULL) { @@ -2641,8 +2643,8 @@ bytearray.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytearray object. @@ -2661,7 +2663,7 @@ Create a string of hexadecimal numbers from a bytearray object. static PyObject * bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=c9563921aff1262b input=d2b23ef057cfcad5]*/ +/*[clinic end generated code: output=c9563921aff1262b input=9ed746203691e894]*/ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 56a461d0dd08a78..414afeb7bb003c7 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -277,8 +277,8 @@ _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) PyDoc_STRVAR_shared(_Py_title__doc__, "B.title() -> copy of B\n\ \n\ -Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ -characters, all remaining cased characters have lowercase."); +Return a titlecased version of B, i.e. ASCII words start with\n\ +uppercase characters, all remaining cased characters have lowercase."); void _Py_bytes_title(char *result, const char *s, Py_ssize_t len) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8a9d1b133affb3e..1135770549c0174 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -11,6 +11,7 @@ #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_list.h" // _PyList_GetItemRef #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE #include "pycore_strhex.h" // _Py_strhex_with_sep() @@ -1848,12 +1849,13 @@ bytes___bytes___impl(PyBytesObject *self) #define BOTHSTRIP 2 /*[clinic input] +@permit_long_summary bytes.split sep: object = None The delimiter according which to split the bytes. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1863,7 +1865,7 @@ Return a list of the sections in the bytes, using sep as the delimiter. static PyObject * bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=52126b5844c1d8ef input=8b809b39074abbfa]*/ +/*[clinic end generated code: output=52126b5844c1d8ef input=330ff95d92544b05]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1885,7 +1887,6 @@ bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] -@permit_long_docstring_body bytes.partition sep: Py_buffer @@ -1893,17 +1894,17 @@ bytes.partition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator sep in the bytes. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original bytes -object and two empty bytes objects. +If the separator is not found, returns a 3-tuple containing the +original bytes object and two empty bytes objects. [clinic start generated code]*/ static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=31c55a0cebaf7722]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=2e6e551ea4f8b95a]*/ { return stringlib_partition( (PyObject*) self, @@ -1913,7 +1914,6 @@ bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] -@permit_long_docstring_body bytes.rpartition sep: Py_buffer @@ -1921,17 +1921,18 @@ bytes.rpartition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator sep in the bytes, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty bytes -objects and the original bytes object. +If the separator is not found, returns a 3-tuple containing two +empty bytes objects and the original bytes object. [clinic start generated code]*/ static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=9ea5a3ab0b02bf52]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=f7d24f722a5470a4]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1941,17 +1942,18 @@ bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary bytes.rsplit = bytes.split Return a list of the sections in the bytes, using sep as the delimiter. -Splitting is done starting at the end of the bytes and working to the front. +Splitting is done starting at the end of the bytes and working to +the front. [clinic start generated code]*/ static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=ba698d9ea01e1c8f input=55b6eaea1f3d7046]*/ +/*[clinic end generated code: output=ba698d9ea01e1c8f input=ba9bee56285f43e4]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -2171,7 +2173,6 @@ do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes) } /*[clinic input] -@permit_long_docstring_body bytes.strip bytes: object = None @@ -2179,12 +2180,13 @@ bytes.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=71904cd278c0ee03]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=9ffea5f752032bd0]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2244,6 +2246,7 @@ bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, /*[clinic input] +@permit_long_summary bytes.translate table: object @@ -2253,14 +2256,15 @@ bytes.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=43be3437f1956211 input=0ecdf159f654233c]*/ +/*[clinic end generated code: output=43be3437f1956211 input=bddcdef0a87895d2]*/ { const char *input; char *output; @@ -2378,7 +2382,6 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, /*[clinic input] @permit_long_summary -@permit_long_docstring_body @staticmethod bytes.maketrans @@ -2388,15 +2391,15 @@ bytes.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=a36f6399d4b77f6f input=a06b75f44d933fb3]*/ +/*[clinic end generated code: output=a36f6399d4b77f6f input=3a577e5badfea8f7]*/ { return _Py_bytes_maketrans(frm, to); } @@ -2431,6 +2434,7 @@ bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, /** End DALKE **/ /*[clinic input] +@permit_long_summary bytes.removeprefix as bytes_removeprefix prefix: Py_buffer @@ -2438,13 +2442,14 @@ bytes.removeprefix as bytes_removeprefix Return a bytes object with the given prefix string removed if present. -If the bytes starts with the prefix string, return bytes[len(prefix):]. -Otherwise, return a copy of the original bytes. +If the bytes starts with the prefix string, return +bytes[len(prefix):]. Otherwise, return a copy of the original +bytes. [clinic start generated code]*/ static PyObject * bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) -/*[clinic end generated code: output=f006865331a06ab6 input=0c93bac817a8502c]*/ +/*[clinic end generated code: output=f006865331a06ab6 input=3a2672bcee61d7a7]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2467,6 +2472,7 @@ bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) } /*[clinic input] +@permit_long_summary bytes.removesuffix as bytes_removesuffix suffix: Py_buffer @@ -2474,14 +2480,14 @@ bytes.removesuffix as bytes_removesuffix Return a bytes object with the given suffix string removed if present. -If the bytes ends with the suffix string and that suffix is not empty, -return bytes[:-len(prefix)]. Otherwise, return a copy of the original -bytes. +If the bytes ends with the suffix string and that suffix is not +empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the +original bytes. [clinic start generated code]*/ static PyObject * bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) -/*[clinic end generated code: output=d887d308e3242eeb input=9f4e1da8c637bbf1]*/ +/*[clinic end generated code: output=d887d308e3242eeb input=04df5f18a36f69d7]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2561,11 +2567,11 @@ bytes.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytes. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytes using the codec registered for encoding. [clinic start generated code]*/ @@ -2573,27 +2579,27 @@ Decode the bytes using the codec registered for encoding. static PyObject * bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=5649a53dde27b314 input=958174769d2a40ca]*/ +/*[clinic end generated code: output=5649a53dde27b314 input=94e9b8524f1d7f37]*/ { return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary bytes.splitlines keepends: bool = False Return a list of the lines in the bytes, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=d17968d2a355fe55]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=8734672f34430514]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), @@ -2744,8 +2750,8 @@ bytes.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytes object. @@ -2763,7 +2769,7 @@ Create a string of hexadecimal numbers from a bytes object. static PyObject * bytes_hex_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=588821f02cb9d8f5 input=bd8eceb755d8230f]*/ +/*[clinic end generated code: output=588821f02cb9d8f5 input=b8d40cf203d172dc]*/ { const char *argbuf = PyBytes_AS_STRING(self); Py_ssize_t arglen = PyBytes_GET_SIZE(self); @@ -2991,8 +2997,10 @@ _PyBytes_FromList(PyObject *x) size = _PyBytesWriter_GetAllocated(writer); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { - PyObject *item = PyList_GET_ITEM(x, i); - Py_INCREF(item); + PyObject *item = _PyList_GetItemRef((PyListObject *)x, i); + if (item == NULL) { + goto error; + } Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 64603adcc1124b1..41ce82c05c57d97 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -679,8 +679,9 @@ PyDoc_STRVAR(bytearray_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTEARRAY_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytearray_translate), METH_FASTCALL|METH_KEYWORDS, bytearray_translate__doc__}, @@ -750,8 +751,8 @@ PyDoc_STRVAR(bytearray_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -901,8 +902,8 @@ PyDoc_STRVAR(bytearray_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -991,12 +992,13 @@ PyDoc_STRVAR(bytearray_partition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray. If the separator is\n" -"found, returns a 3-tuple containing the part before the separator, the\n" -"separator itself, and the part after it as new bytearray objects.\n" +"This will search for the separator sep in the bytearray. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it as new\n" +"bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the copy of the\n" -"original bytearray object and two empty bytearray objects."); +"If the separator is not found, returns a 3-tuple containing the copy\n" +"of the original bytearray object and two empty bytearray objects."); #define BYTEARRAY_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytearray_partition, METH_O, bytearray_partition__doc__}, @@ -1022,13 +1024,14 @@ PyDoc_STRVAR(bytearray_rpartition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray, starting at the end.\n" -"If the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it as new bytearray\n" -"objects.\n" +"This will search for the separator sep in the bytearray, starting at\n" +"the end. If the separator is found, returns a 3-tuple containing\n" +"the part before the separator, the separator itself, and the part\n" +"after it as new bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytearray\n" -"objects and the copy of the original bytearray object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytearray objects and the copy of the original bytearray\n" +"object."); #define BYTEARRAY_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytearray_rpartition, METH_O, bytearray_rpartition__doc__}, @@ -1056,13 +1059,14 @@ PyDoc_STRVAR(bytearray_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytearray and working to the front."); +"Splitting is done starting at the end of the bytearray and working\n" +"to the front."); #define BYTEARRAY_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytearray_rsplit), METH_FASTCALL|METH_KEYWORDS, bytearray_rsplit__doc__}, @@ -1364,7 +1368,8 @@ PyDoc_STRVAR(bytearray_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTEARRAY_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytearray_strip), METH_FASTCALL, bytearray_strip__doc__}, @@ -1475,11 +1480,11 @@ PyDoc_STRVAR(bytearray_decode__doc__, " encoding\n" " The encoding with which to decode the bytearray.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTEARRAY_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytearray_decode), METH_FASTCALL|METH_KEYWORDS, bytearray_decode__doc__}, @@ -1578,7 +1583,8 @@ PyDoc_STRVAR(bytearray_join__doc__, "\n" "Concatenate any number of bytes/bytearray objects.\n" "\n" -"The bytearray whose method is called is inserted in between each pair.\n" +"The bytearray whose method is called is inserted in between each\n" +"pair.\n" "\n" "The result is returned as a new bytearray object."); @@ -1606,8 +1612,8 @@ PyDoc_STRVAR(bytearray_splitlines__doc__, "\n" "Return a list of the lines in the bytearray, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTEARRAY_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytearray_splitlines), METH_FASTCALL|METH_KEYWORDS, bytearray_splitlines__doc__}, @@ -1678,7 +1684,8 @@ PyDoc_STRVAR(bytearray_fromhex__doc__, "Create a bytearray object from a string of hexadecimal numbers.\n" "\n" "Spaces between two numbers are accepted.\n" -"Example: bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); +"Example:\n" +" bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); #define BYTEARRAY_FROMHEX_METHODDEF \ {"fromhex", (PyCFunction)bytearray_fromhex, METH_O|METH_CLASS, bytearray_fromhex__doc__}, @@ -1705,8 +1712,8 @@ PyDoc_STRVAR(bytearray_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = bytearray([0xb9, 0x01, 0xef])\n" @@ -1875,4 +1882,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=2cacb323147202b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6dc315d35de3e670 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 4ff696be91b12de..ee2b737f9e63f97 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -35,8 +35,8 @@ PyDoc_STRVAR(bytes_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -122,12 +122,12 @@ PyDoc_STRVAR(bytes_partition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator sep in the bytes. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original bytes\n" -"object and two empty bytes objects."); +"If the separator is not found, returns a 3-tuple containing the\n" +"original bytes object and two empty bytes objects."); #define BYTES_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytes_partition, METH_O, bytes_partition__doc__}, @@ -161,12 +161,13 @@ PyDoc_STRVAR(bytes_rpartition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator sep in the bytes, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytes\n" -"objects and the original bytes object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytes objects and the original bytes object."); #define BYTES_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytes_rpartition, METH_O, bytes_rpartition__doc__}, @@ -202,13 +203,14 @@ PyDoc_STRVAR(bytes_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytes and working to the front."); +"Splitting is done starting at the end of the bytes and working to\n" +"the front."); #define BYTES_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytes_rsplit), METH_FASTCALL|METH_KEYWORDS, bytes_rsplit__doc__}, @@ -523,7 +525,8 @@ PyDoc_STRVAR(bytes_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTES_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytes_strip), METH_FASTCALL, bytes_strip__doc__}, @@ -677,8 +680,9 @@ PyDoc_STRVAR(bytes_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTES_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytes_translate), METH_FASTCALL|METH_KEYWORDS, bytes_translate__doc__}, @@ -746,8 +750,8 @@ PyDoc_STRVAR(bytes_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -893,8 +897,9 @@ PyDoc_STRVAR(bytes_removeprefix__doc__, "\n" "Return a bytes object with the given prefix string removed if present.\n" "\n" -"If the bytes starts with the prefix string, return bytes[len(prefix):].\n" -"Otherwise, return a copy of the original bytes."); +"If the bytes starts with the prefix string, return\n" +"bytes[len(prefix):]. Otherwise, return a copy of the original\n" +"bytes."); #define BYTES_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)bytes_removeprefix, METH_O, bytes_removeprefix__doc__}, @@ -928,9 +933,9 @@ PyDoc_STRVAR(bytes_removesuffix__doc__, "\n" "Return a bytes object with the given suffix string removed if present.\n" "\n" -"If the bytes ends with the suffix string and that suffix is not empty,\n" -"return bytes[:-len(prefix)]. Otherwise, return a copy of the original\n" -"bytes."); +"If the bytes ends with the suffix string and that suffix is not\n" +"empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the\n" +"original bytes."); #define BYTES_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)bytes_removesuffix, METH_O, bytes_removesuffix__doc__}, @@ -1069,11 +1074,11 @@ PyDoc_STRVAR(bytes_decode__doc__, " encoding\n" " The encoding with which to decode the bytes.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTES_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytes_decode), METH_FASTCALL|METH_KEYWORDS, bytes_decode__doc__}, @@ -1170,8 +1175,8 @@ PyDoc_STRVAR(bytes_splitlines__doc__, "\n" "Return a list of the lines in the bytes, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTES_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytes_splitlines), METH_FASTCALL|METH_KEYWORDS, bytes_splitlines__doc__}, @@ -1267,8 +1272,8 @@ PyDoc_STRVAR(bytes_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = b\'\\xb9\\x01\\xef\'\n" @@ -1450,4 +1455,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b252801ff04a89b3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c20458db7a2123db input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 0cd6e0b56620e85..88333e9d3363eb4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -414,7 +414,8 @@ PyDoc_STRVAR(code__varname_from_oparg__doc__, "\n" "(internal-only) Return the local variable name for the given oparg.\n" "\n" -"WARNING: this method is for internal use only and may change or go away."); +"WARNING: this method is for internal use only and may change or go\n" +"away."); #define CODE__VARNAME_FROM_OPARG_METHODDEF \ {"_varname_from_oparg", _PyCFunction_CAST(code__varname_from_oparg), METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__}, @@ -470,4 +471,4 @@ code__varname_from_oparg(PyObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=c5c6e40fc357defe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c22e29e430401b4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index c0ae9d3ff9b8d3c..8768555c909257a 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -291,8 +291,8 @@ PyDoc_STRVAR(float___getformat____doc__, "It exists mainly to be used in Python\'s test suite.\n" "\n" "This function returns whichever of \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating-point numbers used by the\n" -"C type named by typestr."); +"little-endian\' best describes the format of floating-point numbers\n" +"used by the C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ {"__getformat__", (PyCFunction)float___getformat__, METH_O|METH_CLASS, float___getformat____doc__}, @@ -353,4 +353,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=f0b2af257213c8b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5d7b0bf9e47ff997 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 26ba5b954336da5..f3821ef5f70c21c 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -200,11 +200,11 @@ PyDoc_STRVAR(list_sort__doc__, "\n" "Sort the list in ascending order and return None.\n" "\n" -"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" -"order of two equal elements is maintained).\n" +"The sort is in-place (i.e. the list itself is modified) and stable\n" +"(i.e. the order of two equal elements is maintained).\n" "\n" -"If a key function is given, apply it once to each list item and sort them,\n" -"ascending or descending, according to their function values.\n" +"If a key function is given, apply it once to each list item and sort\n" +"them, ascending or descending, according to their function values.\n" "\n" "The reverse flag can be set to sort in descending order."); @@ -468,4 +468,4 @@ list___reversed__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl((PyListObject *)self); } -/*[clinic end generated code: output=ae13fc2b56dc27c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06c21b0bffbe8d84 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index c88772030ec283d..52ecaffa1f4cf35 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -262,19 +262,20 @@ PyDoc_STRVAR(int_to_bytes__doc__, "Return an array of bytes representing an integer.\n" "\n" " length\n" -" Length of bytes object to use. An OverflowError is raised if the\n" -" integer is not representable with the given number of bytes. Default\n" -" is length 1.\n" +" Length of bytes object to use. An OverflowError is raised if\n" +" the integer is not representable with the given number of bytes.\n" +" Default is length 1.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Determines whether two\'s complement is used to represent the integer.\n" -" If signed is False and a negative integer is given, an OverflowError\n" -" is raised."); +" Determines whether two\'s complement is used to represent the\n" +" integer. If signed is False and a negative integer is given,\n" +" an OverflowError is raised."); #define INT_TO_BYTES_METHODDEF \ {"to_bytes", _PyCFunction_CAST(int_to_bytes), METH_FASTCALL|METH_KEYWORDS, int_to_bytes__doc__}, @@ -383,17 +384,19 @@ PyDoc_STRVAR(int_from_bytes__doc__, "\n" " bytes\n" " Holds the array of bytes to convert. The argument must either\n" -" support the buffer protocol or be an iterable object producing bytes.\n" -" Bytes and bytearray are examples of built-in objects that support the\n" -" buffer protocol.\n" +" support the buffer protocol or be an iterable object producing\n" +" bytes. Bytes and bytearray are examples of built-in objects that\n" +" support the buffer protocol.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Indicates whether two\'s complement is used to represent the integer."); +" Indicates whether two\'s complement is used to represent the\n" +" integer."); #define INT_FROM_BYTES_METHODDEF \ {"from_bytes", _PyCFunction_CAST(int_from_bytes), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, int_from_bytes__doc__}, @@ -490,4 +493,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=e68f4e23ead3f649 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d95766fb7ff46963 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index d97c626532c803e..a0cf3243edc08a0 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -259,11 +259,12 @@ PyDoc_STRVAR(memoryview_tobytes__doc__, "\n" "Return the data in the buffer as a byte string.\n" "\n" -"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of the\n" -"original array is converted to C or Fortran order. For contiguous views,\n" -"\'A\' returns an exact copy of the physical memory. In particular, in-memory\n" -"Fortran order is preserved. For non-contiguous views, the data is converted\n" -"to C first. order=None is the same as order=\'C\'."); +"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of\n" +"the original array is converted to C or Fortran order. For\n" +"contiguous views, \'A\' returns an exact copy of the physical memory.\n" +"In particular, in-memory Fortran order is preserved. For\n" +"non-contiguous views, the data is converted to C first. order=None\n" +"is the same as order=\'C\'."); #define MEMORYVIEW_TOBYTES_METHODDEF \ {"tobytes", _PyCFunction_CAST(memoryview_tobytes), METH_FASTCALL|METH_KEYWORDS, memoryview_tobytes__doc__}, @@ -348,8 +349,8 @@ PyDoc_STRVAR(memoryview_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = memoryview(b\'\\xb9\\x01\\xef\')\n" @@ -505,4 +506,4 @@ memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=348b6ddb98a1f412 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3abf9c80cd49229a input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 894e9be91bbdce9..92129e6444810f4 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -268,7 +268,8 @@ PyDoc_STRVAR(OrderedDict_popitem__doc__, "\n" "Remove and return a (key, value) pair from the dictionary.\n" "\n" -"Pairs are returned in LIFO order if last is true or FIFO order if false."); +"Pairs are returned in LIFO order if last is true or FIFO order if\n" +"false."); #define ORDEREDDICT_POPITEM_METHODDEF \ {"popitem", _PyCFunction_CAST(OrderedDict_popitem), METH_FASTCALL|METH_KEYWORDS, OrderedDict_popitem__doc__}, @@ -451,4 +452,4 @@ OrderedDict_move_to_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=7bc997ca7900f06f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89f7e92de998f9a4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/sentinelobject.c.h b/Objects/clinic/sentinelobject.c.h index 51fd35a5979e318..f8503194ae5c740 100644 --- a/Objects/clinic/sentinelobject.c.h +++ b/Objects/clinic/sentinelobject.c.h @@ -2,33 +2,71 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * -sentinel_new_impl(PyTypeObject *type, PyObject *name); +sentinel_new_impl(PyTypeObject *type, PyObject *name, PyObject *repr); static PyObject * sentinel_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - PyTypeObject *base_tp = &PySentinel_Type; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(repr), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "repr", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sentinel", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *repr = Py_None; - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("sentinel", kwargs)) { + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { goto exit; } - if (!_PyArg_CheckPositional("sentinel", PyTuple_GET_SIZE(args), 1, 1)) { + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("sentinel", "argument 1", "str", fastargs[0]); goto exit; } - if (!PyUnicode_Check(PyTuple_GET_ITEM(args, 0))) { - _PyArg_BadArgument("sentinel", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); - goto exit; + name = fastargs[0]; + if (!noptargs) { + goto skip_optional_kwonly; } - name = PyTuple_GET_ITEM(args, 0); - return_value = sentinel_new_impl(type, name); + repr = fastargs[1]; +skip_optional_kwonly: + return_value = sentinel_new_impl(type, name, repr); exit: return return_value; } -/*[clinic end generated code: output=7f28fc0bf0259cba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=958842ece254c82f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 4b53e24fb7d649f..d0753b38843fccf 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -33,8 +33,8 @@ PyDoc_STRVAR(unicode_title__doc__, "\n" "Return a version of the string where each word is titlecased.\n" "\n" -"More specifically, words start with uppercased characters and all remaining\n" -"cased characters have lower case."); +"More specifically, words start with uppercased characters and all\n" +"remaining cased characters have lower case."); #define UNICODE_TITLE_METHODDEF \ {"title", (PyCFunction)unicode_title, METH_NOARGS, unicode_title__doc__}, @@ -54,8 +54,8 @@ PyDoc_STRVAR(unicode_capitalize__doc__, "\n" "Return a capitalized version of the string.\n" "\n" -"More specifically, make the first character have upper case and the rest lower\n" -"case."); +"More specifically, make the first character have upper case and the\n" +"rest lower case."); #define UNICODE_CAPITALIZE_METHODDEF \ {"capitalize", (PyCFunction)unicode_capitalize, METH_NOARGS, unicode_capitalize__doc__}, @@ -93,7 +93,8 @@ PyDoc_STRVAR(unicode_center__doc__, "\n" "Return a centered string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_CENTER_METHODDEF \ {"center", _PyCFunction_CAST(unicode_center), METH_FASTCALL, unicode_center__doc__}, @@ -142,7 +143,8 @@ PyDoc_STRVAR(unicode_count__doc__, "\n" "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation."); +"Optional arguments start and end are interpreted as in slice\n" +"notation."); #define UNICODE_COUNT_METHODDEF \ {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, @@ -202,8 +204,8 @@ PyDoc_STRVAR(unicode_encode__doc__, " errors\n" " The error handling scheme to use for encoding errors.\n" " The default is \'strict\' meaning that encoding errors raise a\n" -" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\' and\n" -" \'xmlcharrefreplace\' as well as any other name registered with\n" +" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +" and \'xmlcharrefreplace\' as well as any other name registered with\n" " codecs.register_error that can handle UnicodeEncodeErrors."); #define UNICODE_ENCODE_METHODDEF \ @@ -368,8 +370,8 @@ PyDoc_STRVAR(unicode_find__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_FIND_METHODDEF \ {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, @@ -424,8 +426,8 @@ PyDoc_STRVAR(unicode_index__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_INDEX_METHODDEF \ {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, @@ -501,8 +503,8 @@ PyDoc_STRVAR(unicode_islower__doc__, "\n" "Return True if the string is a lowercase string, False otherwise.\n" "\n" -"A string is lowercase if all cased characters in the string are lowercase and\n" -"there is at least one cased character in the string."); +"A string is lowercase if all cased characters in the string are\n" +"lowercase and there is at least one cased character in the string."); #define UNICODE_ISLOWER_METHODDEF \ {"islower", (PyCFunction)unicode_islower, METH_NOARGS, unicode_islower__doc__}, @@ -522,8 +524,8 @@ PyDoc_STRVAR(unicode_isupper__doc__, "\n" "Return True if the string is an uppercase string, False otherwise.\n" "\n" -"A string is uppercase if all cased characters in the string are uppercase and\n" -"there is at least one cased character in the string."); +"A string is uppercase if all cased characters in the string are\n" +"uppercase and there is at least one cased character in the string."); #define UNICODE_ISUPPER_METHODDEF \ {"isupper", (PyCFunction)unicode_isupper, METH_NOARGS, unicode_isupper__doc__}, @@ -564,8 +566,8 @@ PyDoc_STRVAR(unicode_isspace__doc__, "\n" "Return True if the string is a whitespace string, False otherwise.\n" "\n" -"A string is whitespace if all characters in the string are whitespace and there\n" -"is at least one character in the string."); +"A string is whitespace if all characters in the string are\n" +"whitespace and there is at least one character in the string."); #define UNICODE_ISSPACE_METHODDEF \ {"isspace", (PyCFunction)unicode_isspace, METH_NOARGS, unicode_isspace__doc__}, @@ -585,8 +587,8 @@ PyDoc_STRVAR(unicode_isalpha__doc__, "\n" "Return True if the string is an alphabetic string, False otherwise.\n" "\n" -"A string is alphabetic if all characters in the string are alphabetic and there\n" -"is at least one character in the string."); +"A string is alphabetic if all characters in the string are\n" +"alphabetic and there is at least one character in the string."); #define UNICODE_ISALPHA_METHODDEF \ {"isalpha", (PyCFunction)unicode_isalpha, METH_NOARGS, unicode_isalpha__doc__}, @@ -606,8 +608,8 @@ PyDoc_STRVAR(unicode_isalnum__doc__, "\n" "Return True if the string is an alpha-numeric string, False otherwise.\n" "\n" -"A string is alpha-numeric if all characters in the string are alpha-numeric and\n" -"there is at least one character in the string."); +"A string is alpha-numeric if all characters in the string are\n" +"alpha-numeric and there is at least one character in the string."); #define UNICODE_ISALNUM_METHODDEF \ {"isalnum", (PyCFunction)unicode_isalnum, METH_NOARGS, unicode_isalnum__doc__}, @@ -627,8 +629,8 @@ PyDoc_STRVAR(unicode_isdecimal__doc__, "\n" "Return True if the string is a decimal string, False otherwise.\n" "\n" -"A string is a decimal string if all characters in the string are decimal and\n" -"there is at least one character in the string."); +"A string is a decimal string if all characters in the string are\n" +"decimal and there is at least one character in the string."); #define UNICODE_ISDECIMAL_METHODDEF \ {"isdecimal", (PyCFunction)unicode_isdecimal, METH_NOARGS, unicode_isdecimal__doc__}, @@ -648,8 +650,8 @@ PyDoc_STRVAR(unicode_isdigit__doc__, "\n" "Return True if the string is a digit string, False otherwise.\n" "\n" -"A string is a digit string if all characters in the string are digits and there\n" -"is at least one character in the string."); +"A string is a digit string if all characters in the string are\n" +"digits and there is at least one character in the string."); #define UNICODE_ISDIGIT_METHODDEF \ {"isdigit", (PyCFunction)unicode_isdigit, METH_NOARGS, unicode_isdigit__doc__}, @@ -669,8 +671,8 @@ PyDoc_STRVAR(unicode_isnumeric__doc__, "\n" "Return True if the string is a numeric string, False otherwise.\n" "\n" -"A string is numeric if all characters in the string are numeric and there is at\n" -"least one character in the string."); +"A string is numeric if all characters in the string are numeric and\n" +"there is at least one character in the string."); #define UNICODE_ISNUMERIC_METHODDEF \ {"isnumeric", (PyCFunction)unicode_isnumeric, METH_NOARGS, unicode_isnumeric__doc__}, @@ -690,8 +692,8 @@ PyDoc_STRVAR(unicode_isidentifier__doc__, "\n" "Return True if the string is a valid Python identifier, False otherwise.\n" "\n" -"Call keyword.iskeyword(s) to test whether string s is a reserved identifier,\n" -"such as \"def\" or \"class\"."); +"Call keyword.iskeyword(s) to test whether string s is a reserved\n" +"identifier, such as \"def\" or \"class\"."); #define UNICODE_ISIDENTIFIER_METHODDEF \ {"isidentifier", (PyCFunction)unicode_isidentifier, METH_NOARGS, unicode_isidentifier__doc__}, @@ -731,8 +733,8 @@ PyDoc_STRVAR(unicode_join__doc__, "\n" "Concatenate any number of strings.\n" "\n" -"The string whose method is called is inserted in between each given string.\n" -"The result is returned as a new string.\n" +"The string whose method is called is inserted in between each given\n" +"string. The result is returned as a new string.\n" "\n" "Example: \'.\'.join([\'ab\', \'pq\', \'rs\']) -> \'ab.pq.rs\'"); @@ -745,7 +747,8 @@ PyDoc_STRVAR(unicode_ljust__doc__, "\n" "Return a left-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_LJUST_METHODDEF \ {"ljust", _PyCFunction_CAST(unicode_ljust), METH_FASTCALL, unicode_ljust__doc__}, @@ -1008,8 +1011,9 @@ PyDoc_STRVAR(unicode_removeprefix__doc__, "\n" "Return a str with the given prefix string removed if present.\n" "\n" -"If the string starts with the prefix string, return string[len(prefix):].\n" -"Otherwise, return a copy of the original string."); +"If the string starts with the prefix string, return\n" +"string[len(prefix):]. Otherwise, return a copy of the original\n" +"string."); #define UNICODE_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)unicode_removeprefix, METH_O, unicode_removeprefix__doc__}, @@ -1040,9 +1044,9 @@ PyDoc_STRVAR(unicode_removesuffix__doc__, "\n" "Return a str with the given suffix string removed if present.\n" "\n" -"If the string ends with the suffix string and that suffix is not empty,\n" -"return string[:-len(suffix)]. Otherwise, return a copy of the original\n" -"string."); +"If the string ends with the suffix string and that suffix is not\n" +"empty, return string[:-len(suffix)]. Otherwise, return a copy of\n" +"the original string."); #define UNICODE_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)unicode_removesuffix, METH_O, unicode_removesuffix__doc__}, @@ -1073,8 +1077,8 @@ PyDoc_STRVAR(unicode_rfind__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_RFIND_METHODDEF \ {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, @@ -1129,8 +1133,8 @@ PyDoc_STRVAR(unicode_rindex__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_RINDEX_METHODDEF \ {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, @@ -1185,7 +1189,8 @@ PyDoc_STRVAR(unicode_rjust__doc__, "\n" "Return a right-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_RJUST_METHODDEF \ {"rjust", _PyCFunction_CAST(unicode_rjust), METH_FASTCALL, unicode_rjust__doc__}, @@ -1237,18 +1242,18 @@ PyDoc_STRVAR(unicode_split__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the front of the string and works to the end.\n" "\n" -"Note, str.split() is mainly useful for data that has been intentionally\n" -"delimited. With natural text that includes punctuation, consider using\n" -"the regular expression module."); +"Note, str.split() is mainly useful for data that has been\n" +"intentionally delimited. With natural text that includes\n" +"punctuation, consider using the regular expression module."); #define UNICODE_SPLIT_METHODDEF \ {"split", _PyCFunction_CAST(unicode_split), METH_FASTCALL|METH_KEYWORDS, unicode_split__doc__}, @@ -1331,12 +1336,12 @@ PyDoc_STRVAR(unicode_partition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator in the string. If the separator\n" +"is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original string\n" -"and two empty strings."); +"If the separator is not found, returns a 3-tuple containing\n" +"the original string and two empty strings."); #define UNICODE_PARTITION_METHODDEF \ {"partition", (PyCFunction)unicode_partition, METH_O, unicode_partition__doc__}, @@ -1347,12 +1352,13 @@ PyDoc_STRVAR(unicode_rpartition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator in the string, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty strings\n" -"and the original string."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty strings and the original string."); #define UNICODE_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)unicode_rpartition, METH_O, unicode_rpartition__doc__}, @@ -1366,9 +1372,9 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" @@ -1456,8 +1462,8 @@ PyDoc_STRVAR(unicode_splitlines__doc__, "\n" "Return a list of the lines in the string, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define UNICODE_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(unicode_splitlines), METH_FASTCALL|METH_KEYWORDS, unicode_splitlines__doc__}, @@ -1543,13 +1549,14 @@ PyDoc_STRVAR(unicode_maketrans__doc__, "\n" "Return a translation table usable for str.translate().\n" "\n" -"If there is only one argument, it must be a dictionary mapping Unicode\n" -"ordinals (integers) or characters to Unicode ordinals, strings or None.\n" -"Character keys will be then converted to ordinals.\n" -"If there are two arguments, they must be strings of equal length, and\n" -"in the resulting dictionary, each character in x will be mapped to the\n" -"character at the same position in y. If there is a third argument, it\n" -"must be a string, whose characters will be mapped to None in the result."); +"If there is only one argument, it must be a dictionary mapping\n" +"Unicode ordinals (integers) or characters to Unicode ordinals,\n" +"strings or None. Character keys will be then converted to ordinals.\n" +"If there are two arguments, they must be strings of equal length,\n" +"and in the resulting dictionary, each character in x will be mapped\n" +"to the character at the same position in y. If there is a third\n" +"argument, it must be a string, whose characters will be mapped to\n" +"None in the result."); #define UNICODE_MAKETRANS_METHODDEF \ {"maketrans", _PyCFunction_CAST(unicode_maketrans), METH_FASTCALL|METH_STATIC, unicode_maketrans__doc__}, @@ -1599,12 +1606,13 @@ PyDoc_STRVAR(unicode_translate__doc__, "Replace each character in the string using the given translation table.\n" "\n" " table\n" -" Translation table, which must be a mapping of Unicode ordinals to\n" -" Unicode ordinals, strings, or None.\n" +" Translation table, which must be a mapping of Unicode ordinals\n" +" to Unicode ordinals, strings, or None.\n" "\n" -"The table must implement lookup/indexing via __getitem__, for instance a\n" -"dictionary or list. If this operation raises LookupError, the character is\n" -"left untouched. Characters mapped to None are deleted."); +"The table must implement lookup/indexing via __getitem__, for\n" +"instance a dictionary or list. If this operation raises\n" +"LookupError, the character is left untouched. Characters mapped to\n" +"None are deleted."); #define UNICODE_TRANSLATE_METHODDEF \ {"translate", (PyCFunction)unicode_translate, METH_O, unicode_translate__doc__}, @@ -1908,4 +1916,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=13eaf65699ea9fc9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d243c63e951e31d input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 8be85b1accbdca2..4ede8de6e8adc5f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2840,12 +2840,13 @@ code._varname_from_oparg (internal-only) Return the local variable name for the given oparg. -WARNING: this method is for internal use only and may change or go away. +WARNING: this method is for internal use only and may change or go +away. [clinic start generated code]*/ static PyObject * code__varname_from_oparg_impl(PyCodeObject *self, int oparg) -/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/ +/*[clinic end generated code: output=1fd1130413184206 input=6ba7d6df0d566463]*/ { PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg); if (name == NULL) { diff --git a/Objects/descrobject.c b/Objects/descrobject.c index a5926616eeb3cbf..30444b7d6774247 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1178,7 +1178,7 @@ static PyMethodDef mappingproxy_methods[] = { {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("mappingproxy objects are generic over two types, signifying (respectively) the types of their keys and values")}, {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 42bc63acd9049ce..e279c8765dd464a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -900,7 +900,7 @@ free_values(PyDictValues *values, bool use_qsbr) static inline PyObject * new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, - int free_values_on_failure) + int free_values_on_failure, int frozendict) { assert(keys != NULL); if (mp == NULL) { @@ -915,6 +915,9 @@ new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, mp->ma_values = values; mp->ma_used = used; mp->_ma_watcher_tag = 0; + if (frozendict) { + ((PyFrozenDictObject *)mp)->ma_hash = -1; + } ASSERT_CONSISTENT(mp); _PyObject_GC_TRACK(mp); return (PyObject *)mp; @@ -931,7 +934,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, } assert(mp == NULL || Py_IS_TYPE(mp, &PyDict_Type)); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 0); } /* Consumes a reference to the keys object */ @@ -940,7 +943,7 @@ new_frozendict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp = PyObject_GC_New(PyDictObject, &PyFrozenDict_Type); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 1); } static PyObject * @@ -3080,10 +3083,12 @@ clear_lock_held(PyObject *op) set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { - Py_CLEAR(oldvalues->values[i]); + PyObject *tmp = oldvalues->values[i]; + FT_ATOMIC_STORE_PTR_RELEASE(oldvalues->values[i], NULL); + Py_XDECREF(tmp); } free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(oldkeys, false); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } @@ -3887,6 +3892,7 @@ PyDict_Items(PyObject *dict) } /*[clinic input] +@permit_long_summary @classmethod dict.fromkeys iterable: object @@ -3898,7 +3904,7 @@ Create a new dictionary with keys from iterable and values set to value. static PyObject * dict_fromkeys_impl(PyTypeObject *type, PyObject *iterable, PyObject *value) -/*[clinic end generated code: output=8fb98e4b10384999 input=382ba4855d0f74c3]*/ +/*[clinic end generated code: output=8fb98e4b10384999 input=3903715eb48b287e]*/ { return _PyDict_FromKeys((PyObject *)type, iterable, value); } @@ -5124,7 +5130,8 @@ static PyMethodDef mapp_methods[] = { DICT_CLEAR_METHODDEF DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("dicts are generic over two types, signifying (respectively) the types of their keys and values")}, {NULL, NULL} /* sentinel */ }; @@ -8015,13 +8022,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -8033,7 +8046,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, (1LL << watcher_id)); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -8048,36 +8062,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, ~(1LL << watcher_id)); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Some watchers are reserved for CPython, start at the first available one */ for (int i = FIRST_AVAILABLE_WATCHER; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -8102,7 +8128,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the @@ -8172,7 +8199,8 @@ static PyMethodDef frozendict_methods[] = { DICT_FROMKEYS_METHODDEF FROZENDICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozendicts are generic over two types, signifying (respectively) the types of the frozendict's keys and values")}, {"__getnewargs__", frozendict_getnewargs, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -8205,6 +8233,39 @@ _shuffle_bits(Py_uhash_t h) return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; } +// Compute hash((key, value)). +// Code copied from tuple_hash(). +static Py_hash_t +frozendict_pair_hash(Py_hash_t key_hash, PyObject *value) +{ + assert(key_hash != -1); + + const Py_ssize_t len = 2; + Py_uhash_t acc = _PyTuple_HASH_XXPRIME_5; + + Py_uhash_t lane = key_hash; + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + lane = PyObject_Hash(value); + if (lane == (Py_uhash_t)-1) { + return -1; + } + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + /* Add input length, mangled to keep the historical value of hash(()). */ + acc += len ^ (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL); + + if (acc == (Py_uhash_t)-1) { + acc = 1546275796; + } + return acc; +} + + // Code copied from frozenset_hash() static Py_hash_t frozendict_hash(PyObject *op) @@ -8218,20 +8279,15 @@ frozendict_hash(PyObject *op) PyDictObject *mp = _PyAnyDict_CAST(op); Py_uhash_t hash = 0; - PyObject *key, *value; // borrowed refs + PyObject *value; // borrowed ref Py_ssize_t pos = 0; - while (PyDict_Next(op, &pos, &key, &value)) { - Py_hash_t key_hash = PyObject_Hash(key); - if (key_hash == -1) { - return -1; - } - hash ^= _shuffle_bits(key_hash); - - Py_hash_t value_hash = PyObject_Hash(value); - if (value_hash == -1) { + Py_hash_t key_hash; + while (_PyDict_Next(op, &pos, NULL, &value, &key_hash)) { + Py_hash_t pair_hash = frozendict_pair_hash(key_hash, value); + if (pair_hash == -1) { return -1; } - hash ^= _shuffle_bits(value_hash); + hash ^= _shuffle_bits(pair_hash); } /* Factor in the number of active entries */ diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 364d508dd01822c..fc53f1bfee8dde4 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -290,7 +290,7 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { {"__reduce__", enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("'enumerate' objects are generic over the type of their values")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 5e5e87cd6d7559f..10d100384be7aa5 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1743,7 +1743,8 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Exception groups are generic over the type of their contained exceptions")}, BASEEXCEPTIONGROUP_DERIVE_METHODDEF BASEEXCEPTIONGROUP_SPLIT_METHODDEF BASEEXCEPTIONGROUP_SUBGROUP_METHODDEF diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d91468dddded9bf..17e6a729dcd83fc 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1670,7 +1670,6 @@ float___getnewargs___impl(PyObject *self) /*[clinic input] -@permit_long_docstring_body @classmethod float.__getformat__ @@ -1683,13 +1682,13 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. This function returns whichever of 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating-point numbers used by the -C type named by typestr. +little-endian' best describes the format of floating-point numbers +used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=0ae1ba35d192f704]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=eb1cf45e9bddab72]*/ { if (strcmp(typestr, "double") != 0 && strcmp(typestr, "float") != 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5ae85c5bca61b94..f60cdb2dd1bf20d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1890,6 +1890,7 @@ frame_trace_set_impl(PyFrameObject *self, PyObject *value) } /*[clinic input] +@permit_long_summary @critical_section @getter frame.f_generator as frame_generator @@ -1899,7 +1900,7 @@ Return the generator or coroutine associated with this frame, or None. static PyObject * frame_generator_get_impl(PyFrameObject *self) -/*[clinic end generated code: output=97aeb2392562e55b input=00a2bd008b239ab0]*/ +/*[clinic end generated code: output=97aeb2392562e55b input=3ffba57ba10f84be]*/ { if (self->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(self->f_frame); diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index e3bc8eb2739e3fa..9c797e8dd6fd2cc 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -412,6 +412,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; diff --git a/Objects/genobject.c b/Objects/genobject.c index 8c5d720c0b9035c..38d493343454fce 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1023,7 +1023,8 @@ static PyMethodDef gen_methods[] = { {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("generators are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1374,7 +1375,8 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("coroutines are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1820,7 +1822,7 @@ static PyMethodDef async_gen_methods[] = { {"aclose", async_gen_aclose, METH_NOARGS, async_aclose_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("async generators are generic over the types of their yield and send values")}, {NULL, NULL} /* Sentinel */ }; diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index b58adb693f0cae5..e37724fb7852a27 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -138,7 +138,7 @@ static PyMethodDef interpolation_methods[] = { {"__reduce__", interpolation_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Interpolations are generic over the types of their values")}, {NULL, NULL}, }; diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 451f335e033f16b..fa1eb25047d9617 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -135,7 +135,7 @@ PyDoc_STRVAR(lazy_import_doc, "lazy_import(builtins, name, fromlist=None, /)\n" "--\n" "\n" -"Represents a deferred import that will be resolved on first use.\n" +"Represents a lazy import that will be resolved on first use.\n" "\n" "Instances of this object accessed from the global scope will be\n" "automatically imported based upon their name and then replaced with\n" diff --git a/Objects/listobject.c b/Objects/listobject.c index 10e25bbdcdcb6c5..8a9c9bda68269b8 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2922,7 +2922,6 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] -@permit_long_docstring_body @critical_section list.sort @@ -2932,18 +2931,18 @@ list.sort Sort the list in ascending order and return None. -The sort is in-place (i.e. the list itself is modified) and stable (i.e. the -order of two equal elements is maintained). +The sort is in-place (i.e. the list itself is modified) and stable +(i.e. the order of two equal elements is maintained). -If a key function is given, apply it once to each list item and sort them, -ascending or descending, according to their function values. +If a key function is given, apply it once to each list item and sort +them, ascending or descending, according to their function values. The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=e4f6b6069181ad7d]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=c145526281e1fb9f]*/ { MergeState ms; Py_ssize_t nremaining; @@ -3611,7 +3610,8 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("lists are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -3793,16 +3793,13 @@ list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) lim = Py_SIZE(self) - cur - 1; } - memmove(self->ob_item + cur - i, - self->ob_item + cur + 1, - lim * sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - i, + self->ob_item + cur + 1, lim); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + cur - slicelength, - self->ob_item + cur, - (Py_SIZE(self) - cur) * - sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - slicelength, + self->ob_item + cur, Py_SIZE(self) - cur); } Py_SET_SIZE(self, Py_SIZE(self) - slicelength); diff --git a/Objects/longobject.c b/Objects/longobject.c index 549cf0b8f12b4e4..6e6011cb19aab5f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6367,20 +6367,21 @@ int_as_integer_ratio_impl(PyObject *self) int.to_bytes length: Py_ssize_t(allow_negative=False) = 1 - Length of bytes object to use. An OverflowError is raised if the - integer is not representable with the given number of bytes. Default - is length 1. + Length of bytes object to use. An OverflowError is raised if + the integer is not representable with the given number of bytes. + Default is length 1. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Determines whether two's complement is used to represent the integer. - If signed is False and a negative integer is given, an OverflowError - is raised. + Determines whether two's complement is used to represent the + integer. If signed is False and a negative integer is given, + an OverflowError is raised. Return an array of bytes representing an integer. [clinic start generated code]*/ @@ -6388,7 +6389,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=66f9d0c20529b44f]*/ +/*[clinic end generated code: output=89c801df114050a3 input=c74a93c07b2f6526]*/ { int little_endian; if (byteorder == NULL) @@ -6424,18 +6425,20 @@ int.from_bytes bytes as bytes_obj: object Holds the array of bytes to convert. The argument must either - support the buffer protocol or be an iterable object producing bytes. - Bytes and bytearray are examples of built-in objects that support the - buffer protocol. + support the buffer protocol or be an iterable object producing + bytes. Bytes and bytearray are examples of built-in objects that + support the buffer protocol. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Indicates whether two's complement is used to represent the integer. + Indicates whether two's complement is used to represent the + integer. Return the integer represented by the given array of bytes. [clinic start generated code]*/ @@ -6443,7 +6446,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=95801e50b942e164]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6508,6 +6511,7 @@ long_long_getter(PyObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@permit_long_summary int.is_integer Returns True. Exists for duck type compatibility with float.is_integer. @@ -6515,7 +6519,7 @@ Returns True. Exists for duck type compatibility with float.is_integer. static PyObject * int_is_integer_impl(PyObject *self) -/*[clinic end generated code: output=90f8e794ce5430ef input=7e41c4d4416e05f2]*/ +/*[clinic end generated code: output=90f8e794ce5430ef input=aacf01a2c81c0244]*/ { Py_RETURN_TRUE; } @@ -6597,7 +6601,8 @@ If x is not a number or if base is given, then x must be a string,\n\ bytes, or bytearray instance representing an integer literal in the\n\ given base. The literal can be preceded by '+' or '-' and be surrounded\n\ by whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\n\ -Base 0 means to interpret the base from the string as an integer literal.\n\ +Base 0 means to interpret the base from the string as an integer\n\ +iteral.\n\ >>> int('0b100', base=0)\n\ 4"); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 900db864621a84c..a05d3c3b7a7f469 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1629,11 +1629,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, 1); -#else - self->exports++; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, 1); return 0; } @@ -1642,11 +1638,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, -1); -#else - self->exports--; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -2347,23 +2339,23 @@ memoryview_tolist_impl(PyMemoryViewObject *self) } /*[clinic input] -@permit_long_docstring_body memoryview.tobytes order: str(accept={str, NoneType}, c_default="NULL") = 'C' Return the data in the buffer as a byte string. -Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the -original array is converted to C or Fortran order. For contiguous views, -'A' returns an exact copy of the physical memory. In particular, in-memory -Fortran order is preserved. For non-contiguous views, the data is converted -to C first. order=None is the same as order='C'. +Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of +the original array is converted to C or Fortran order. For +contiguous views, 'A' returns an exact copy of the physical memory. +In particular, in-memory Fortran order is preserved. For +non-contiguous views, the data is converted to C first. order=None +is the same as order='C'. [clinic start generated code]*/ static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=23c9faf372cfdbcc]*/ +/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; @@ -2404,8 +2396,8 @@ memoryview.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Return the data in the buffer as a str of hexadecimal numbers. @@ -2424,7 +2416,7 @@ Return the data in the buffer as a str of hexadecimal numbers. static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=c9bb00c7a8e86056 input=dc48a56ed3b058ae]*/ +/*[clinic end generated code: output=c9bb00c7a8e86056 input=3f1c5d08906e3b70]*/ { Py_buffer *src = VIEW_ADDR(self); @@ -2434,9 +2426,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, // Prevent 'self' from being freed if computing len(sep) mutates 'self' // in _Py_strhex_with_sep(). // See: https://github.com/python/cpython/issues/143195. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return ret; } @@ -3363,9 +3355,9 @@ memory_hash(PyObject *_self) if (view->obj != NULL) { // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); Py_hash_t h = PyObject_Hash(view->obj); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); if (h == -1) { /* Keep the original error message */ return -1; @@ -3587,7 +3579,8 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("memoryviews are generic over the type of their underlying data")}, {NULL, NULL} }; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b7d2e5ffde4fe7d..f447403ef31b43a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,6 +1299,33 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } +// Check if `name` is a lazily pending submodule of module `m`. +// Returns a new reference on success, or NULL with no error set. +static PyObject * +try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +{ + PyObject *mod_name; + int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); + if (rc <= 0) { + return NULL; + } + if (!PyUnicode_Check(mod_name)) { + Py_DECREF(mod_name); + return NULL; + } + PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + Py_DECREF(mod_name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (PyDict_SetItem(m->md_dict, name, result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { @@ -1307,6 +1334,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { if (PyLazyImport_CheckExact(attr)) { + // gh-144957: Module __getattr__ should get a chance to provide + // the attribute before resolving a lazy import placeholder. + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_DECREF(attr); + return NULL; + } + if (getattr) { + PyObject *result = PyObject_CallOneArg(getattr, name); + Py_DECREF(getattr); + if (result != NULL) { + Py_DECREF(attr); + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(attr); + return NULL; + } + PyErr_Clear(); + } PyObject *new_value = _PyImport_LoadLazyImportTstate( PyThreadState_GET(), attr); if (new_value == NULL) { @@ -1344,6 +1390,13 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); + attr = try_load_lazy_submodule(m, name); + if (attr != NULL) { + return attr; + } + if (PyErr_Occurred()) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b391283e83795d0..6f05395b18d781f 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1149,12 +1149,13 @@ OrderedDict.popitem Remove and return a (key, value) pair from the dictionary. -Pairs are returned in LIFO order if last is true or FIFO order if false. +Pairs are returned in LIFO order if last is true or FIFO order if +false. [clinic start generated code]*/ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) -/*[clinic end generated code: output=98e7d986690d49eb input=8aafc7433e0a40e7]*/ +/*[clinic end generated code: output=98e7d986690d49eb input=ebf1cc91579c9e54]*/ { PyObject *key, *value; _ODictNode *node; diff --git a/Objects/sentinelobject.c b/Objects/sentinelobject.c index e7e9f60e3edfbe9..77bffbc397be585 100644 --- a/Objects/sentinelobject.c +++ b/Objects/sentinelobject.c @@ -14,6 +14,7 @@ typedef struct { PyObject_HEAD PyObject *name; PyObject *module; + PyObject *repr; } sentinelobject; #define sentinelobject_CAST(op) \ @@ -46,7 +47,7 @@ caller(void) } static PyObject * -sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) +sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module, PyObject *repr) { assert(PyUnicode_Check(name)); @@ -56,6 +57,7 @@ sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) } self->name = Py_NewRef(name); self->module = Py_NewRef(module); + self->repr = Py_XNewRef(repr); _PyObject_GC_TRACK(self); return (PyObject *)self; } @@ -66,37 +68,56 @@ sentinel.__new__ as sentinel_new name: object(subclass_of='&PyUnicode_Type') / + * + repr: object = None [clinic start generated code]*/ static PyObject * -sentinel_new_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=4af55c6048bed30d input=3ab75704f39c119c]*/ +sentinel_new_impl(PyTypeObject *type, PyObject *name, PyObject *repr) +/*[clinic end generated code: output=1eb7fab52e57d8c8 input=28cab6c468997b35]*/ { + if (repr == Py_None) { + repr = NULL; + } + else if (!PyUnicode_Check(repr)) { + _PyArg_BadArgument("sentinel", "argument 'repr'", "str or None", repr); + return NULL; + } PyObject *module = caller(); - PyObject *self = sentinel_new_with_module(type, name, module); + PyObject *self = sentinel_new_with_module(type, name, module, repr); Py_DECREF(module); return self; } PyObject * -PySentinel_New(const char *name, const char *module_name) +PySentinel_New(const char *name, const char *module_name, const char *repr) { PyObject *name_obj = PyUnicode_FromString(name); if (name_obj == NULL) { return NULL; } + PyObject *repr_obj = NULL; + if (repr != NULL) { + repr_obj = PyUnicode_FromString(repr); + if (repr_obj == NULL) { + Py_DECREF(name_obj); + return NULL; + } + } PyObject *module_obj = module_name == NULL ? Py_None : PyUnicode_FromString(module_name); if (module_obj == NULL) { Py_DECREF(name_obj); + Py_XDECREF(repr_obj); return NULL; } PyObject *sentinel = sentinel_new_with_module( - &PySentinel_Type, name_obj, module_obj); + &PySentinel_Type, name_obj, module_obj, repr_obj); Py_DECREF(module_obj); Py_DECREF(name_obj); + Py_XDECREF(repr_obj); return sentinel; } @@ -106,6 +127,7 @@ sentinel_clear(PyObject *op) sentinelobject *self = sentinelobject_CAST(op); Py_CLEAR(self->name); Py_CLEAR(self->module); + Py_CLEAR(self->repr); return 0; } @@ -123,6 +145,7 @@ sentinel_traverse(PyObject *op, visitproc visit, void *arg) sentinelobject *self = sentinelobject_CAST(op); Py_VISIT(self->name); Py_VISIT(self->module); + Py_VISIT(self->repr); return 0; } @@ -130,6 +153,9 @@ static PyObject * sentinel_repr(PyObject *op) { sentinelobject *self = sentinelobject_CAST(op); + if (self->repr != NULL) { + return Py_NewRef(self->repr); + } return Py_NewRef(self->name); } @@ -161,7 +187,7 @@ static PyMethodDef sentinel_methods[] = { static PyMemberDef sentinel_members[] = { {"__name__", Py_T_OBJECT_EX, offsetof(sentinelobject, name), Py_READONLY}, - {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), Py_READONLY}, + {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), 0}, {NULL} }; @@ -170,7 +196,7 @@ static PyNumberMethods sentinel_as_number = { }; PyDoc_STRVAR(sentinel_doc, -"sentinel(name, /)\n" +"sentinel(name, /, *, repr=None)\n" "--\n\n" "Create a unique sentinel object with the given name."); diff --git a/Objects/setobject.c b/Objects/setobject.c index 1e6305636045529..7644ea0baf73dd1 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2111,6 +2111,7 @@ set_difference(PySetObject *so, PyObject *other) } /*[clinic input] +@permit_long_summary set.difference as set_difference_multi so: setobject *others: array @@ -2121,7 +2122,7 @@ Return a new set with elements in the set that are not in the others. static PyObject * set_difference_multi_impl(PySetObject *so, PyObject * const *others, Py_ssize_t others_length) -/*[clinic end generated code: output=b0d33fb05d5477a7 input=c1eb448d483416ad]*/ +/*[clinic end generated code: output=b0d33fb05d5477a7 input=e0fbedbf79d91d4e]*/ { Py_ssize_t i; PyObject *result, *other; @@ -2293,6 +2294,7 @@ set_symmetric_difference_update_impl(PySetObject *so, PyObject *other) } /*[clinic input] +@permit_long_summary @critical_section so other set.symmetric_difference so: setobject @@ -2304,7 +2306,7 @@ Return a new set with elements in either the set or other but not both. static PyObject * set_symmetric_difference_impl(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ +/*[clinic end generated code: output=270ee0b5d42b0797 input=8c29b0be90d47feb]*/ { PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); if (result == NULL) { @@ -2790,7 +2792,8 @@ static PyMethodDef set_methods[] = { SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF SET_UNION_METHODDEF SET_UPDATE_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("sets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2894,7 +2897,8 @@ static PyMethodDef frozenset_methods[] = { SET___SIZEOF___METHODDEF SET_SYMMETRIC_DIFFERENCE_METHODDEF SET_UNION_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozensets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 95f10815687757e..0d05eb7a47bb1d0 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -342,7 +342,9 @@ PyDoc_STRVAR(slice_doc, "slice(stop)\n\ slice(start, stop[, step])\n\ \n\ -Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); +Create a slice object.\n\ +\n\ +This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PyObject *op) @@ -570,7 +572,8 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef slice_methods[] = { {"indices", slice_indices, METH_O, slice_indices_doc}, {"__reduce__", slice_reduce, METH_NOARGS, reduce_doc}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + "slices are generic over the types of their start, end, and step values"}, {NULL, NULL} }; diff --git a/Objects/templateobject.c b/Objects/templateobject.c index a05208e4c8fc8e8..1609e82b444516c 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -372,7 +372,11 @@ template_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef template_methods[] = { {"__reduce__", template_reduce, METH_NOARGS, NULL}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + // note that this is not supported in typeshed, and it is not clear if the + // type for this is a simple TypeVar or a TypeVarTuple + // for details, see: https://github.com/python/typeshed/issues/14878 + PyDoc_STR("Template supports [] for generic usage")}, {NULL, NULL}, }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 753c270f525976f..5aa5188905305a4 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -363,6 +363,9 @@ tuple_repr(PyObject *self) https://github.com/Cyan4973/xxHash/blob/master/doc/xxhash_spec.md The constants for the hash function are defined in pycore_tuple.h. + + If you update this code, update also frozendict_pair_hash() which copied + this code. */ static Py_hash_t @@ -951,11 +954,17 @@ tuple___getnewargs___impl(PyTupleObject *self) return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } + +PyDoc_STRVAR(tuple_class_getitem_doc, +"Tuples are generic over the types of their contents.\n\n\ +For example, use ``tuple[int, str]`` for a pair whose first element is an int and second element is a string.\n\n\ +Tuples also support the form ``tuple[T, ...]`` to indicate an arbitrary length tuple of elements of type T."); + static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, tuple_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4f43747ba83fd9d..e0464fe6475cfd2 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4841,6 +4841,18 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(dict) < 0) { return -1; } + +#ifdef Py_GIL_DISABLED + // enable deferred reference counting on functions and descriptors + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + if (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL) { + PyUnstable_Object_EnableDeferredRefcount(value); + } + } +#endif + return 0; } @@ -6478,9 +6490,25 @@ set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) { + BEGIN_TYPE_LOCK(); + /* Ideally, changing flags and invalidating the old version tag would + happen in one step. But type_modified_unlocked() is re-entrant and + cannot run with the world stopped, so we must invalidate first. + Immutable/static-builtin types are skipped because + set_flags_recursive() does not modify them. */ + if (!PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) && + (self->tp_flags & mask) != flags) + { + type_modified_unlocked(self); + } + /* Keep TYPE_LOCK held while waiting for stop-the-world so no thread + can reassign a version tag before the flag update. */ + type_lock_prevent_release(); types_stop_world(); set_flags_recursive(self, mask, flags); types_start_world(); + type_lock_allow_release(); + END_TYPE_LOCK(); } /* This is similar to PyObject_GenericGetAttr(), @@ -6746,12 +6774,11 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); #ifdef Py_GIL_DISABLED - // gh-139103: Enable deferred refcounting for functions assigned - // to type objects. This is important for `dataclass.__init__`, - // which is generated dynamically. - if (value != NULL && - PyFunction_Check(value) && - !_PyObject_HasDeferredRefcount(value)) + // gh-139103: Enable deferred refcounting for functions and descriptors + // assigned to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically, and for descriptor scaling on + // free-threaded builds. + if (value != NULL && (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL)) { PyUnstable_Object_EnableDeferredRefcount(value); } @@ -11079,14 +11106,22 @@ slot_tp_iternext(PyObject *self) return vectorcall_method(&_Py_ID(__next__), stack, 1); } +int +_PyType_HasSlotTpIternext(PyTypeObject *type) +{ + return type->tp_iternext == slot_tp_iternext; +} + static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { PyTypeObject *tp = Py_TYPE(self); - PyObject *get; - - get = _PyType_LookupRef(tp, &_Py_ID(__get__)); - if (get == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + _PyType_LookupStackRefAndVersion(tp, &_Py_ID(__get__), &cref.ref); + if (PyStackRef_IsNull(cref.ref)) { + _PyThreadState_PopCStackRef(tstate, &cref); #ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -11098,9 +11133,10 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) obj = Py_None; if (type == NULL) type = Py_None; + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); PyObject *stack[3] = {self, obj, type}; PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); return res; } @@ -12905,7 +12941,8 @@ PyDoc_STRVAR(super_doc, "super() -> same as super(__class__, <first argument>)\n" "super(type) -> unbound super object\n" "super(type, obj) -> bound super object; requires isinstance(obj, type)\n" -"super(type, type2) -> bound super object; requires issubclass(type2, type)\n" +"super(type, type2) -> bound super object; requires\n" +" issubclass(type2, type)\n" "Typical use to call a cooperative superclass method:\n" "class C(B):\n" " def meth(self, arg):\n" diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9aee7120c811de8..74d6ba4db9f2b88 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -10794,35 +10794,33 @@ replace(PyObject *self, PyObject *str1, /* --- Unicode Object Methods --------------------------------------------- */ /*[clinic input] -@permit_long_docstring_body str.title as unicode_title Return a version of the string where each word is titlecased. -More specifically, words start with uppercased characters and all remaining -cased characters have lower case. +More specifically, words start with uppercased characters and all +remaining cased characters have lower case. [clinic start generated code]*/ static PyObject * unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=533ce0eb6a7f5d1b]*/ +/*[clinic end generated code: output=c75ae03809574902 input=2a07e2c7df94627a]*/ { return case_operation(self, do_title); } /*[clinic input] -@permit_long_docstring_body str.capitalize as unicode_capitalize Return a capitalized version of the string. -More specifically, make the first character have upper case and the rest lower -case. +More specifically, make the first character have upper case and the +rest lower case. [clinic start generated code]*/ static PyObject * unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=a4a15ade41f6f9e9]*/ +/*[clinic end generated code: output=e49a4c333cdb7667 input=e50e50ed45a654cf]*/ { if (PyUnicode_GET_LENGTH(self) == 0) return unicode_result_unchanged(self); @@ -10876,12 +10874,13 @@ str.center as unicode_center Return a centered string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_center_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=420c8859effc7c0c input=b42b247eb26e6519]*/ +/*[clinic end generated code: output=420c8859effc7c0c input=df91017dfd186a78]*/ { Py_ssize_t marg, left; @@ -11441,13 +11440,14 @@ str.count as unicode_count -> Py_ssize_t Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -Optional arguments start and end are interpreted as in slice notation. +Optional arguments start and end are interpreted as in slice +notation. [clinic start generated code]*/ static Py_ssize_t unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=8fcc3aef0b18edbf input=8590716ee228b935]*/ +/*[clinic end generated code: output=8fcc3aef0b18edbf input=c9209e05438cc352]*/ { assert(PyUnicode_Check(str)); assert(PyUnicode_Check(substr)); @@ -11520,8 +11520,8 @@ str.encode as unicode_encode errors: str(c_default="NULL") = 'strict' The error handling scheme to use for encoding errors. The default is 'strict' meaning that encoding errors raise a - UnicodeEncodeError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with + UnicodeEncodeError. Other possible values are 'ignore', 'replace' + and 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle UnicodeEncodeErrors. Encode the string using the codec registered for encoding. @@ -11529,7 +11529,7 @@ Encode the string using the codec registered for encoding. static PyObject * unicode_encode_impl(PyObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=bf78b6e2a9470e3c input=f0a9eb293d08fe02]*/ +/*[clinic end generated code: output=bf78b6e2a9470e3c input=b85a9645cb33b729]*/ { return PyUnicode_AsEncodedString(self, encoding, errors); } @@ -11626,14 +11626,14 @@ str.find as unicode_find = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51dbe6255712e278 input=3a9d650fe4c24695]*/ +/*[clinic end generated code: output=51dbe6255712e278 input=f57e93c59d1ee927]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result < 0) { @@ -11690,14 +11690,14 @@ str.index as unicode_index = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=77558288837cdf40 input=ae5e48f69ed75b06]*/ +/*[clinic end generated code: output=77558288837cdf40 input=5900ab84de55e628]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result == -1) { @@ -11710,6 +11710,7 @@ unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, } /*[clinic input] +@permit_long_summary str.isascii as unicode_isascii Return True if all characters in the string are ASCII, False otherwise. @@ -11720,24 +11721,23 @@ Empty string is ASCII too. static PyObject * unicode_isascii_impl(PyObject *self) -/*[clinic end generated code: output=c5910d64b5a8003f input=5a43cbc6399621d5]*/ +/*[clinic end generated code: output=c5910d64b5a8003f input=dc74e1ced821159f]*/ { return PyBool_FromLong(PyUnicode_IS_ASCII(self)); } /*[clinic input] -@permit_long_docstring_body str.islower as unicode_islower Return True if the string is a lowercase string, False otherwise. -A string is lowercase if all cased characters in the string are lowercase and -there is at least one cased character in the string. +A string is lowercase if all cased characters in the string are +lowercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=c6fc0295241a1aaa]*/ +/*[clinic end generated code: output=dbd41995bd005b81 input=1879b48dfc628366]*/ { Py_ssize_t i, length; int kind; @@ -11770,18 +11770,17 @@ unicode_islower_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isupper as unicode_isupper Return True if the string is an uppercase string, False otherwise. -A string is uppercase if all cased characters in the string are uppercase and -there is at least one cased character in the string. +A string is uppercase if all cased characters in the string are +uppercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=8d5cb33e67efde72]*/ +/*[clinic end generated code: output=049209c8e7f15f59 input=77d29904aef0e3a0]*/ { Py_ssize_t i, length; int kind; @@ -11870,18 +11869,17 @@ unicode_istitle_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isspace as unicode_isspace Return True if the string is a whitespace string, False otherwise. -A string is whitespace if all characters in the string are whitespace and there -is at least one character in the string. +A string is whitespace if all characters in the string are +whitespace and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=44fe05e248c6e159]*/ +/*[clinic end generated code: output=163a63bfa08ac2b9 input=29e09560fc23fbeb]*/ { Py_ssize_t i, length; int kind; @@ -11909,18 +11907,17 @@ unicode_isspace_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isalpha as unicode_isalpha Return True if the string is an alphabetic string, False otherwise. -A string is alphabetic if all characters in the string are alphabetic and there -is at least one character in the string. +A string is alphabetic if all characters in the string are +alphabetic and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=c233000624a56e0d]*/ +/*[clinic end generated code: output=cc81b9ac3883ec4f input=9906a07f3e04892e]*/ { Py_ssize_t i, length; int kind; @@ -11947,18 +11944,18 @@ unicode_isalpha_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.isalnum as unicode_isalnum Return True if the string is an alpha-numeric string, False otherwise. -A string is alpha-numeric if all characters in the string are alpha-numeric and -there is at least one character in the string. +A string is alpha-numeric if all characters in the string are +alpha-numeric and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5d63ba9c9bafdb6b]*/ +/*[clinic end generated code: output=a5a23490ffc3660c input=892f64ebc171fd4f]*/ { int kind; const void *data; @@ -11987,18 +11984,17 @@ unicode_isalnum_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isdecimal as unicode_isdecimal Return True if the string is a decimal string, False otherwise. -A string is a decimal string if all characters in the string are decimal and -there is at least one character in the string. +A string is a decimal string if all characters in the string are +decimal and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=8e84a58b414935a3]*/ +/*[clinic end generated code: output=fb2dcdb62d3fc548 input=63b0453c48cad0af]*/ { Py_ssize_t i, length; int kind; @@ -12025,18 +12021,17 @@ unicode_isdecimal_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isdigit as unicode_isdigit Return True if the string is a digit string, False otherwise. -A string is a digit string if all characters in the string are digits and there -is at least one character in the string. +A string is a digit string if all characters in the string are +digits and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=99e284affb54d4a0]*/ +/*[clinic end generated code: output=10a6985311da6858 input=353b03747b062e4b]*/ { Py_ssize_t i, length; int kind; @@ -12064,18 +12059,17 @@ unicode_isdigit_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isnumeric as unicode_isnumeric Return True if the string is a numeric string, False otherwise. -A string is numeric if all characters in the string are numeric and there is at -least one character in the string. +A string is numeric if all characters in the string are numeric and +there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=e9f5b6b8b29b0ee6]*/ +/*[clinic end generated code: output=9172a32d9013051a input=83b2a072ed7aff48]*/ { Py_ssize_t i, length; int kind; @@ -12145,18 +12139,18 @@ PyUnicode_IsIdentifier(PyObject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.isidentifier as unicode_isidentifier Return True if the string is a valid Python identifier, False otherwise. -Call keyword.iskeyword(s) to test whether string s is a reserved identifier, -such as "def" or "class". +Call keyword.iskeyword(s) to test whether string s is a reserved +identifier, such as "def" or "class". [clinic start generated code]*/ static PyObject * unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=86315dd889d7bd04]*/ +/*[clinic end generated code: output=fe585a9666572905 input=cabde62c20a3be6b]*/ { return PyBool_FromLong(PyUnicode_IsIdentifier(self)); } @@ -12196,7 +12190,6 @@ unicode_isprintable_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.join as unicode_join iterable: object @@ -12204,15 +12197,15 @@ str.join as unicode_join Concatenate any number of strings. -The string whose method is called is inserted in between each given string. -The result is returned as a new string. +The string whose method is called is inserted in between each given +string. The result is returned as a new string. Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' [clinic start generated code]*/ static PyObject * unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=bac724ed412ef3f8]*/ +/*[clinic end generated code: output=6857e7cecfe7bf98 input=fd330a11ee845fb2]*/ { return PyUnicode_Join(self, iterable); } @@ -12232,12 +12225,13 @@ str.ljust as unicode_ljust Return a left-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/ +/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=8a55f06694c20ed6]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12593,7 +12587,6 @@ unicode_replace_impl(PyObject *self, PyObject *old, PyObject *new, } /*[clinic input] -@permit_long_docstring_body str.removeprefix as unicode_removeprefix prefix: unicode @@ -12601,13 +12594,14 @@ str.removeprefix as unicode_removeprefix Return a str with the given prefix string removed if present. -If the string starts with the prefix string, return string[len(prefix):]. -Otherwise, return a copy of the original string. +If the string starts with the prefix string, return +string[len(prefix):]. Otherwise, return a copy of the original +string. [clinic start generated code]*/ static PyObject * unicode_removeprefix_impl(PyObject *self, PyObject *prefix) -/*[clinic end generated code: output=f1e5945e9763bcb9 input=1989a856dbb813f1]*/ +/*[clinic end generated code: output=f1e5945e9763bcb9 input=90d162724944bfa7]*/ { int match = tailmatch(self, prefix, 0, PY_SSIZE_T_MAX, -1); if (match == -1) { @@ -12628,14 +12622,14 @@ str.removesuffix as unicode_removesuffix Return a str with the given suffix string removed if present. -If the string ends with the suffix string and that suffix is not empty, -return string[:-len(suffix)]. Otherwise, return a copy of the original -string. +If the string ends with the suffix string and that suffix is not +empty, return string[:-len(suffix)]. Otherwise, return a copy of +the original string. [clinic start generated code]*/ static PyObject * unicode_removesuffix_impl(PyObject *self, PyObject *suffix) -/*[clinic end generated code: output=d36629e227636822 input=12cc32561e769be4]*/ +/*[clinic end generated code: output=d36629e227636822 input=6efc96152d4bfcd5]*/ { int match = tailmatch(self, suffix, 0, PY_SSIZE_T_MAX, +1); if (match == -1) { @@ -12745,14 +12739,14 @@ str.rfind as unicode_rfind = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=880b29f01dd014c8 input=7f7e97d5cd3299a2]*/ +/*[clinic end generated code: output=880b29f01dd014c8 input=2e67789533baf2f5]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { @@ -12767,14 +12761,14 @@ str.rindex as unicode_rindex = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=5f3aef124c867fe1 input=0363a324740b3e62]*/ +/*[clinic end generated code: output=5f3aef124c867fe1 input=e29d446c8234c9d9]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result == -1) { @@ -12795,12 +12789,13 @@ str.rjust as unicode_rjust Return a right-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/ +/*[clinic end generated code: output=804a1a57fbe8d5cf input=1256a8d659589907]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12824,9 +12819,9 @@ str.split as unicode_split sep: object = None The separator used to split the string. - When set to None (the default value), will split on any whitespace - character (including \n \r \t \f and spaces) and will discard - empty strings from the result. + When set to None (the default value), will split on any + whitespace character (including \n \r \t \f and spaces) and + will discard empty strings from the result. maxsplit: Py_ssize_t = -1 Maximum number of splits. -1 (the default value) means no limit. @@ -12835,15 +12830,15 @@ Return a list of the substrings in the string, using sep as the separator string Splitting starts at the front of the string and works to the end. -Note, str.split() is mainly useful for data that has been intentionally -delimited. With natural text that includes punctuation, consider using -the regular expression module. +Note, str.split() is mainly useful for data that has been +intentionally delimited. With natural text that includes +punctuation, consider using the regular expression module. [clinic start generated code]*/ static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=2c1fd08a78e038b8]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=288cfd6bc8828f5a]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -12960,7 +12955,6 @@ PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) } /*[clinic input] -@permit_long_docstring_body str.partition as unicode_partition sep: object @@ -12968,38 +12962,38 @@ str.partition as unicode_partition Partition the string into three parts using the given separator. -This will search for the separator in the string. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator in the string. If the separator +is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original string -and two empty strings. +If the separator is not found, returns a 3-tuple containing +the original string and two empty strings. [clinic start generated code]*/ static PyObject * unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=4d854b520d7b0e97]*/ +/*[clinic end generated code: output=e4ced7bd253ca3c4 input=e45faa8c26270cb1]*/ { return PyUnicode_Partition(self, sep); } /*[clinic input] -@permit_long_docstring_body str.rpartition as unicode_rpartition = str.partition Partition the string into three parts using the given separator. -This will search for the separator in the string, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator in the string, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty strings -and the original string. +If the separator is not found, returns a 3-tuple containing two +empty strings and the original string. [clinic start generated code]*/ static PyObject * unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=a6adabe91e75b486]*/ +/*[clinic end generated code: output=1aa13cf1156572aa input=53a7f8cb19975b7c]*/ { return PyUnicode_RPartition(self, sep); } @@ -13038,20 +13032,20 @@ unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.splitlines as unicode_splitlines keepends: bool = False Return a list of the lines in the string, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=39eeafbfef61c827]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=b45ea0f87645a06d]*/ { return PyUnicode_Splitlines(self, keepends); } @@ -13130,18 +13124,19 @@ str.maketrans as unicode_maketrans Return a translation table usable for str.translate(). -If there is only one argument, it must be a dictionary mapping Unicode -ordinals (integers) or characters to Unicode ordinals, strings or None. -Character keys will be then converted to ordinals. -If there are two arguments, they must be strings of equal length, and -in the resulting dictionary, each character in x will be mapped to the -character at the same position in y. If there is a third argument, it -must be a string, whose characters will be mapped to None in the result. +If there is only one argument, it must be a dictionary mapping +Unicode ordinals (integers) or characters to Unicode ordinals, +strings or None. Character keys will be then converted to ordinals. +If there are two arguments, they must be strings of equal length, +and in the resulting dictionary, each character in x will be mapped +to the character at the same position in y. If there is a third +argument, it must be a string, whose characters will be mapped to +None in the result. [clinic start generated code]*/ static PyObject * unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) -/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/ +/*[clinic end generated code: output=a925c89452bd5881 input=66bc00a1b4258a6e]*/ { PyObject *new = NULL, *key, *value; Py_ssize_t i = 0; @@ -13221,24 +13216,25 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.translate as unicode_translate table: object - Translation table, which must be a mapping of Unicode ordinals to - Unicode ordinals, strings, or None. + Translation table, which must be a mapping of Unicode ordinals + to Unicode ordinals, strings, or None. / Replace each character in the string using the given translation table. -The table must implement lookup/indexing via __getitem__, for instance a -dictionary or list. If this operation raises LookupError, the character is -left untouched. Characters mapped to None are deleted. +The table must implement lookup/indexing via __getitem__, for +instance a dictionary or list. If this operation raises +LookupError, the character is left untouched. Characters mapped to +None are deleted. [clinic start generated code]*/ static PyObject * unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=699e5fa0ebf9f5e9]*/ +/*[clinic end generated code: output=3cb448ff2fd96bf3 input=48cf0efe06bc1b75]*/ { return _PyUnicode_TranslateCharmap(self, table, "ignore"); } @@ -13434,6 +13430,7 @@ Return a formatted version of the string, using substitutions from mapping.\n\ The substitutions are identified by braces ('{' and '}')."); /*[clinic input] +@permit_long_summary str.__format__ as unicode___format__ format_spec: unicode @@ -13444,7 +13441,7 @@ Return a formatted version of the string as described by format_spec. static PyObject * unicode___format___impl(PyObject *self, PyObject *format_spec) -/*[clinic end generated code: output=45fceaca6d2ba4c8 input=5e135645d167a214]*/ +/*[clinic end generated code: output=45fceaca6d2ba4c8 input=77a2a19f3f7969f2]*/ { _PyUnicodeWriter writer; int ret; diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 0f6b1e44bc2402c..1dc2927b6e6ac70 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -518,7 +518,8 @@ union_mro_entries(PyObject *self, PyObject *args) static PyMethodDef union_methods[] = { {"__mro_entries__", union_mro_entries, METH_O}, - {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, + PyDoc_STR("Create a union containing the given types")}, {0} }; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 61fa3ddad0bfd83..8446a2dbcf75593 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -491,7 +491,8 @@ static PyMemberDef weakref_members[] = { static PyMethodDef weakref_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Weakrefs are generic over the type of the referenced object.")}, {NULL} /* Sentinel */ }; diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 86df2c7183c30d5..b9a229b1398ec8d 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -224,20 +224,19 @@ wait_event(HANDLE event, DWORD timeout) /*[clinic input] -@permit_long_docstring_body _wmi.exec_query query: unicode Runs a WMI query against the local machine. -This returns a single string with 'name=value' pairs in a flat array separated -by null characters. +This returns a single string with 'name=value' pairs in a flat array +separated by null characters. [clinic start generated code]*/ static PyObject * _wmi_exec_query_impl(PyObject *module, PyObject *query) -/*[clinic end generated code: output=a62303d5bb5e003f input=621f5c50c56d06d0]*/ +/*[clinic end generated code: output=a62303d5bb5e003f input=a8d5710acdfbf515]*/ /*[clinic end generated code]*/ { diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index 38d52d0329dcc0d..6c18990f056b5f7 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_wmi_exec_query__doc__, "\n" "Runs a WMI query against the local machine.\n" "\n" -"This returns a single string with \'name=value\' pairs in a flat array separated\n" -"by null characters."); +"This returns a single string with \'name=value\' pairs in a flat array\n" +"separated by null characters."); #define _WMI_EXEC_QUERY_METHODDEF \ {"exec_query", _PyCFunction_CAST(_wmi_exec_query), METH_FASTCALL|METH_KEYWORDS, _wmi_exec_query__doc__}, @@ -72,4 +72,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=802bcbcba69e8d0e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f246d0e568cc2d2c input=a9049054013a1b77]*/ diff --git a/PC/layout/main.py b/PC/layout/main.py index 3566b8bd873874c..f70a26b2b296591 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -22,6 +22,7 @@ __path__ = [str(Path(__file__).resolve().parent)] from .support.appxmanifest import * +from .support.builddetails import * from .support.catalog import * from .support.constants import * from .support.filesets import * @@ -32,7 +33,8 @@ from .support.pymanager import * from .support.nuspec import * -TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_3_13", "xxlimited_35", + "_ctypes_test", "_test*") TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") @@ -316,6 +318,9 @@ def _c(d): for dest, src in get_appx_layout(ns): yield dest, src + for dest, src in get_builddetails(ns): + yield dest, src + if ns.include_cat: if ns.flat_dlls: yield ns.include_cat.name, ns.include_cat diff --git a/PC/layout/support/builddetails.py b/PC/layout/support/builddetails.py new file mode 100644 index 000000000000000..6ef860eeb043545 --- /dev/null +++ b/PC/layout/support/builddetails.py @@ -0,0 +1,119 @@ +import io +import json +from . import constants + +_LEVELS = { + 0xA0: "alpha", + 0xB0: "beta", + 0xC0: "candidate", + 0xF0: "final", +} + + +_TEMPLATE = { + "schema_version": "1.0", + "base_prefix": ".", + "base_interpreter": "python.exe", + "platform": None, # Set later + "language": { + "version": f"{constants.VER_MAJOR}.{constants.VER_MINOR}", + "version_info": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + }, + "implementation": { + "name": "cpython", + "cache_tag": f"cpython-{constants.VER_MAJOR}{constants.VER_MINOR}", + "version": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + "hexversion": constants.VER_HEXVERSION, + }, + "abi": { + "flags": [], + "extension_suffix": ".pyd", + "stable_abi_suffix": ".pyd", + }, + "suffixes": { + "source": [".py", ".pyw"], + "bytecode": [".pyc"], + "extensions": [".pyd"], + }, + "libpython": { + "dynamic": constants.PYTHON_DLL_NAME, + "dynamic_stableabi": constants.PYTHON_STABLE_DLL_NAME, + "link_extensions": True, + }, + "c_api": { + }, +} + + +def _with_d(path): + pre, sep, post = path.partition(".") + return pre + "_d" + sep + post + + +def _add_d(data, *args): + for a in args[:-1]: + data = data[a] + a = args[-1] + v = data[a] + if isinstance(v, list): + data[a] = [_with_d(i) for i in data[a]] + else: + data[a] = _with_d(data[a]) + + +def get_builddetails(ns): + if not ns.include_builddetails_json: + return + + details = dict(_TEMPLATE) + + plat = { + "win32": "win32", + "amd64": "win-amd64", + "arm64": "win-arm64", + }.get(ns.arch, ns.arch) + + pyd_abi_flags = "" + if ns.include_freethreaded: + details["abi"]["flags"].append("t") + pyd_abi_flags += "t" + if ns.debug: + details["abi"]["flags"].append("d") + + norm_plat = plat.replace("-", "_") + ext_suffix = f".cp{constants.VER_MAJOR}{constants.VER_MINOR}{pyd_abi_flags}-{norm_plat}.pyd" + details["abi"]["extension_suffix"] = ext_suffix + details["suffixes"]["extensions"].insert(0, ext_suffix) + + details["platform"] = plat + + if ns.include_dev: + details["c_api"]["headers"] = "Include" + + if ns.include_freethreaded: + details["libpython"]["dynamic"] = constants.FREETHREADED_PYTHON_DLL_NAME + details["libpython"]["dynamic_stableabi"] = constants.FREETHREADED_PYTHON_STABLE_DLL_NAME + + if ns.debug: + _add_d(details, "base_interpreter") + _add_d(details, "abi", "stable_abi_suffix") + _add_d(details, "abi", "extension_suffix") + _add_d(details, "suffixes", "extensions") + _add_d(details, "libpython", "dynamic") + _add_d(details, "libpython", "dynamic_stableabi") + + buffer = io.StringIO() + json.dump(details, buffer, indent=2) + yield "build-details.json", ("build-details.json", buffer.getvalue().encode()) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 6b8c915e519743f..cb16f534685c8f8 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -23,6 +23,14 @@ def _unpack_hexversion(): return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) except OSError: pass + # Manual search for a '-s <source dir>` arument + try: + src = sys.argv[sys.argv.index("-s") + 1] + return _read_patchlevel_version(pathlib.Path(src) / "Include") + except (IndexError, ValueError): + pass + except OSError: + pass return struct.pack(">i", sys.hexversion) @@ -68,6 +76,7 @@ def check_patchlevel_version(sources): VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() +VER_HEXVERSION = (VER_MAJOR << 24) | (VER_MINOR << 16) | (VER_MICRO << 8) | (VER_FIELD4) VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 VER_DOT = "{}.{}".format(VER_MAJOR, VER_MINOR) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index e8c393385425e72..f67d8ba04d90703 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -39,6 +39,7 @@ def public(f): "install-json": {"help": "a PyManager __install__.json file"}, "install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"}, "install-test-json": {"help": "a PyManager __install__.json for the test distro"}, + "builddetails-json": {"help": "a PEP 739 build-details.json"}, } @@ -69,6 +70,7 @@ def public(f): "props", "nuspec", "alias", + "builddetails-json", ], }, "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, @@ -85,6 +87,7 @@ def public(f): "symbols", "html-doc", "alias", + "builddetails-json", ], }, "embed": { @@ -96,6 +99,7 @@ def public(f): "flat-dlls", "underpth", "precompile", + "builddetails-json", ], }, "pymanager": { @@ -108,7 +112,9 @@ def public(f): "venv", "dev", "html-doc", + "alias", "install-json", + "builddetails-json", ], }, "pymanager-test": { @@ -123,7 +129,9 @@ def public(f): "html-doc", "symbols", "tests", + "alias", "install-test-json", + "builddetails-json", ], }, } diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 831d49ea3f9b46f..f6316e0295c74af 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -66,8 +66,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): if ns.include_freethreaded: # Free-threaded distro comes with a tag suffix TAG_SUFFIX = "t" - TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" - TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" + if not ns.include_alias: + TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" + TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" DISPLAY_TAGS.append("free-threaded") FILE_SUFFIX = f"t-{ns.arch}" diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 8cdb8d722cdb9a0..1b44216dc20d1e6 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -13,6 +13,7 @@ #if defined(__clang__) #define _SILENCE_CLANG_COROUTINE_MESSAGE #endif +#define _SILENCE_EXPERIMENTAL_COROUTINE_DEPRECATION_WARNINGS #include <appmodel.h> #include <winrt\Windows.ApplicationModel.h> diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 405285b65dd270a..f6ba3d0fef3a60b 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -56,9 +56,9 @@ set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.6 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.50.4.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 +set libraries=%libraries% sqlite-3.53.1.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -80,7 +80,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.6 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index bb7d8042176d8f1..9d077bbd3f0ba27 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -84,6 +84,7 @@ <TestModules Include="_ctypes_test;_testbuffer;_testcapi;_testlimitedcapi;_testinternalcapi;_testembed;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testclinic;_testclinic_limited" /> <TestModules Include="xxlimited" Condition="'$(Configuration)' == 'Release'" /> <TestModules Include="xxlimited_35" Condition="'$(Configuration)' == 'Release'" /> + <TestModules Include="xxlimited_3_13" Condition="'$(Configuration)' == 'Release'" /> <Projects Include="@(TestModules->'%(Identity).vcxproj')" Condition="$(IncludeTests)"> <!-- Disable parallel build for test modules --> <BuildInParallel>false</BuildInParallel> diff --git a/PCbuild/python.props b/PCbuild/python.props index f70321f887ef8c0..edcda8fd8fc55d9 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -98,7 +98,7 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.50.4.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.53.1.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.8.1.1\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 70dabaa3c8bc027..417ede34c54af3a 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -135,6 +135,14 @@ set PYTHONPATH=$(PySourcePath)Lib "$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> </Target> + <Target Name="CopyFreethreadedBinary" AfterTargets="AfterBuild" + Condition="$(DisableGil) == 'true' and $(Configuration) != 'PGInstrument'"> + <Message Text="Duplicating $(TargetPath) to $(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe for free-threaded compatibility" /> + <Copy SourceFiles="$(TargetPath)" + DestinationFiles="$(OutDir)\$(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe" + SkipUnchangedFiles="true" + UseHardLinksIfPossible="true" /> + </Target> <Target Name="GeneratePythonBat" AfterTargets="AfterBuild"> <PropertyGroup> <_Content>@rem This script invokes the most recently built Python with all arguments diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index c6a5b8ce90a0d9b..244cdf622ad915c 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -115,4 +115,12 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> + <Target Name="CopyFreethreadedBinary" AfterTargets="AfterBuild" + Condition="$(DisableGil) == 'true' and $(Configuration) != 'PGInstrument'"> + <Message Text="Duplicating $(TargetPath) to $(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe for free-threaded compatibility" /> + <Copy SourceFiles="$(TargetPath)" + DestinationFiles="$(OutDir)\$(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe" + SkipUnchangedFiles="true" + UseHardLinksIfPossible="true" /> + </Target> </Project> \ No newline at end of file diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index c291b7f86325f2f..ea8adf21c279a68 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -168,8 +168,9 @@ xxlimited builds an example module that makes use of the PEP 384 Stable ABI, see Modules\xxlimited.c xxlimited_35 - ditto for testing the Python 3.5 stable ABI, see - Modules\xxlimited_35.c +xxlimited_3_13 + ditto for testing older Limited API, see + Modules\xxlimited_*.c The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to @@ -241,12 +242,12 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.53.1, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter - Wraps version 8.6.15 of the Tk windowing system, which is downloaded + Wraps version 9.0.3 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index a1da1155b881fd7..28e8c0db4d1eafd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Import Project="pyproject.props" Condition="$(__PyProject_Props_Imported) != 'true'" /> <PropertyGroup> - <TclVersion Condition="$(TclVersion) == ''">8.6.15.0</TclVersion> + <TclVersion Condition="$(TclVersion) == ''">9.0.3.0</TclVersion> <TkVersion Condition="$(TkVersion) == ''">$(TclVersion)</TkVersion> <TclMajorVersion>$([System.Version]::Parse($(TclVersion)).Major)</TclMajorVersion> <TclMinorVersion>$([System.Version]::Parse($(TclVersion)).Minor)</TclMinorVersion> @@ -12,7 +12,9 @@ <TkMinorVersion>$([System.Version]::Parse($(TkVersion)).Minor)</TkMinorVersion> <TkPatchLevel>$([System.Version]::Parse($(TkVersion)).Build)</TkPatchLevel> <TkRevision>$([System.Version]::Parse($(TkVersion)).Revision)</TkRevision> - <tclDir Condition="$(tclDir) == ''">$(ExternalsDir)tcl-core-$(TclVersion)\</tclDir> + <!-- Back compat hack. If you're here wondering about -core vs not, do us both a favor and just override the whole tclDir variable --> + <tclDir Condition="$(tclDir) == '' and $(TclMajorVersion) == '8'">$(ExternalsDir)tcl-core-$(TclVersion)\</tclDir> + <tclDir Condition="$(tclDir) == ''">$(ExternalsDir)tcl-$(TclVersion)\</tclDir> <tkDir Condition="$(tkDir) == ''">$(ExternalsDir)tk-$(TkVersion)\</tkDir> <tcltkDir Condition="$(tcltkDir) == ''">$(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\</tcltkDir> <tcltkSuffix Condition="'$(TclMajorVersion)' == '8'">t</tcltkSuffix> diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index abaf3a979af2681..a2e8ffa82b10eb7 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -89,10 +89,13 @@ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> <Import Project="pyproject.props" /> </ImportGroup> - <PropertyGroup Label="UserMacros" /> + <PropertyGroup Label="UserMacros"> + <ExeName>$(PyExeName)$(PyDebugExt).exe</ExeName> + <ExeName Condition="$(DisableGil) == 'true'">$(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe</ExeName> + </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>EXENAME=L"$(ExeName)";_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <ResourceCompile> diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index c58280deb8abeb3..f2aaf83fe2b3785 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -89,10 +89,13 @@ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> <Import Project="pyproject.props" /> </ImportGroup> - <PropertyGroup Label="UserMacros" /> + <PropertyGroup Label="UserMacros"> + <ExeName>$(PyWExeName)$(PyDebugExt).exe</ExeName> + <ExeName Condition="$(DisableGil) == 'true'">$(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe</ExeName> + </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>EXENAME=L"$(PyWExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>EXENAME=L"$(ExeName)";_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <ResourceCompile> diff --git a/PCbuild/xxlimited_3_13.vcxproj b/PCbuild/xxlimited_3_13.vcxproj new file mode 100644 index 000000000000000..7a9760fd43121ef --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj @@ -0,0 +1,111 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|ARM"> + <Configuration>Debug</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|ARM64"> + <Configuration>Debug</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM64"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|Win32"> + <Configuration>PGInstrument</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|x64"> + <Configuration>PGInstrument</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM64"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|Win32"> + <Configuration>PGUpdate</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|x64"> + <Configuration>PGUpdate</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM"> + <Configuration>Release</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM64"> + <Configuration>Release</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{fb868ea7-f93a-4d9b-be78-ca4e9ba14fff}</ProjectGuid> + <RootNamespace>xxlimited_3_13</RootNamespace> + <Keyword>Win32Proj</Keyword> + </PropertyGroup> + <Import Project="python.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Label="Configuration"> + <ConfigurationType>DynamicLibrary</ConfigurationType> + <CharacterSet>NotSet</CharacterSet> + <SupportPGO>false</SupportPGO> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <PropertyGroup> + <TargetExt>$(PyStdlibPydExt)</TargetExt> + </PropertyGroup> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="pyproject.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> + </PropertyGroup> + <ItemDefinitionGroup> + <Link> + <AdditionalDependencies>wsock32.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="..\Modules\xxlimited_3_13.c" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="python3dll.vcxproj"> + <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> + </ProjectReference> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project> diff --git a/PCbuild/xxlimited_3_13.vcxproj.filters b/PCbuild/xxlimited_3_13.vcxproj.filters new file mode 100644 index 000000000000000..3dfb7800edc4419 --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj.filters @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{5be27194-6530-452d-8d86-3767b991fa83}</UniqueIdentifier> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="..\Modules\xxlimited_3_13.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> +</Project> diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 3663dc3eb7f9f69..5cf9b4d768c3ebb 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void) struct tok_state *tok = (struct tok_state *)PyMem_Calloc( 1, sizeof(struct tok_state)); - if (tok == NULL) + if (tok == NULL) { + PyErr_NoMemory(); return NULL; + } + tok->buf = tok->cur = tok->inp = NULL; tok->fp_interactive = 0; tok->interactive_src_start = NULL; diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 8c836a3f7258296..a11702557a07af3 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -378,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 9542969ad3127b9..c69e66d0ab9b7a8 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -193,6 +193,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) char* result = (char *)PyMem_Malloc(len + 1); if (!result) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } memcpy(result, s, len); @@ -221,6 +222,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf buf = PyMem_Malloc(needed_length); if (buf == NULL) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } for (current = buf; *s; s++, current++) { diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 0f7769aeb8fd570..917f7b40cfbbfed 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index d2546cf76c206b0..5c41aaca6ebf0b4 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -220,7 +220,7 @@ def unpack_deps(host, prefix_dir, cache_dir): "bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.5.6-0", - "sqlite-3.50.4-0", + "sqlite-3.53.1-0", "xz-5.4.6-1", "zstd-1.5.7-2" ]: diff --git a/Platforms/Apple/testbed/__main__.py b/Platforms/Apple/testbed/__main__.py index 0dd77ab8b827974..b3eed38571d9708 100644 --- a/Platforms/Apple/testbed/__main__.py +++ b/Platforms/Apple/testbed/__main__.py @@ -21,7 +21,7 @@ LOG_PREFIX_REGEX = re.compile( r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID + r"\s+iOSTestbed\[\d+:\w+\] " # Process/thread ID ) diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 76d2853163baa9e..c1a91a9c833b8e8 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -222,10 +222,8 @@ def wasi_sdk(context): if wasi_sdk_path := context.wasi_sdk_path: if not wasi_sdk_path.exists(): raise ValueError( - "WASI SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" + "WASI SDK not found at " + f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" ) return wasi_sdk_path @@ -237,7 +235,8 @@ def wasi_sdk(context): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) if not wasi_sdk_path.exists(): raise ValueError( - f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " + "(via $WASI_SDK_PATH)" ) else: opt_path = pathlib.Path("/opt") @@ -272,6 +271,14 @@ def wasi_sdk(context): f" Found WASI SDK {major_version}, " f"but WASI SDK {wasi_sdk_version} is the supported version", ) + elif not wasi_sdk_path: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and install in " + f"{os.fsdecode(opt_path)!r} or specify the SDK via " + "$WASI_SDK_PATH or --wasi-sdk" + ) # Cache the result. context.wasi_sdk_path = wasi_sdk_path diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py index c1eac8005474fda..c2fb1c4c36e6087 100644 --- a/Platforms/emscripten/__main__.py +++ b/Platforms/emscripten/__main__.py @@ -650,7 +650,7 @@ def add_cross_build_dir_option(subcommand): help=( "Path to the cross-build directory " f"(default: {DEFAULT_CROSS_BUILD_DIR}). " - "Can also be set with the CROSS_BUILD_DIR environment variable.", + "Can also be set with the CROSS_BUILD_DIR environment variable." ), ) @@ -743,7 +743,7 @@ def main(): nargs=argparse.REMAINDER, help=( "Arguments to pass to the emscripten Python " - "(use '--' to separate from run options)", + "(use '--' to separate from run options)" ), ) add_cross_build_dir_option(run) diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index ba2dc8f4a482bfa..401e9396ddbb009 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -1,7 +1,7 @@ # Any data that can vary between Python versions is to be kept in this file. # This allows for blanket copying of the Emscripten build code between supported # Python versions. -emscripten-version = "4.0.12" +emscripten-version = "4.0.19" node-version = "24" test-args = [ "-m", "test", diff --git a/Platforms/emscripten/streams.mjs b/Platforms/emscripten/streams.mjs index 76ad79f9247f4cf..1b121d48d4e76c6 100644 --- a/Platforms/emscripten/streams.mjs +++ b/Platforms/emscripten/streams.mjs @@ -112,7 +112,7 @@ const prepareBuffer = (buffer, offset, length) => const TTY_OPS = { ioctl_tiocgwinsz(tty) { - return tty.devops.ioctl_tiocgwinsz?.(); + return tty.devops.ioctl_tiocgwinsz?.() ?? [24, 80]; }, }; @@ -188,6 +188,10 @@ class NodeReader { fsync() { nodeFsync(this.nodeStream.fd); } + + ioctl_tiocgwinsz() { + return [this.nodeStream.rows ?? 24, this.nodeStream.columns ?? 80]; + } } class NodeWriter { diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 93adae3d99038f8..9f3ca909ab7d852 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,3 +1,5 @@ +#include "asm_trampoline_aarch64.h" + .text #if defined(__APPLE__) .globl __Py_trampoline_func_start @@ -29,10 +31,12 @@ _Py_trampoline_func_start: #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) // ARM64 little endian, 64bit ABI // generate with aarch64-linux-gnu-gcc 12.1 + SIGN_LR stp x29, x30, [sp, -16]! mov x29, sp blr x3 ldp x29, x30, [sp], 16 + VERIFY_LR ret #endif #ifdef __riscv diff --git a/Python/asm_trampoline_aarch64.h b/Python/asm_trampoline_aarch64.h new file mode 100644 index 000000000000000..bc83aa460b6860d --- /dev/null +++ b/Python/asm_trampoline_aarch64.h @@ -0,0 +1,56 @@ +#ifndef ASM_TRAMPOLINE_AARCH_64_H_ +#define ASM_TRAMPOLINE_AARCH_64_H_ + +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif + +#endif diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 35b30a243318cc3..d5129bf6a5a6bc0 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -252,7 +252,6 @@ PyDoc_STRVAR(build_class_doc, Internal helper function used by the class statement."); /*[clinic input] -@permit_long_docstring_body __import__ as builtin___import__ name: object @@ -273,15 +272,16 @@ should be a list of names to emulate ``from name import ...``, or an empty list to emulate ``import name``. When importing a module from a package, note that __import__('A.B', ...) returns package A when fromlist is empty, but its submodule B when -fromlist is not empty. The level argument is used to determine whether to -perform absolute or relative imports: 0 is absolute, while a positive number -is the number of parent directories to search relative to the current module. +fromlist is not empty. The level argument is used to determine whether +to perform absolute or relative imports: 0 is absolute, while a positive +number is the number of parent directories to search relative to the +current module. [clinic start generated code]*/ static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=01a3283590eae93a]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=e3096a230383f72d]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -299,15 +299,15 @@ __lazy_import__ as builtin___lazy_import__ Lazily imports a module. -Returns either the module to be imported or a imp.lazy_module object which -indicates the module to be lazily imported. +Returns either the module to be imported or a imp.lazy_module object +which indicates the module to be lazily imported. [clinic start generated code]*/ static PyObject * builtin___lazy_import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=300f1771094b9e8c input=9394874f340b2948]*/ +/*[clinic end generated code: output=300f1771094b9e8c input=9c85cccd6a885b9b]*/ { PyObject *builtins; PyThreadState *tstate = PyThreadState_GET(); @@ -696,8 +696,9 @@ PyDoc_STRVAR(filter_doc, "filter(function, iterable, /)\n\ --\n\ \n\ -Return an iterator yielding those items of iterable for which function(item)\n\ -is true. If function is None, return the items that are true."); +Return an iterator yielding those items of iterable for which\n\ +function(item) is true. If function is None, return the items that\n\ +are true."); PyTypeObject PyFilter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -770,6 +771,7 @@ builtin_format_impl(PyObject *module, PyObject *value, PyObject *format_spec) } /*[clinic input] +@permit_long_summary chr as builtin_chr i: object @@ -780,7 +782,7 @@ Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. static PyObject * builtin_chr(PyObject *module, PyObject *i) -/*[clinic end generated code: output=d34f25b8035a9b10 input=f919867f0ba2f496]*/ +/*[clinic end generated code: output=d34f25b8035a9b10 input=a9b255f2d2e503f0]*/ { int overflow; long v = PyLong_AsLongAndOverflow(i, &overflow); @@ -804,6 +806,7 @@ builtin_chr(PyObject *module, PyObject *i) /*[clinic input] +@permit_long_summary compile as builtin_compile source: object @@ -818,23 +821,24 @@ compile as builtin_compile Compile source into a code object that can be executed by exec() or eval(). -The source code may represent a Python module, statement or expression. +The source code may represent a Python module, statement or +expression. The filename will be used for run-time error messages. The mode must be 'exec' to compile a module, 'single' to compile a single (interactive) statement, or 'eval' to compile an expression. -The flags argument, if present, controls which future statements influence -the compilation of the code. +The flags argument, if present, controls which future statements +influence the compilation of the code. The dont_inherit argument, if true, stops the compilation inheriting the effects of any future statements in effect in the code calling -compile; if absent or false these statements do influence the compilation, -in addition to any features explicitly specified. +compile; if absent or false these statements do influence the +compilation, in addition to any features explicitly specified. [clinic start generated code]*/ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, PyObject *modname, int feature_version) -/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ +/*[clinic end generated code: output=9a0dce1945917a86 input=444c4fe466a97279]*/ { PyObject *source_copy; const char *str; @@ -982,10 +986,10 @@ PyDoc_STRVAR(dir_doc, "dir([object]) -> list of strings\n" "\n" "If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" +"Else, return an alphabetized list of names comprising (some of) the\n" +"attributes of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used;\n" +"otherwise the default dir() logic is used and returns:\n" " for a module object: the module's attributes.\n" " for a class object: its attributes, and recursively the attributes\n" " of its bases.\n" @@ -1326,9 +1330,11 @@ builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(getattr_doc, "getattr(object, name[, default]) -> value\n\ \n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); +Get a named attribute from an object.\n\ +\n\ +getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute\n\ +doesn't exist; without it, an exception is raised in that case."); /*[clinic input] @@ -1336,13 +1342,13 @@ globals as builtin_globals Return the dictionary containing the current scope's global variables. -NOTE: Updates to this dictionary *will* affect name lookups in the current -global scope and vice-versa. +NOTE: Updates to this dictionary *will* affect name lookups in the +current global scope and vice-versa. [clinic start generated code]*/ static PyObject * builtin_globals_impl(PyObject *module) -/*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ +/*[clinic end generated code: output=e5dd1527067b94d2 input=6d725a9b48d1eaeb]*/ { PyObject *globals; if (_PyEval_GetFrame() != NULL) { @@ -1695,8 +1701,8 @@ PyDoc_STRVAR(map_doc, Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError."); +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError."); PyTypeObject PyMap_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -1783,8 +1789,8 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(next_doc, "next(iterator[, default])\n\ \n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); +Return the next item from the iterator. If default is given and the\n\ +iterator is exhausted, it is returned instead of raising StopIteration."); /*[clinic input] @@ -1907,7 +1913,8 @@ iter(callable, sentinel) -> iterator\n\ \n\ Get an iterator from an object. In the first form, the argument must\n\ supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); +In the second form, the callable is called until it returns the\n\ +sentinel."); /*[clinic input] @@ -2001,14 +2008,15 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +NOTE: Whether or not updates to this dictionary will affect name +lookups in the local scope and vice-versa is *implementation +dependent* and not covered by any backwards compatibility +guarantees. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=989cc75c22167c42]*/ { PyObject *locals; if (_PyEval_GetFrame() != NULL) { @@ -2260,6 +2268,7 @@ builtin_ord(PyObject *module, PyObject *c) /*[clinic input] +@permit_long_summary pow as builtin_pow base: object @@ -2268,14 +2277,14 @@ pow as builtin_pow Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments -Some types, such as ints, are able to use a more efficient algorithm when -invoked using the three argument form. +Some types, such as ints, are able to use a more efficient algorithm +when invoked using the three argument form. [clinic start generated code]*/ static PyObject * builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/ +/*[clinic end generated code: output=3ca1538221bbf15f input=0cd5c3ecc8003aec]*/ { return PyNumber_Power(base, exp, mod); } @@ -2396,13 +2405,14 @@ Read a string from standard input. The trailing newline is stripped. The prompt string, if given, is printed to standard output without a trailing newline before reading input. -If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. +If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise +EOFError. On *nix systems, readline is used if available. [clinic start generated code]*/ static PyObject * builtin_input_impl(PyObject *module, PyObject *prompt) -/*[clinic end generated code: output=83db5a191e7a0d60 input=159c46d4ae40977e]*/ +/*[clinic end generated code: output=83db5a191e7a0d60 input=ebb939c954639427]*/ { PyObject *fin = NULL; PyObject *fout = NULL; @@ -2670,13 +2680,14 @@ round as builtin_round Round a number to a given precision in decimal digits. -The return value is an integer if ndigits is omitted or None. Otherwise -the return value has the same type as the number. ndigits may be negative. +The return value is an integer if ndigits is omitted or None. +Otherwise the return value has the same type as the number. ndigits +may be negative. [clinic start generated code]*/ static PyObject * builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) -/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/ +/*[clinic end generated code: output=ff0d9dd176c02ede input=bdcb7c67bf4a4320]*/ { PyObject *result; if (ndigits == Py_None) { @@ -2708,8 +2719,8 @@ sorted as builtin_sorted Return a new list containing all items from the iterable in ascending order. -A custom key function can be supplied to customize the sort order, and the -reverse flag can be set to request the result in descending order. +A custom key function can be supplied to customize the sort order, and +the reverse flag can be set to request the result in descending order. [end disabled clinic input]*/ PyDoc_STRVAR(builtin_sorted__doc__, @@ -2843,6 +2854,7 @@ cs_to_double(CompensatedSum total) } /*[clinic input] +@permit_long_summary sum as builtin_sum iterable: object @@ -2852,13 +2864,13 @@ sum as builtin_sum Return the sum of a 'start' value (default: 0) plus an iterable of numbers When the iterable is empty, return the start value. -This function is intended specifically for use with numeric values and may -reject non-numeric types. +This function is intended specifically for use with numeric values and +may reject non-numeric types. [clinic start generated code]*/ static PyObject * builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) -/*[clinic end generated code: output=df758cec7d1d302f input=162b50765250d222]*/ +/*[clinic end generated code: output=df758cec7d1d302f input=d464d57815196b73]*/ { PyObject *result = start; PyObject *temp, *item, *iter; @@ -3094,6 +3106,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) /*[clinic input] +@permit_long_summary isinstance as builtin_isinstance obj: object @@ -3102,15 +3115,15 @@ isinstance as builtin_isinstance Return whether an object is an instance of a class or of a subclass thereof. -A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B) -or ...`` etc. +A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``isinstance(x, A) or +isinstance(x, B) or ...`` etc. [clinic start generated code]*/ static PyObject * builtin_isinstance_impl(PyObject *module, PyObject *obj, PyObject *class_or_tuple) -/*[clinic end generated code: output=6faf01472c13b003 input=ffa743db1daf7549]*/ +/*[clinic end generated code: output=6faf01472c13b003 input=5d74d547df498f38]*/ { int retval; @@ -3130,15 +3143,15 @@ issubclass as builtin_issubclass Return whether 'cls' is derived from another class or is the same class. -A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B) -or ...``. +A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``issubclass(x, A) or +issubclass(x, B) or ...``. [clinic start generated code]*/ static PyObject * builtin_issubclass_impl(PyObject *module, PyObject *cls, PyObject *class_or_tuple) -/*[clinic end generated code: output=358412410cd7a250 input=a24b9f3d58c370d6]*/ +/*[clinic end generated code: output=358412410cd7a250 input=a91ce96345a6705d]*/ { int retval; @@ -3368,13 +3381,13 @@ PyDoc_STRVAR(zip_doc, "zip(*iterables, strict=False)\n\ --\n\ \n\ -The zip object yields n-length tuples, where n is the number of iterables\n\ -passed as positional arguments to zip(). The i-th element in every tuple\n\ -comes from the i-th iterable argument to zip(). This continues until the\n\ -shortest argument is exhausted.\n\ +The zip object yields n-length tuples, where n is the number of\n\ +iterables passed as positional arguments to zip(). The i-th element\n\ +in every tuple comes from the i-th iterable argument to zip(). This\n\ +continues until the shortest argument is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError.\n\ +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError.\n\ \n\ >>> list(zip('abcdefg', range(3), range(4)))\n\ [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]"); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3bd489122da9d42..f7487c7136962f1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1867,8 +1867,9 @@ dummy_func( assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/ceval.c b/Python/ceval.c index 060e948e6b01c9f..5661200e74d0a55 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3059,6 +3059,13 @@ check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, return res; } +static int +is_lazy_import_module_level(void) +{ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + return frame != NULL && frame->f_globals == frame->f_locals; +} + PyObject * _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, @@ -3067,21 +3074,24 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *res = NULL; // Check if global policy overrides the local syntax switch (PyImport_GetLazyImportsMode()) { - case PyImport_LAZY_NONE: - lazy = 0; - break; case PyImport_LAZY_ALL: - lazy = 1; + if (!lazy) { + lazy = is_lazy_import_module_level(); + } break; case PyImport_LAZY_NORMAL: break; } - if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { + if (!lazy) { // See if __lazy_modules__ forces this to be lazy. - lazy = check_lazy_import_compatibility(tstate, globals, name, level); - if (lazy < 0) { - return NULL; + // __lazy_modules__ only applies at module level; exec() inside + // functions or classes should remain eager. + if (is_lazy_import_module_level()) { + lazy = check_lazy_import_compatibility(tstate, globals, name, level); + if (lazy < 0) { + return NULL; + } } } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index e6b845cd375d73a..4a38e0df61708c0 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -25,9 +25,10 @@ PyDoc_STRVAR(builtin___import____doc__, "empty list to emulate ``import name``.\n" "When importing a module from a package, note that __import__(\'A.B\', ...)\n" "returns package A when fromlist is empty, but its submodule B when\n" -"fromlist is not empty. The level argument is used to determine whether to\n" -"perform absolute or relative imports: 0 is absolute, while a positive number\n" -"is the number of parent directories to search relative to the current module."); +"fromlist is not empty. The level argument is used to determine whether\n" +"to perform absolute or relative imports: 0 is absolute, while a positive\n" +"number is the number of parent directories to search relative to the\n" +"current module."); #define BUILTIN___IMPORT___METHODDEF \ {"__import__", _PyCFunction_CAST(builtin___import__), METH_FASTCALL|METH_KEYWORDS, builtin___import____doc__}, @@ -120,8 +121,8 @@ PyDoc_STRVAR(builtin___lazy_import____doc__, "\n" "Lazily imports a module.\n" "\n" -"Returns either the module to be imported or a imp.lazy_module object which\n" -"indicates the module to be lazily imported."); +"Returns either the module to be imported or a imp.lazy_module object\n" +"which indicates the module to be lazily imported."); #define BUILTIN___LAZY_IMPORT___METHODDEF \ {"__lazy_import__", _PyCFunction_CAST(builtin___lazy_import__), METH_FASTCALL|METH_KEYWORDS, builtin___lazy_import____doc__}, @@ -339,16 +340,17 @@ PyDoc_STRVAR(builtin_compile__doc__, "\n" "Compile source into a code object that can be executed by exec() or eval().\n" "\n" -"The source code may represent a Python module, statement or expression.\n" +"The source code may represent a Python module, statement or\n" +"expression.\n" "The filename will be used for run-time error messages.\n" "The mode must be \'exec\' to compile a module, \'single\' to compile a\n" "single (interactive) statement, or \'eval\' to compile an expression.\n" -"The flags argument, if present, controls which future statements influence\n" -"the compilation of the code.\n" +"The flags argument, if present, controls which future statements\n" +"influence the compilation of the code.\n" "The dont_inherit argument, if true, stops the compilation inheriting\n" "the effects of any future statements in effect in the code calling\n" -"compile; if absent or false these statements do influence the compilation,\n" -"in addition to any features explicitly specified."); +"compile; if absent or false these statements do influence the\n" +"compilation, in addition to any features explicitly specified."); #define BUILTIN_COMPILE_METHODDEF \ {"compile", _PyCFunction_CAST(builtin_compile), METH_FASTCALL|METH_KEYWORDS, builtin_compile__doc__}, @@ -683,8 +685,8 @@ PyDoc_STRVAR(builtin_globals__doc__, "\n" "Return the dictionary containing the current scope\'s global variables.\n" "\n" -"NOTE: Updates to this dictionary *will* affect name lookups in the current\n" -"global scope and vice-versa."); +"NOTE: Updates to this dictionary *will* affect name lookups in the\n" +"current global scope and vice-versa."); #define BUILTIN_GLOBALS_METHODDEF \ {"globals", (PyCFunction)builtin_globals, METH_NOARGS, builtin_globals__doc__}, @@ -910,9 +912,10 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"NOTE: Whether or not updates to this dictionary will affect name\n" +"lookups in the local scope and vice-versa is *implementation\n" +"dependent* and not covered by any backwards compatibility\n" +"guarantees."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -959,8 +962,8 @@ PyDoc_STRVAR(builtin_pow__doc__, "\n" "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n" "\n" -"Some types, such as ints, are able to use a more efficient algorithm when\n" -"invoked using the three argument form."); +"Some types, such as ints, are able to use a more efficient algorithm\n" +"when invoked using the three argument form."); #define BUILTIN_POW_METHODDEF \ {"pow", _PyCFunction_CAST(builtin_pow), METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__}, @@ -1136,7 +1139,8 @@ PyDoc_STRVAR(builtin_input__doc__, "The prompt string, if given, is printed to standard output without a\n" "trailing newline before reading input.\n" "\n" -"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\n" +"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise\n" +"EOFError.\n" "On *nix systems, readline is used if available."); #define BUILTIN_INPUT_METHODDEF \ @@ -1182,8 +1186,9 @@ PyDoc_STRVAR(builtin_round__doc__, "\n" "Round a number to a given precision in decimal digits.\n" "\n" -"The return value is an integer if ndigits is omitted or None. Otherwise\n" -"the return value has the same type as the number. ndigits may be negative."); +"The return value is an integer if ndigits is omitted or None.\n" +"Otherwise the return value has the same type as the number. ndigits\n" +"may be negative."); #define BUILTIN_ROUND_METHODDEF \ {"round", _PyCFunction_CAST(builtin_round), METH_FASTCALL|METH_KEYWORDS, builtin_round__doc__}, @@ -1251,8 +1256,8 @@ PyDoc_STRVAR(builtin_sum__doc__, "Return the sum of a \'start\' value (default: 0) plus an iterable of numbers\n" "\n" "When the iterable is empty, return the start value.\n" -"This function is intended specifically for use with numeric values and may\n" -"reject non-numeric types."); +"This function is intended specifically for use with numeric values and\n" +"may reject non-numeric types."); #define BUILTIN_SUM_METHODDEF \ {"sum", _PyCFunction_CAST(builtin_sum), METH_FASTCALL|METH_KEYWORDS, builtin_sum__doc__}, @@ -1319,9 +1324,9 @@ PyDoc_STRVAR(builtin_isinstance__doc__, "\n" "Return whether an object is an instance of a class or of a subclass thereof.\n" "\n" -"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\n" -"or ...`` etc."); +"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``isinstance(x, A) or\n" +"isinstance(x, B) or ...`` etc."); #define BUILTIN_ISINSTANCE_METHODDEF \ {"isinstance", _PyCFunction_CAST(builtin_isinstance), METH_FASTCALL, builtin_isinstance__doc__}, @@ -1354,9 +1359,9 @@ PyDoc_STRVAR(builtin_issubclass__doc__, "\n" "Return whether \'cls\' is derived from another class or is the same class.\n" "\n" -"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\n" -"or ...``."); +"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``issubclass(x, A) or\n" +"issubclass(x, B) or ...``."); #define BUILTIN_ISSUBCLASS_METHODDEF \ {"issubclass", _PyCFunction_CAST(builtin_issubclass), METH_FASTCALL, builtin_issubclass__doc__}, @@ -1382,4 +1387,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f1fc836a63d89826 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=84efa9c5cc737ce5 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 5ed74e6e6ddb6bf..ece7341d65d5fb6 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -10,8 +10,8 @@ PyDoc_STRVAR(_contextvars_Context_get__doc__, "\n" "Return the value for `key` if `key` has the value in the context object.\n" "\n" -"If `key` does not exist, return `default`. If `default` is not given,\n" -"return None."); +"If `key` does not exist, return `default`. If `default` is not\n" +"given, return None."); #define _CONTEXTVARS_CONTEXT_GET_METHODDEF \ {"get", _PyCFunction_CAST(_contextvars_Context_get), METH_FASTCALL, _contextvars_Context_get__doc__}, @@ -122,10 +122,12 @@ PyDoc_STRVAR(_contextvars_ContextVar_get__doc__, "\n" "Return a value for the context variable for the current context.\n" "\n" -"If there is no value for the variable in the current context, the method will:\n" -" * return the value of the default argument of the method, if provided; or\n" -" * return the default value for the context variable, if it was created\n" -" with one; or\n" +"If there is no value for the variable in the current context, the\n" +"method will:\n" +" * return the value of the default argument of the method, if\n" +" provided; or\n" +" * return the default value for the context variable, if it was\n" +" created with one; or\n" " * raise a LookupError."); #define _CONTEXTVARS_CONTEXTVAR_GET_METHODDEF \ @@ -160,10 +162,11 @@ PyDoc_STRVAR(_contextvars_ContextVar_set__doc__, "\n" "Call to set a new value for the context variable in the current context.\n" "\n" -"The required value argument is the new value for the context variable.\n" +"The required value argument is the new value for the context\n" +"variable.\n" "\n" -"Returns a Token object that can be used to restore the variable to its previous\n" -"value via the `ContextVar.reset()` method."); +"Returns a Token object that can be used to restore the variable to\n" +"its previous value via the `ContextVar.reset()` method."); #define _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF \ {"set", (PyCFunction)_contextvars_ContextVar_set, METH_O, _contextvars_ContextVar_set__doc__}, @@ -187,8 +190,8 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, "\n" "Reset the context variable.\n" "\n" -"The variable is reset to the value it had before the `ContextVar.set()` that\n" -"created the token was used."); +"The variable is reset to the value it had before the\n" +"`ContextVar.set()` that created the token was used."); #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, @@ -256,4 +259,4 @@ token_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3a04b2fddf24c3e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90ec3e4375804e9b input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index de62714ebddafa6..2e4e178b3084063 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -34,8 +34,9 @@ PyDoc_STRVAR(_imp_acquire_lock__doc__, "\n" "Acquires the interpreter\'s import lock for the current thread.\n" "\n" -"This lock should be used by import hooks to ensure thread-safety when importing\n" -"modules. On platforms without threads, this function does nothing."); +"This lock should be used by import hooks to ensure thread-safety when\n" +"importing modules. On platforms without threads, this function does\n" +"nothing."); #define _IMP_ACQUIRE_LOCK_METHODDEF \ {"acquire_lock", (PyCFunction)_imp_acquire_lock, METH_NOARGS, _imp_acquire_lock__doc__}, @@ -664,4 +665,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=5fa42f580441b3fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index 6c00b2b31b007fa..ec0d2eb8a2af543 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -195,8 +195,8 @@ PyDoc_STRVAR(marshal_dumps__doc__, " allow_code\n" " Allow to write code objects.\n" "\n" -"Raise a ValueError exception if value has (or contains an object that has) an\n" -"unsupported type."); +"Raise a ValueError exception if value has (or contains an object that\n" +"has) an unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, @@ -280,8 +280,8 @@ PyDoc_STRVAR(marshal_loads__doc__, " allow_code\n" " Allow to load code objects.\n" "\n" -"If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" -"bytes in the input are ignored."); +"If no valid value is found, raise EOFError, ValueError or TypeError.\n" +"Extra bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, @@ -351,4 +351,4 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -/*[clinic end generated code: output=3e4bfc070a3c78ac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a574570c3717f60e input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 86e942ec2b8afbc..067b0b0a1fb035a 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1395,7 +1395,8 @@ PyDoc_STRVAR(sys__stats_dump__doc__, "\n" "Dump stats to file, and clears the stats.\n" "\n" -"Return False if no statistics were not dumped because stats gathering was off."); +"Return False if no statistics were not dumped because stats gathering\n" +"was off."); #define SYS__STATS_DUMP_METHODDEF \ {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__}, @@ -1543,16 +1544,16 @@ PyDoc_STRVAR(sys_remote_exec__doc__, "Executes a file containing Python code in a given remote Python process.\n" "\n" "This function returns immediately, and the code will be executed by the\n" -"target process\'s main thread at the next available opportunity, similarly\n" -"to how signals are handled. There is no interface to determine when the\n" -"code has been executed. The caller is responsible for making sure that\n" -"the file still exists whenever the remote process tries to read it and that\n" -"it hasn\'t been overwritten.\n" +"target process\'s main thread at the next available opportunity,\n" +"similarly to how signals are handled. There is no interface to\n" +"determine when the code has been executed. The caller is responsible\n" +"for making sure that the file still exists whenever the remote process\n" +"tries to read it and that it hasn\'t been overwritten.\n" "\n" -"The remote process must be running a CPython interpreter of the same major\n" -"and minor version as the local process. If either the local or remote\n" -"interpreter is pre-release (alpha, beta, or release candidate) then the\n" -"local and remote interpreters must be the same exact version.\n" +"The remote process must be running a CPython interpreter of the same\n" +"major and minor version as the local process. If either the local or\n" +"remote interpreter is pre-release (alpha, beta, or release candidate)\n" +"then the local and remote interpreters must be the same exact version.\n" "\n" "Args:\n" " pid (int): The process ID of the target Python process.\n" @@ -1914,8 +1915,8 @@ PyDoc_STRVAR(sys_set_lazy_imports__doc__, "\n" "The mode parameter must be one of the following strings:\n" "- \"all\": All top-level imports become potentially lazy\n" -"- \"none\": All lazy imports are suppressed (even explicitly marked ones)\n" -"- \"normal\": Only explicitly marked imports (with \'lazy\' keyword) are lazy\n" +"- \"normal\": Only explicitly marked imports (with \'lazy\' keyword) are\n" +" lazy\n" "\n" "In addition to the mode, lazy imports can be controlled via the filter\n" "provided to sys.set_lazy_imports_filter"); @@ -1979,7 +1980,6 @@ PyDoc_STRVAR(sys_get_lazy_imports__doc__, "Gets the global lazy imports mode.\n" "\n" "Returns \"all\" if all top level imports are potentially lazy.\n" -"Returns \"none\" if all explicitly marked lazy imports are suppressed.\n" "Returns \"normal\" if only explicitly marked imports are lazy."); #define SYS_GET_LAZY_IMPORTS_METHODDEF \ @@ -2121,4 +2121,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=e8333fe10c01ae66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3ccdb73f733fc82c input=a9049054013a1b77]*/ diff --git a/Python/context.c b/Python/context.c index 62b582f271ffe57..593e6ef90037cfa 100644 --- a/Python/context.c +++ b/Python/context.c @@ -618,6 +618,7 @@ context_tp_contains(PyObject *op, PyObject *key) /*[clinic input] +@permit_long_summary _contextvars.Context.get key: object default: object = None @@ -625,14 +626,14 @@ _contextvars.Context.get Return the value for `key` if `key` has the value in the context object. -If `key` does not exist, return `default`. If `default` is not given, -return None. +If `key` does not exist, return `default`. If `default` is not +given, return None. [clinic start generated code]*/ static PyObject * _contextvars_Context_get_impl(PyContext *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=0c54aa7664268189 input=c8eeb81505023995]*/ +/*[clinic end generated code: output=0c54aa7664268189 input=d669a0d56fabb0a5]*/ { if (context_check_key_type(key)) { return NULL; @@ -1006,23 +1007,24 @@ contextvar_tp_repr(PyObject *op) /*[clinic input] -@permit_long_docstring_body _contextvars.ContextVar.get default: object = NULL / Return a value for the context variable for the current context. -If there is no value for the variable in the current context, the method will: - * return the value of the default argument of the method, if provided; or - * return the default value for the context variable, if it was created - with one; or +If there is no value for the variable in the current context, the +method will: + * return the value of the default argument of the method, if + provided; or + * return the default value for the context variable, if it was + created with one; or * raise a LookupError. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) -/*[clinic end generated code: output=0746bd0aa2ced7bf input=da66664d5d0af4ad]*/ +/*[clinic end generated code: output=0746bd0aa2ced7bf input=83814c6aef4a9fe3]*/ { PyObject *val; if (PyContextVar_Get((PyObject *)self, default_value, &val) < 0) { @@ -1038,41 +1040,41 @@ _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _contextvars.ContextVar.set value: object / Call to set a new value for the context variable in the current context. -The required value argument is the new value for the context variable. +The required value argument is the new value for the context +variable. -Returns a Token object that can be used to restore the variable to its previous -value via the `ContextVar.reset()` method. +Returns a Token object that can be used to restore the variable to +its previous value via the `ContextVar.reset()` method. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_set_impl(PyContextVar *self, PyObject *value) -/*[clinic end generated code: output=1b562d35cc79c806 input=73ebbbfc7c98f6cd]*/ +/*[clinic end generated code: output=1b562d35cc79c806 input=04ef8dcd810f5be6]*/ { return PyContextVar_Set((PyObject *)self, value); } /*[clinic input] -@permit_long_docstring_body _contextvars.ContextVar.reset token: object / Reset the context variable. -The variable is reset to the value it had before the `ContextVar.set()` that -created the token was used. +The variable is reset to the value it had before the +`ContextVar.set()` that created the token was used. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_reset_impl(PyContextVar *self, PyObject *token) -/*[clinic end generated code: output=3205d2bdff568521 input=b8bc514a9245242a]*/ +/*[clinic end generated code: output=3205d2bdff568521 input=dd33cfcb18c00e37]*/ { if (!PyContextToken_CheckExact(token)) { PyErr_Format(PyExc_TypeError, @@ -1098,7 +1100,8 @@ static PyMethodDef PyContextVar_methods[] = { _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("ContextVars are generic over the type of their contained values")}, {NULL, NULL} }; @@ -1264,7 +1267,8 @@ token_exit_impl(PyContextToken *self, PyObject *type, PyObject *val, static PyMethodDef PyContextTokenType_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Tokens are generic over the same type as the ContextVar which created them.")}, TOKEN_ENTER_METHODDEF TOKEN_EXIT_METHODDEF {NULL} diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b6a2821db3007ef..efa61d7de74e88c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -9346,8 +9346,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/gc.c b/Python/gc.c index 54ac1b089e503d0..201c621bcc3cb9b 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1876,6 +1876,8 @@ _PyGC_Fini(PyInterpreterState *interp) GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); + PyMem_RawFree(gcstate->generation_stats); + gcstate->generation_stats = NULL; /* Prevent a subtle bug that affects sub-interpreters that use basic * single-phase init extensions (m_size == -1). Those extensions cause objects diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2623105656c90c3..53e09a8f4523c7c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7945,8 +7945,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif @@ -13053,8 +13054,9 @@ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE && defined(Py_DEBUG) if (!PyStackRef_IsNone(frame->f_executable)) { - int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); - int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; assert(opcode == SEND || opcode == FOR_ITER); } #endif diff --git a/Python/hamt.c b/Python/hamt.c index e4719e71a5259a5..95998ae5062ac7e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -702,6 +702,7 @@ hamt_node_bitmap_assoc(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *ret = hamt_node_bitmap_clone(self); if (ret == NULL) { + Py_DECREF(sub_node); return NULL; } Py_SETREF(ret->b_array[val_idx], (PyObject*)sub_node); @@ -994,6 +995,7 @@ hamt_node_bitmap_without(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *clone = hamt_node_bitmap_clone(self); if (clone == NULL) { + Py_DECREF(sub_node); return W_ERROR; } diff --git a/Python/import.c b/Python/import.c index 60a5ee6e770f598..63021208a23d3b7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -94,6 +94,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define LAZY_MODULES(interp) \ (interp)->imports.lazy_modules +#define LAZY_PENDING_SUBMODULES(interp) \ + (interp)->imports.lazy_pending_submodules #define IMPORTLIB(interp) \ (interp)->imports.importlib #define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ @@ -271,8 +273,11 @@ import_get_module(PyThreadState *tstate, PyObject *name) PyObject * _PyImport_InitLazyModules(PyInterpreterState *interp) { - assert(LAZY_MODULES(interp) == NULL); - LAZY_MODULES(interp) = PyDict_New(); + assert(LAZY_MODULES(interp) == NULL && + LAZY_PENDING_SUBMODULES(interp) == NULL); + + LAZY_PENDING_SUBMODULES(interp) = PyDict_New(); + LAZY_MODULES(interp) = PySet_New(0); return LAZY_MODULES(interp); } @@ -280,6 +285,7 @@ void _PyImport_ClearLazyModules(PyInterpreterState *interp) { Py_CLEAR(LAZY_MODULES(interp)); + Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } static int @@ -3162,7 +3168,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (nameobj == NULL || nameobj == Py_None) { return FROZEN_BAD_NAME; } - const char *name = PyUnicode_AsUTF8(nameobj); + const char *name = _PyUnicode_AsUTF8NoNUL(nameobj); if (name == NULL) { // Note that this function previously used // _PyUnicode_EqualToASCIIString(). We clear the error here @@ -3928,7 +3934,6 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) return NULL; } else if (PySet_Add(importing, lazy_import) < 0) { - _PyImport_ReleaseLock(interp); goto error; } @@ -4049,7 +4054,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Create a cause exception showing where the lazy import was declared. PyObject *msg = PyUnicode_FromFormat( - "deferred import of '%U' raised an exception during resolution", + "lazy import of '%U' raised an exception during resolution", import_name ); Py_DECREF(import_name); // Done with import_name. @@ -4326,20 +4331,10 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -static PyObject * -get_mod_dict(PyObject *module) -{ - if (PyModule_Check(module)) { - return Py_NewRef(_PyModule_GetDict(module)); - } - - return PyObject_GetAttr(module, &_Py_ID(__dict__)); -} - // ensure we have the set for the parent module name in sys.lazy_modules. // Returns a new reference. static PyObject * -ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) +ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) { PyObject *lazy_submodules; Py_BEGIN_CRITICAL_SECTION(lazy_modules); @@ -4358,27 +4353,28 @@ ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// Records all parent-child relationships in lazy_pending_submodules +// for a lazily imported module name. When a parent module's attribute +// is accessed, _Py_module_getattro_impl will check lazy_pending_submodules +// and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, - PyObject *builtins) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { int ret = -1; PyObject *parent = NULL; PyObject *child = NULL; - PyObject *parent_module = NULL; - PyObject *parent_dict = NULL; PyInterpreterState *interp = tstate->interp; - PyObject *lazy_modules = LAZY_MODULES(interp); - assert(lazy_modules != NULL); + PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); + assert(lazy_pending_submodules != NULL); Py_INCREF(name); while (true) { Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, name); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, name); if (lazy_submodules == NULL) { goto done; } @@ -4387,9 +4383,6 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } parent = PyUnicode_Substring(name, 0, dot); - // If `parent` is NULL then this has hit the end of the import, no - // more "parent.child" in the import name. The entire import will be - // resolved lazily. if (parent == NULL) { goto done; } @@ -4399,9 +4392,8 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } - // Record the child as being lazily imported from the parent. - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, parent); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, parent); if (lazy_submodules == NULL) { goto done; } @@ -4412,44 +4404,11 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, } Py_DECREF(lazy_submodules); - // Add the lazy import for the child to the parent. - Py_XSETREF(parent_module, PyImport_GetModule(parent)); - if (parent_module != NULL) { - Py_XSETREF(parent_dict, get_mod_dict(parent_module)); - if (parent_dict == NULL) { - goto done; - } - if (PyDict_CheckExact(parent_dict)) { - int contains = PyDict_Contains(parent_dict, child); - if (contains < 0) { - goto done; - } - if (!contains) { - PyObject *lazy_module_attr = _PyLazyImport_New( - tstate->current_frame, builtins, parent, child - ); - if (lazy_module_attr == NULL) { - goto done; - } - if (PyDict_SetItem(parent_dict, child, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - goto done; - } - Py_DECREF(lazy_module_attr); - } - } - ret = 0; - goto done; - } - Py_SETREF(name, parent); parent = NULL; } done: - Py_XDECREF(parent_dict); - Py_XDECREF(parent_module); Py_XDECREF(child); Py_XDECREF(parent); Py_XDECREF(name); @@ -4458,17 +4417,73 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from, PyObject *builtins) + PyObject *from) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { return -1; } - int res = register_lazy_on_parent(tstate, fromname, builtins); + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, fromname) < 0) { + Py_DECREF(fromname); + return -1; + } + + int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); + if (lazy_pending == NULL) { + return NULL; + } + + PyObject *pending_set; + int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); + if (rc <= 0) { + return NULL; + } + + int contains = PySet_Contains(pending_set, attr_name); + if (contains <= 0) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + if (full_name == NULL) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *mod = PyImport_ImportModuleLevelObject( + full_name, NULL, NULL, NULL, 0); + if (mod == NULL) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(mod); + + if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(pending_set); + + PyObject *submod = PyImport_GetModule(full_name); + Py_DECREF(full_name); + return submod; +} + PyObject * _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, @@ -4555,9 +4570,15 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, Py_DECREF(abs_name); return NULL; } + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, abs_name) < 0) { + goto error; + } + if (fromlist && PyUnicode_Check(fromlist)) { - if (register_from_lazy_on_parent(tstate, abs_name, fromlist, - builtins) < 0) { + if (register_from_lazy_on_parent(tstate, abs_name, fromlist) < 0) { goto error; } } @@ -4565,14 +4586,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i), - builtins) < 0) + PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name, builtins) < 0) { + else if (register_lazy_on_parent(tstate, abs_name) < 0) { goto error; } @@ -4791,6 +4811,7 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(IMPORTLIB(interp)); Py_CLEAR(IMPORT_FUNC(interp)); Py_CLEAR(LAZY_IMPORT_FUNC(interp)); + Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); @@ -5003,18 +5024,18 @@ _imp_lock_held_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body _imp.acquire_lock Acquires the interpreter's import lock for the current thread. -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. +This lock should be used by import hooks to ensure thread-safety when +importing modules. On platforms without threads, this function does +nothing. [clinic start generated code]*/ static PyObject * _imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=e1a4ef049d34e7dd]*/ +/*[clinic end generated code: output=1aff58cb0ee1b026 input=60e9c1b4ab471ead]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyImport_AcquireLock(interp); @@ -5172,6 +5193,7 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) } /*[clinic input] +@permit_long_summary _imp.find_frozen name: unicode @@ -5192,7 +5214,7 @@ The returned info (a 2-tuple): static PyObject * _imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata) -/*[clinic end generated code: output=8c1c3c7f925397a5 input=22a8847c201542fd]*/ +/*[clinic end generated code: output=8c1c3c7f925397a5 input=30a7a50da49eca97]*/ { struct frozen_info info; frozen_status status = find_frozen(name, &info); @@ -5379,6 +5401,7 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) } /*[clinic input] +@permit_long_summary _imp._override_multi_interp_extensions_check override: int @@ -5392,7 +5415,7 @@ _imp._override_multi_interp_extensions_check static PyObject * _imp__override_multi_interp_extensions_check_impl(PyObject *module, int override) -/*[clinic end generated code: output=3ff043af52bbf280 input=e086a2ea181f92ae]*/ +/*[clinic end generated code: output=3ff043af52bbf280 input=24f23f8510a7f6e7]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); if (_Py_IsMainInterpreter(interp)) { @@ -5580,46 +5603,6 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) return PyBytes_FromStringAndSize(hash.data, sizeof(hash.data)); } -static int -publish_lazy_imports_on_module(PyThreadState *tstate, - PyObject *lazy_submodules, - PyObject *name, - PyObject *module_dict) -{ - PyObject *builtins = _PyEval_GetBuiltins(tstate); - PyObject *attr_name; - Py_ssize_t pos = 0; - Py_hash_t hash; - - // Enumerate the set of lazy submodules which have been imported from the - // parent module. - while (_PySet_NextEntryRef(lazy_submodules, &pos, &attr_name, &hash)) { - if (_PyDict_Contains_KnownHash(module_dict, attr_name, hash)) { - Py_DECREF(attr_name); - continue; - } - // Create a new lazy module attr for the subpackage which was - // previously lazily imported. - PyObject *lazy_module_attr = _PyLazyImport_New(tstate->current_frame, builtins, - name, attr_name); - if (lazy_module_attr == NULL) { - Py_DECREF(attr_name); - return -1; - } - - // Publish on the module that was just imported. - if (PyDict_SetItem(module_dict, attr_name, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - return -1; - } - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - } - return 0; -} - /*[clinic input] _imp._set_lazy_attributes modobj: object @@ -5633,43 +5616,11 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) /*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *module_dict = NULL; - PyObject *ret = NULL; - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - assert(lazy_modules != NULL); - - PyObject *lazy_submodules; - if (PyDict_GetItemRef(lazy_modules, name, &lazy_submodules) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } - else if (lazy_submodules == NULL) { - Py_RETURN_NONE; - } - - module_dict = get_mod_dict(modobj); - if (module_dict == NULL || !PyDict_CheckExact(module_dict)) { - Py_DECREF(lazy_submodules); - goto done; - } - - assert(PyAnySet_CheckExact(lazy_submodules)); - Py_BEGIN_CRITICAL_SECTION(lazy_submodules); - publish_lazy_imports_on_module(tstate, lazy_submodules, name, module_dict); - Py_END_CRITICAL_SECTION(); - Py_DECREF(lazy_submodules); - - // once a module is imported it is removed from sys.lazy_modules - if (PyDict_DelItem(lazy_modules, name) < 0) { - goto error; - } - -done: - ret = Py_NewRef(Py_None); - -error: - Py_XDECREF(module_dict); - return ret; + Py_RETURN_NONE; } PyDoc_STRVAR(doc_imp, diff --git a/Python/initconfig.c b/Python/initconfig.c index a996fb117aab9d2..bebadcc76111b77 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -455,7 +455,7 @@ static const char usage_xoptions[] = " log imports of already-loaded modules; also #e{PYTHONPROFILEIMPORTTIME}\n" "#s{-X} #L{int_max_str_digits}#b{=N}: limit the size of int<->str conversions;\n" " 0 disables the limit; also #e{PYTHONINTMAXSTRDIGITS}\n" -"#s{-X} #L{lazy_imports}#b{=[all|none|normal]}: control global lazy imports;\n" +"#s{-X} #L{lazy_imports}#b{=[all|normal]}: control global lazy imports;\n" " default is #B{normal}; also #e{PYTHON_LAZY_IMPORTS}\n" "#s{-X} #L{no_debug_ranges}: don't include extra location information in code objects;\n" " also #e{PYTHONNODEBUGRANGES}\n" @@ -1065,7 +1065,8 @@ config_check_consistency(const PyConfig *config) assert(config->int_max_str_digits >= 0); // cpu_count can be -1 if the user doesn't override it. assert(config->cpu_count != 0); - // lazy_imports can be -1 (default), 0 (off), or 1 (on). + // lazy_imports can be -1 (default) or 1 (on). 0 is rejected later + // for embedders with an error message. assert(config->lazy_imports >= -1 && config->lazy_imports <= 1); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). @@ -2437,15 +2438,12 @@ config_init_lazy_imports(PyConfig *config) if (strcmp(env, "all") == 0) { lazy_imports = 1; } - else if (strcmp(env, "none") == 0) { - lazy_imports = 0; - } else if (strcmp(env, "normal") == 0) { lazy_imports = -1; } else { return _PyStatus_ERR("PYTHON_LAZY_IMPORTS: invalid value; " - "expected 'all', 'none', or 'normal'"); + "expected 'all' or 'normal'"); } config->lazy_imports = lazy_imports; } @@ -2455,15 +2453,12 @@ config_init_lazy_imports(PyConfig *config) if (wcscmp(x_value, L"all") == 0) { lazy_imports = 1; } - else if (wcscmp(x_value, L"none") == 0) { - lazy_imports = 0; - } else if (wcscmp(x_value, L"normal") == 0) { lazy_imports = -1; } else { return _PyStatus_ERR("-X lazy_imports: invalid value; " - "expected 'all', 'none', or 'normal'"); + "expected 'all' or 'normal'"); } config->lazy_imports = lazy_imports; } diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 646106f0a9655c0..0941ed593ff7d14 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -60,6 +60,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif DWRF_CFA_restore = 0xc0 // Restore register }; @@ -562,6 +565,13 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes @@ -570,6 +580,9 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 DWRF_UV(DWRF_REG_FP); DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! diff --git a/Python/marshal.c b/Python/marshal.c index 990afefe0d3b419..9688d426419c2fa 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -2061,7 +2061,6 @@ marshal_load_impl(PyObject *module, PyObject *file, int allow_code) /*[clinic input] @permit_long_summary -@permit_long_docstring_body marshal.dumps value: object @@ -2075,14 +2074,14 @@ marshal.dumps Return the bytes object that would be written to a file by dump(value, file). -Raise a ValueError exception if value has (or contains an object that has) an -unsupported type. +Raise a ValueError exception if value has (or contains an object that +has) an unsupported type. [clinic start generated code]*/ static PyObject * marshal_dumps_impl(PyObject *module, PyObject *value, int version, int allow_code) -/*[clinic end generated code: output=115f90da518d1d49 input=80cd3f30c1637ade]*/ +/*[clinic end generated code: output=115f90da518d1d49 input=dc1edcafd43124c5]*/ { return _PyMarshal_WriteObjectToString(value, version, allow_code); } @@ -2098,13 +2097,13 @@ marshal.loads Convert the bytes-like object to a value. -If no valid value is found, raise EOFError, ValueError or TypeError. Extra -bytes in the input are ignored. +If no valid value is found, raise EOFError, ValueError or TypeError. +Extra bytes in the input are ignored. [clinic start generated code]*/ static PyObject * marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) -/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ +/*[clinic end generated code: output=62c0c538d3edc31f input=286f1dbd6811d2ad]*/ { RFILE rf; char *s = bytes->buf; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1dc3a248f45f0c8..e726dc0e6fd1114 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_uop_metadata.h" #include "pycore_long.h" @@ -127,7 +128,7 @@ static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, (1 << DICT_MAX_WATCHERS)); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -156,6 +157,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static void watch_type(PyTypeObject *type, _PyBloomFilter *filter) { @@ -580,10 +592,8 @@ optimize_uops( // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); _Py_uop_abstractcontext_init(ctx, dependencies); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, NULL, 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e10a096baa33188..96dbaea5a5797ef 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" @@ -1459,7 +1460,8 @@ dummy_func(void) { type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { @@ -2041,7 +2043,16 @@ dummy_func(void) { PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + /* LOAD_SPECIAL expands to _RECORD_TOS_TYPE + _INSERT_NULL + + * _LOAD_SPECIAL. Insert _GUARD_TYPE_VERSION before the + * already-emitted _INSERT_NULL so deopt sees the original + * stack shape.*/ + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); + bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01ecb3790aa2cdb..f336549d2ed2440 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3706,7 +3706,8 @@ type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { @@ -3895,7 +3896,11 @@ PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 0c460282feceef9..32b147199544cfc 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -82,6 +82,9 @@ #if defined(__linux__) # include <sys/syscall.h> // System call interface #endif +#if defined(__APPLE__) +# include <mach/mach_time.h> // mach_absolute_time, mach_timebase_info +#endif // ============================================================================= // CONSTANTS AND CONFIGURATION @@ -217,11 +220,7 @@ struct BaseEvent { typedef struct { struct BaseEvent base; // Common event header uint32_t process_id; // Process ID where code was generated -#if defined(__APPLE__) - uint64_t thread_id; // Thread ID where code was generated -#else uint32_t thread_id; // Thread ID where code was generated -#endif uint64_t vma; // Virtual memory address where code is loaded uint64_t code_address; // Address of the actual machine code uint64_t code_size; // Size of the machine code in bytes @@ -295,7 +294,9 @@ static PerfMapJitState perf_jit_map_state; // ============================================================================= /* Time conversion constant */ +#if !defined(__APPLE__) static const intptr_t nanoseconds_per_second = 1000000000; +#endif /* * Get current monotonic time in nanoseconds @@ -307,6 +308,18 @@ static const intptr_t nanoseconds_per_second = 1000000000; * Returns: Current monotonic time in nanoseconds since an arbitrary epoch */ static int64_t get_current_monotonic_ticks(void) { +#if defined(__APPLE__) + // On macOS the jitdump file is consumed by profilers (such as samply) that + // timestamp their samples using mach_absolute_time(). The jitdump event + // timestamps must use the same clock domain, otherwise the JIT code + // mappings cannot be lined up with the samples. + static mach_timebase_info_data_t timebase = {0, 0}; + if (timebase.denom == 0) { + (void)mach_timebase_info(&timebase); + } + uint64_t ticks = mach_absolute_time(); + return (int64_t)(ticks * timebase.numer / timebase.denom); +#else struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { Py_UNREACHABLE(); // Should never fail on supported systems @@ -318,6 +331,7 @@ static int64_t get_current_monotonic_ticks(void) { result *= nanoseconds_per_second; result += ts.tv_nsec; return result; +#endif } /* @@ -652,7 +666,12 @@ static void perf_map_jit_write_entry_with_name( ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); #if defined(__APPLE__) - pthread_threadid_np(NULL, &ev.thread_id); + // The jitdump format defines the thread id field as a 32-bit value, but + // pthread_threadid_np() returns a 64-bit id. Truncate it to 32 bits to + // keep the record layout identical to other platforms. + uint64_t thread_id = 0; + pthread_threadid_np(NULL, &thread_id); + ev.thread_id = (uint32_t)thread_id; #else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call #endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 58c61e64bfc4e99..d90b789c2b57126 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -210,9 +210,8 @@ enum perf_trampoline_type { static void free_code_arenas(void); static void -perf_trampoline_reset_state(void) +perf_trampoline_clear_code_watcher(void) { - free_code_arenas(); if (code_watcher_id >= 0) { PyCode_ClearWatcher(code_watcher_id); code_watcher_id = -1; @@ -220,6 +219,13 @@ perf_trampoline_reset_state(void) extra_code_index = -1; } +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + perf_trampoline_clear_code_watcher(); +} + static int perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) { @@ -621,9 +627,10 @@ _PyPerfTrampoline_AfterFork_Child(void) // After fork, Fini may leave the old code watcher registered // if trampolined code objects from the parent still exist // (trampoline_refcount > 0). Clear it unconditionally before - // Init registers a new one, to prevent two watchers sharing - // the same globals and double-decrementing trampoline_refcount. - perf_trampoline_reset_state(); + // Init registers a new one, but keep the old arenas mapped: the + // child may still need to return through trampoline frames that + // were on the C stack at fork(). + perf_trampoline_clear_code_watcher(); _PyPerfTrampoline_Init(1); } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 46579a45f4cc397..0bdc7ddd92dc823 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1486,15 +1486,11 @@ init_interp_main(PyThreadState *tstate) // Initialize lazy imports based on configuration. Do this after site // module is imported to avoid circular imports during startup. - if (config->lazy_imports != -1) { - PyImport_LazyImportsMode lazy_mode; - if (config->lazy_imports == 1) { - lazy_mode = PyImport_LAZY_ALL; - } - else { - lazy_mode = PyImport_LAZY_NONE; - } - if (PyImport_SetLazyImportsMode(lazy_mode) < 0) { + if (config->lazy_imports == 0) { + return _PyStatus_ERR("PyConfig.lazy_imports=0 is not supported"); + } + if (config->lazy_imports == 1) { + if (PyImport_SetLazyImportsMode(PyImport_LAZY_ALL) < 0) { return _PyStatus_ERR("failed to set lazy imports mode"); } } @@ -3324,7 +3320,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. - os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + // Using {public} ensures "dynamic" string messages are visible + // in the log without special configuration. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%{public}s", text); Py_RETURN_NONE; } diff --git a/Python/pystate.c b/Python/pystate.c index bf2616a49148a74..530bd567b770be3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -320,6 +320,7 @@ _Py_COMP_DIAG_POP &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void @@ -633,7 +634,7 @@ init_interpreter(PyInterpreterState *interp, // Trace fitness configuration init_policy(&interp->opt_config.fitness_initial, "PYTHON_JIT_FITNESS_INITIAL", - FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, UOP_MAX_TRACE_LENGTH - 1); + FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, FITNESS_INITIAL); interp->opt_config.specialization_enabled = !is_env_enabled("PYTHON_SPECIALIZATION_OFF"); interp->opt_config.uops_optimize_enabled = !is_env_disabled("PYTHON_UOPS_OPTIMIZE"); diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 645bb013581288e..8fb1fa36f85e739 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -36,7 +36,7 @@ _Py_hexlify_scalar(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) adds a ton of complication. Who ever really hexes huge data? The 16-64 byte boosts align nicely with md5 - sha512 hexdigests. */ -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR /* 128-bit vector of 16 unsigned bytes */ typedef unsigned char v16u8 __attribute__((vector_size(16))); @@ -110,7 +110,7 @@ _Py_hexlify_simd(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) _Py_hexlify_scalar(src + i, dst, len - i); } -#endif /* HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ +#endif /* _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ static PyObject * _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, @@ -191,7 +191,7 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, unsigned char c; if (bytes_per_sep_group == 0) { -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR if (arglen >= 16) { _Py_hexlify_simd((const unsigned char *)argbuf, retbuf, arglen); } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 6c089a834dcd40d..6fecc23502b46ef 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -100,9 +100,16 @@ extern "C" { # define HAVE_PROCESS_VM_READV 0 #endif +static inline int +_Py_RemoteDebug_HasPermissionError(void) +{ + return PyErr_Occurred() + && PyErr_ExceptionMatches(PyExc_PermissionError); +} + #define _set_debug_exception_cause(exception, format, ...) \ do { \ - if (!PyErr_ExceptionMatches(PyExc_PermissionError)) { \ + if (!_Py_RemoteDebug_HasPermissionError()) { \ PyThreadState *tstate = _PyThreadState_GET(); \ if (!_PyErr_Occurred(tstate)) { \ _PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \ @@ -112,6 +119,20 @@ extern "C" { } \ } while (0) +#define _set_debug_oserror_from_errno(err, format, ...) \ + do { \ + errno = (err); \ + PyErr_SetFromErrno(PyExc_OSError); \ + _set_debug_exception_cause(PyExc_OSError, format, ##__VA_ARGS__); \ + } while (0) + +#define _set_debug_oserror_from_errno_with_filename(err, filename, format, ...) \ + do { \ + errno = (err); \ + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); \ + _set_debug_exception_cause(PyExc_OSError, format, ##__VA_ARGS__); \ + } while (0) + static inline size_t get_page_size(void) { size_t page_size = 0; @@ -147,6 +168,7 @@ typedef struct { int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; + int page_cache_count; Py_ssize_t page_size; } proc_handle_t; @@ -169,7 +191,9 @@ _Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address } char buf[sizeof(_Py_Debug_Cookie) - 1]; if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) { - PyErr_Clear(); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyErr_Clear(); + } return 0; } return memcmp(buf, _Py_Debug_Cookie, sizeof(buf)) == 0; @@ -185,14 +209,16 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) handle->pages[i].data = NULL; handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { - for (int i = 0; i < MAX_PAGES; i++) { + for (int i = 0; i < handle->page_cache_count; i++) { handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -202,6 +228,21 @@ static mach_port_t pid_to_task(pid_t pid); // Initialize the process handle UNUSED static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { + handle->pid = 0; +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + handle->task = 0; +#elif defined(MS_WINDOWS) + handle->hProcess = NULL; +#elif defined(__linux__) + handle->memfd = -1; +#endif + handle->page_size = get_page_size(); + handle->page_cache_count = 0; + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } + handle->pid = pid; #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX handle->task = pid_to_task(handle->pid); @@ -214,18 +255,12 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION | PROCESS_SUSPEND_RESUME, FALSE, pid); if (handle->hProcess == NULL) { - PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } -#elif defined(__linux__) - handle->memfd = -1; #endif - handle->page_size = get_page_size(); - for (int i = 0; i < MAX_PAGES; i++) { - handle->pages[i].data = NULL; - handle->pages[i].valid = 0; - } return 0; } @@ -390,17 +425,19 @@ return_section_address_fat( size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64); if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno(err, "Failed to determine CPU type via sysctlbyname " "for fat binary analysis at 0x%lx: %s", - base, strerror(errno)); + base, strerror(err)); return 0; } if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno(err, "Failed to determine CPU ABI capability via sysctlbyname " "for fat binary analysis at 0x%lx: %s", - base, strerror(errno)); + base, strerror(err)); return 0; } @@ -453,26 +490,29 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ { int fd = open(path, O_RDONLY); if (fd == -1) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot open binary file '%s' for section '%s' search: %s", - path, secname, strerror(errno)); + path, secname, strerror(err)); return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot get file size for binary '%s' during section '%s' search: %s", - path, secname, strerror(errno)); + path, secname, strerror(err)); close(fd); return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s", - path, (long long)fs.st_size, secname, strerror(errno)); + path, (long long)fs.st_size, secname, strerror(err)); close(fd); return 0; } @@ -501,15 +541,21 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ } if (munmap(map, fs.st_size) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to unmap binary file '%s' (size: %lld bytes): %s", - path, (long long)fs.st_size, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, + "Failed to unmap binary file '%s' (size: %lld bytes): %s", + path, (long long)fs.st_size, strerror(err)); + } result = 0; } if (close(fd) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close binary file '%s': %s", - path, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, + "Failed to close binary file '%s': %s", + path, strerror(err)); + } result = 0; } return result; @@ -554,14 +600,15 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s char map_filename[MAXPATHLEN + 1]; - while (mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)&region_info, - &count, - &object_name) == KERN_SUCCESS) + kern_return_t kr; + while ((kr = mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&region_info, + &count, + &object_name)) == KERN_SUCCESS) { if ((region_info.protection & VM_PROT_READ) == 0 @@ -585,18 +632,32 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s } if (strncmp(filename, substr, strlen(substr)) == 0) { + PyErr_Clear(); uintptr_t result = search_section_in_file( secname, map_filename, address, size, proc_ref); - if (result != 0 - && (validator == NULL || validator(handle, result))) - { - return result; + if (result != 0) { + if (validator == NULL || validator(handle, result)) { + return result; + } + if (_Py_RemoteDebug_HasPermissionError()) { + return 0; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { + return 0; } } address += size; } + if (kr != KERN_INVALID_ADDRESS && !PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while searching PID %d for section '%s' " + "(kern_return_t: %d)", + handle->pid, secname, kr); + } + return 0; } @@ -619,25 +680,29 @@ search_elf_file_for_section( int fd = open(elf_file, O_RDONLY); if (fd < 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot open ELF file '%s' for section '%s' search: %s", - elf_file, secname, strerror(errno)); + elf_file, secname, strerror(err)); goto exit; } struct stat file_stats; if (fstat(fd, &file_stats) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot get file size for ELF file '%s' during section '%s' search: %s", - elf_file, secname, strerror(errno)); + elf_file, secname, strerror(err)); goto exit; } file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (file_memory == MAP_FAILED) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot memory map ELF file '%s' (size: %lld bytes) for section '%s' search: %s", - elf_file, (long long)file_stats.st_size, secname, strerror(errno)); + elf_file, (long long)file_stats.st_size, secname, strerror(err)); + file_memory = NULL; goto exit; } @@ -694,12 +759,23 @@ search_elf_file_for_section( exit: if (file_memory != NULL) { - munmap(file_memory, file_stats.st_size); + if (munmap(file_memory, file_stats.st_size) != 0) { + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, + "Failed to unmap ELF file '%s' (size: %lld bytes): %s", + elf_file, (long long)file_stats.st_size, strerror(err)); + } + result = 0; + } } if (fd >= 0 && close(fd) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close ELF file '%s': %s", - elf_file, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, + "Failed to close ELF file '%s': %s", + elf_file, strerror(err)); + } result = 0; } return result; @@ -714,9 +790,10 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, "Cannot open process memory map file '%s' for PID %d section search: %s", - maps_file_path, handle->pid, strerror(errno)); + maps_file_path, handle->pid, strerror(err)); return 0; } @@ -783,20 +860,37 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c if (strstr(filename, substr)) { PyErr_Clear(); retval = search_elf_file_for_section(handle, secname, start, path); - if (retval - && (validator == NULL || validator(handle, retval))) - { + if (retval) { + if (validator == NULL || validator(handle, retval)) { + break; + } + if (_Py_RemoteDebug_HasPermissionError()) { + retval = 0; + break; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { break; } retval = 0; } } + if (retval == 0 && !PyErr_Occurred() && ferror(maps_file)) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, + "Failed to read process map file '%s' for PID %d section search: %s", + maps_file_path, handle->pid, strerror(err)); + } + PyMem_Free(line); if (fclose(maps_file) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close process map file '%s': %s", - maps_file_path, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, + "Failed to close process map file '%s': %s", + maps_file_path, strerror(err)); + } retval = 0; } @@ -819,9 +913,9 @@ static int is_process_alive(HANDLE hProcess) { static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* secname) { HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot open PE file for section '%s' analysis (error %lu)", secname, error); return NULL; @@ -829,9 +923,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, 0); if (!hMap) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot create file mapping for PE file section '%s' analysis (error %lu)", secname, error); CloseHandle(hFile); @@ -840,9 +934,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* BYTE* mapView = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0); if (!mapView) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot map view of PE file for section '%s' analysis (error %lu)", secname, error); CloseHandle(hMap); @@ -900,9 +994,9 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const } while (hProcSnap == INVALID_HANDLE_VALUE && GetLastError() == ERROR_BAD_LENGTH); if (hProcSnap == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_PermissionError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Unable to create module snapshot for PID %d section '%s' " "search (error %lu). Check permissions or PID validity", handle->pid, secname, error); @@ -913,17 +1007,46 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const moduleEntry.dwSize = sizeof(moduleEntry); void* runtime_addr = NULL; - for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) { + if (!Module32FirstW(hProcSnap, &moduleEntry)) { + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, + "Unable to enumerate modules for PID %d section '%s' " + "search (error %lu)", + handle->pid, secname, error); + CloseHandle(hProcSnap); + return 0; + } + + do { // Look for either python executable or DLL if (wcsstr(moduleEntry.szModule, substr)) { + PyErr_Clear(); void *candidate = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); - if (candidate != NULL - && (validator == NULL || validator(handle, (uintptr_t)candidate))) - { - runtime_addr = candidate; + if (candidate != NULL) { + if (validator == NULL || validator(handle, (uintptr_t)candidate)) { + runtime_addr = candidate; + break; + } + if (_Py_RemoteDebug_HasPermissionError()) { + break; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { break; } } + } while (Module32NextW(hProcSnap, &moduleEntry)); + + if (runtime_addr == NULL && !PyErr_Occurred()) { + DWORD error = GetLastError(); + if (error != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, + "Module enumeration failed for PID %d section '%s' " + "search (error %lu)", + handle->pid, secname, error); + } } CloseHandle(hProcSnap); @@ -944,24 +1067,28 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) address = search_windows_map_for_section(handle, "PyRuntime", L"python", _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d on Windows platform", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Windows platform", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__linux__) && HAVE_PROCESS_VM_READV // On Linux, search for 'python' in executable or DLL address = search_linux_map_for_section(handle, "PyRuntime", "python", _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d on Linux platform", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Linux platform", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python @@ -970,17 +1097,19 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) PyErr_Clear(); address = search_map_for_section(handle, "PyRuntime", *candidate, _Py_RemoteDebug_ValidatePyRuntimeCookie); - if (address != 0) { + if (address != 0 || _Py_RemoteDebug_HasPermissionError()) { break; } } if (address == 0) { - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d " - "on macOS platform (tried both libpython and python)", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d " + "on macOS platform (tried both libpython and python)", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #else _set_debug_exception_cause(PyExc_RuntimeError, @@ -1001,9 +1130,9 @@ open_proc_mem_fd(proc_handle_t *handle) handle->memfd = open(mem_file_path, O_RDWR); if (handle->memfd == -1) { - PyErr_SetFromErrno(PyExc_OSError); - _set_debug_exception_cause(PyExc_OSError, - "failed to open file %s: %s", mem_file_path, strerror(errno)); + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, mem_file_path, + "failed to open file %s: %s", mem_file_path, strerror(err)); return -1; } return 0; @@ -1014,6 +1143,9 @@ open_proc_mem_fd(proc_handle_t *handle) static int read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { + if (len == 0) { + return 0; + } if (handle->memfd == -1) { if (open_proc_mem_fd(handle) < 0) { return -1; @@ -1031,14 +1163,23 @@ read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, siz read_bytes = preadv(handle->memfd, local, 1, offset); if (read_bytes < 0) { + int err = errno; + errno = err; PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "preadv failed for PID %d at address 0x%lx " "(size %zu, partial read %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "preadv returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; @@ -1050,11 +1191,15 @@ read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, siz static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { + if (len == 0) { + return 0; + } #ifdef MS_WINDOWS SIZE_T read_bytes = 0; SIZE_T result = 0; do { if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) { + DWORD error = GetLastError(); // Check if the process is still alive: we need to be able to tell our caller // that the process is dead and not just that the read failed. if (!is_process_alive(handle->hProcess)) { @@ -1062,14 +1207,20 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address PyErr_SetFromErrno(PyExc_OSError); return -1; } - PyErr_SetFromWindowsErr(0); - DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_OSError, "ReadProcessMemory failed for PID %d at address 0x%lx " "(size %zu, partial read %zu bytes): Windows error %lu", handle->pid, remote_address + result, len - result, result, error); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "ReadProcessMemory returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zu bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while (result < len); return 0; @@ -1090,31 +1241,40 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { - if (errno == ENOSYS) { + int err = errno; + if (err == ENOSYS) { return read_remote_memory_fallback(handle, remote_address, len, dst); } + errno = err; PyErr_SetFromErrno(PyExc_OSError); - if (errno == ESRCH) { + if (err == ESRCH) { return -1; } _set_debug_exception_cause(PyExc_OSError, "process_vm_readv failed for PID %d at address 0x%lx " "(size %zu, partial read %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "process_vm_readv returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; #elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX - Py_ssize_t result = -1; + mach_vm_size_t bytes_read = 0; kern_return_t kr = mach_vm_read_overwrite( handle->task, (mach_vm_address_t)remote_address, len, (mach_vm_address_t)dst, - (mach_vm_size_t*)&result); + &bytes_read); if (kr != KERN_SUCCESS) { switch (err_get_code(kr)) { @@ -1158,6 +1318,13 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address } return -1; } + if (bytes_read != (mach_vm_size_t)len) { + PyErr_Format(PyExc_OSError, + "mach_vm_read_overwrite read %llu of %zu bytes for PID %d at " + "address 0x%lx", + (unsigned long long)bytes_read, len, handle->pid, remote_address); + return -1; + } return 0; #else Py_UNREACHABLE(); @@ -1169,6 +1336,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address static int _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { + if (len == 0) { + return 0; + } if (handle->memfd == -1) { if (open_proc_mem_fd(handle) < 0) { return -1; @@ -1186,10 +1356,19 @@ _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remot written = pwritev(handle->memfd, local, 1, offset); if (written < 0) { + int err = errno; + errno = err; PyErr_SetFromErrno(PyExc_OSError); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "pwritev wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while ((size_t)written != local[0].iov_len); return 0; @@ -1200,19 +1379,29 @@ _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remot UNUSED static int _Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { + if (len == 0) { + return 0; + } #ifdef MS_WINDOWS SIZE_T written = 0; SIZE_T result = 0; do { if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_OSError, "WriteProcessMemory failed for PID %d at address 0x%lx " "(size %zu, partial write %zu bytes): Windows error %lu", handle->pid, remote_address + result, len - result, result, error); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "WriteProcessMemory wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zu bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while (result < len); return 0; @@ -1233,17 +1422,26 @@ _Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_addres written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); if (written < 0) { - if (errno == ENOSYS) { + int err = errno; + if (err == ENOSYS) { return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); } + errno = err; PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "process_vm_writev failed for PID %d at address 0x%lx " "(size %zu, partial write %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "process_vm_writev wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while ((size_t)written != local[0].iov_len); return 0; @@ -1287,8 +1485,9 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } - // Search for valid cached page - for (int i = 0; i < MAX_PAGES; i++) { + // Search only the pages used since the last clear. The cache is cleared + // between profiler samples, so entries are packed at the front. + for (int i = 0; i < handle->page_cache_count; i++) { page_cache_entry_t *entry = &handle->pages[i]; if (entry->valid && entry->page_addr == page_base) { memcpy(out, entry->data + offset_in_page, size); @@ -1296,33 +1495,31 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } } - // Find reusable slot - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (!entry->valid) { + if (handle->page_cache_count < MAX_PAGES) { + page_cache_entry_t *entry = &handle->pages[handle->page_cache_count]; + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); if (entry->data == NULL) { - entry->data = PyMem_RawMalloc(page_size); - if (entry->data == NULL) { - PyErr_NoMemory(); - _set_debug_exception_cause(PyExc_MemoryError, - "Cannot allocate %zu bytes for page cache entry " - "during read from PID %d at address 0x%lx", - page_size, handle->pid, addr); - return -1; - } - } - - if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact amount as a fallback - PyErr_Clear(); - goto fallback; + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; } + } - entry->page_addr = page_base; - entry->valid = 1; - memcpy(out, entry->data + offset_in_page, size); - return 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact amount as a fallback + PyErr_Clear(); + goto fallback; } + + entry->page_addr = page_base; + entry->valid = 1; + handle->page_cache_count++; + memcpy(out, entry->data + offset_in_page, size); + return 0; } fallback: @@ -1330,6 +1527,49 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } +typedef struct { + uintptr_t remote_addr; + void *local_buf; + size_t size; +} _Py_RemoteReadSegment; + +#define _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS 4 + +// Batched read of multiple remote regions in a single syscall when supported. +// Returns total bytes read (>= 0) on success, -1 if batched reads are +// unavailable or the syscall failed. Callers compare the return value against +// cumulative segment sizes to determine which segments were fully populated. +UNUSED static Py_ssize_t +_Py_RemoteDebug_BatchedReadRemoteMemory( + proc_handle_t *handle, + const _Py_RemoteReadSegment *segments, + int nsegs) +{ +#if defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd == -1 + && nsegs > 0 + && nsegs <= _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS) { + struct iovec local[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + struct iovec remote[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + for (int i = 0; i < nsegs; i++) { + local[i].iov_base = segments[i].local_buf; + local[i].iov_len = segments[i].size; + remote[i].iov_base = (void *)segments[i].remote_addr; + remote[i].iov_len = segments[i].size; + } + ssize_t nread = process_vm_readv(handle->pid, local, nsegs, remote, nsegs, 0); + if (nread >= 0) { + return (Py_ssize_t)nread; + } + } +#else + (void)handle; + (void)segments; + (void)nsegs; +#endif + return -1; +} + UNUSED static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, diff --git a/Python/symtable.c b/Python/symtable.c index 2263a2d8db9097d..9a2e278caaf9e2c 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2870,6 +2870,7 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; if (current_type == ClassBlock && !future_annotations) { st->st_cur->ste_can_see_class_scope = 1; + parent_ste->ste_needs_classdict = 1; if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { return 0; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c6447d03369a949..b79ebf56371ff2a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1893,6 +1893,7 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] +@permit_long_summary sys.get_int_max_str_digits Return the maximum string digits limit for non-binary int<->str conversions. @@ -1900,7 +1901,7 @@ Return the maximum string digits limit for non-binary int<->str conversions. static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=77fb74e987ba7ecb]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); return PyLong_FromLong(interp->long_state.max_str_digits); @@ -1908,6 +1909,7 @@ sys_get_int_max_str_digits_impl(PyObject *module) /*[clinic input] +@permit_long_summary sys.set_int_max_str_digits maxdigits: int @@ -1917,7 +1919,7 @@ Set the maximum string digits limit for non-binary int<->str conversions. static PyObject * sys_set_int_max_str_digits_impl(PyObject *module, int maxdigits) -/*[clinic end generated code: output=734d4c2511f2a56d input=d7e3f325db6910c5]*/ +/*[clinic end generated code: output=734d4c2511f2a56d input=d4c0bf50c466d57a]*/ { if (_PySys_SetIntMaxStrDigits(maxdigits) < 0) { return NULL; @@ -2129,6 +2131,7 @@ sys__getframe_impl(PyObject *module, int depth) } /*[clinic input] +@permit_long_summary sys._current_frames Return a dict mapping each thread's thread id to its current stack frame. @@ -2138,7 +2141,7 @@ This function should be used for specialized purposes only. static PyObject * sys__current_frames_impl(PyObject *module) -/*[clinic end generated code: output=d2a41ac0a0a3809a input=2a9049c5f5033691]*/ +/*[clinic end generated code: output=d2a41ac0a0a3809a input=e1ce34f43501e0d6]*/ { return _PyThread_CurrentFrames(); } @@ -2317,17 +2320,17 @@ sys__stats_clear_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body sys._stats_dump -> bool Dump stats to file, and clears the stats. -Return False if no statistics were not dumped because stats gathering was off. +Return False if no statistics were not dumped because stats gathering +was off. [clinic start generated code]*/ static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=6e346b4ba0de4489 input=5a3ab40d2fb5af47]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=7f3b7758cb59d2ff]*/ { int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); @@ -2471,16 +2474,16 @@ sys.remote_exec Executes a file containing Python code in a given remote Python process. This function returns immediately, and the code will be executed by the -target process's main thread at the next available opportunity, similarly -to how signals are handled. There is no interface to determine when the -code has been executed. The caller is responsible for making sure that -the file still exists whenever the remote process tries to read it and that -it hasn't been overwritten. +target process's main thread at the next available opportunity, +similarly to how signals are handled. There is no interface to +determine when the code has been executed. The caller is responsible +for making sure that the file still exists whenever the remote process +tries to read it and that it hasn't been overwritten. -The remote process must be running a CPython interpreter of the same major -and minor version as the local process. If either the local or remote -interpreter is pre-release (alpha, beta, or release candidate) then the -local and remote interpreters must be the same exact version. +The remote process must be running a CPython interpreter of the same +major and minor version as the local process. If either the local or +remote interpreter is pre-release (alpha, beta, or release candidate) +then the local and remote interpreters must be the same exact version. Args: pid (int): The process ID of the target Python process. @@ -2490,7 +2493,7 @@ local and remote interpreters must be the same exact version. static PyObject * sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) -/*[clinic end generated code: output=7d94c56afe4a52c0 input=39908ca2c5fe1eb0]*/ +/*[clinic end generated code: output=7d94c56afe4a52c0 input=7bd58f8da20cb74c]*/ { PyObject *path; const char *debugger_script_path; @@ -2842,8 +2845,8 @@ Sets the global lazy imports mode. The mode parameter must be one of the following strings: - "all": All top-level imports become potentially lazy -- "none": All lazy imports are suppressed (even explicitly marked ones) -- "normal": Only explicitly marked imports (with 'lazy' keyword) are lazy +- "normal": Only explicitly marked imports (with 'lazy' keyword) are + lazy In addition to the mode, lazy imports can be controlled via the filter provided to sys.set_lazy_imports_filter @@ -2852,12 +2855,12 @@ provided to sys.set_lazy_imports_filter static PyObject * sys_set_lazy_imports_impl(PyObject *module, PyObject *mode) -/*[clinic end generated code: output=1ff34ba6c4feaf73 input=f04e70d8bf9fe4f6]*/ +/*[clinic end generated code: output=1ff34ba6c4feaf73 input=db3242f0ff6e5dcc]*/ { PyImport_LazyImportsMode lazy_mode; if (!PyUnicode_Check(mode)) { PyErr_SetString(PyExc_TypeError, - "mode must be a string: 'normal', 'all', or 'none'"); + "mode must be a string: 'normal' or 'all'"); return NULL; } if (PyUnicode_CompareWithASCIIString(mode, "normal") == 0) { @@ -2866,12 +2869,9 @@ sys_set_lazy_imports_impl(PyObject *module, PyObject *mode) else if (PyUnicode_CompareWithASCIIString(mode, "all") == 0) { lazy_mode = PyImport_LAZY_ALL; } - else if (PyUnicode_CompareWithASCIIString(mode, "none") == 0) { - lazy_mode = PyImport_LAZY_NONE; - } else { PyErr_SetString(PyExc_ValueError, - "mode must be 'normal', 'all', or 'none'"); + "mode must be 'normal' or 'all'"); return NULL; } @@ -2887,22 +2887,19 @@ sys.get_lazy_imports Gets the global lazy imports mode. Returns "all" if all top level imports are potentially lazy. -Returns "none" if all explicitly marked lazy imports are suppressed. Returns "normal" if only explicitly marked imports are lazy. [clinic start generated code]*/ static PyObject * sys_get_lazy_imports_impl(PyObject *module) -/*[clinic end generated code: output=4147dec48c51ae99 input=8cb574f1e4e3003c]*/ +/*[clinic end generated code: output=4147dec48c51ae99 input=6f8dd4f2c82893f2]*/ { switch (PyImport_GetLazyImportsMode()) { case PyImport_LAZY_NORMAL: return PyUnicode_FromString("normal"); case PyImport_LAZY_ALL: return PyUnicode_FromString("all"); - case PyImport_LAZY_NONE: - return PyUnicode_FromString("none"); default: PyErr_SetString(PyExc_RuntimeError, "unknown lazy imports mode"); return NULL; diff --git a/README.rst b/README.rst index e9dd44382972d59..ac84a8a7d054bda 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.15.0 beta 1 +This is Python version 3.15.0 beta 2 ==================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index f23c1d5286f92af..c619a9a0c1c5a1b 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -463,7 +463,7 @@ def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] - return + return # type: ignore[unreachable] try: assert spec.origin is not None st = os.stat(spec.origin) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda725396406118..f8828a56b4c7da7 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -42,6 +42,7 @@ 'test', 'xxlimited', 'xxlimited_35', + 'xxlimited_3_13', 'xxsubtype', } diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 5465e2d4b6171f1..485c9314cf70015 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -24,8 +24,6 @@ python_version = 3.10 # ...And be strict: strict = True -strict_bytes = True -local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 4a55a1a24ee1bed..92134bc1321e1b5 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -11,6 +11,7 @@ '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', + 'xxlimited_3_13.c', )) # C files in the fhe following directories must not be built with diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 43ed552fcf75d90..404a81af11e39ff 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -77,6 +77,7 @@ 'PyStructSequence_Field[]', 'PyStructSequence_Desc', 'PyABIInfo', + 'PySlot[]', } # XXX We should normalize all cases to a single name, diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7af64ed017ba73d..bf08e5568205e7a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -327,6 +327,7 @@ Modules/pyexpat.c - error_info_of - Modules/pyexpat.c - handler_info - Modules/termios.c - termios_constants - Modules/timemodule.c init_timezone YEAR - +Modules/unicodedata.c unicodedata_create_capi capi - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - @@ -467,6 +468,7 @@ Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index dfec0524cfe016d..fa53b205c4ff6af 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -43,10 +43,6 @@ PyDescr_TYPE PyWrapperFlag_KEYWORDS # cpython/fileobject.h Py_UniversalNewlineFgets -# cpython/ceval.h -PyUnstable_CopyPerfMapFile -PyUnstable_PerfTrampoline_CompileCode -PyUnstable_PerfTrampoline_SetPersistAfterFork # cpython/pyframe.h PyUnstable_EXECUTABLE_KINDS PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index f981f0bcaf89f0f..1c643caea98e3b5 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -173,12 +173,12 @@ def docstring_line_width(self) -> int: Pydoc adds indentation when displaying functions and methods. To keep the total width of within 80 characters, we use a - maximum of 76 characters for global functions and classes, - and 72 characters for methods. + maximum of 72 characters for global functions and classes, + and 68 characters for methods. """ if self.cls is not None and not self.kind.new_or_init: - return 72 - return 76 + return 68 + return 72 def __repr__(self) -> str: return f'<clinic.Function {self.name!r}>' diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 60f43b99c0f69dd..c8a914c22a9e137 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -279,6 +279,23 @@ def staticmethod_call(): for _ in range(1000 * WORK_SCALE): obj.my_staticmethod() + +class MyDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + def __set__(self, obj, value): + pass + +class MyClassWithDescriptor: + attr = MyDescriptor() + +@register_benchmark +def descriptor(): + obj = MyClassWithDescriptor() + for _ in range(1000 * WORK_SCALE): + obj.attr + @register_benchmark def deepcopy(): x = {'list': [1, 2], 'tuple': (1, None)} diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index fee3435496da0bd..8e367422a961da2 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -151,6 +151,45 @@ def create_threads(n): time.sleep(0.05) ''' +ASYNC_CODE = '''\ +import asyncio +import contextlib +import math + +def compute_slice(seed): + result = 0.0 + for i in range(2000): + result += math.sin(seed + i) * math.sqrt(i + 1) + return result + +async def leaf_task(seed): + total = 0.0 + while True: + total += compute_slice(seed) + await asyncio.sleep(0) + +async def parent_task(seed): + child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") + try: + while True: + compute_slice(seed) + await asyncio.sleep(0.001) + finally: + child.cancel() + with contextlib.suppress(asyncio.CancelledError): + await child + +async def main(): + tasks = [ + asyncio.create_task(parent_task(i), name=f"parent-{i}") + for i in range(8) + ] + await asyncio.gather(*tasks) + +if __name__ == "__main__": + asyncio.run(main()) +''' + CODE_EXAMPLES = { "basic": { "code": CODE, @@ -164,10 +203,29 @@ def create_threads(n): "code": CODE_WITH_TONS_OF_THREADS, "description": "Tons of threads doing mixed CPU/IO work", }, + "asyncio": { + "code": ASYNC_CODE, + "description": "Asyncio tasks with active and awaited coroutine chains", + }, +} + +OPERATIONS = { + "stack_trace": { + "method": "get_stack_trace", + "label": "get_stack_trace()", + }, + "async_stack_trace": { + "method": "get_async_stack_trace", + "label": "get_async_stack_trace()", + }, + "all_awaited_by": { + "method": "get_all_awaited_by", + "label": "get_all_awaited_by()", + }, } -def benchmark(unwinder, duration_seconds=10, blocking=False): +def benchmark(unwinder, duration_seconds=10, blocking=False, operation="stack_trace"): """Benchmark mode - measure raw sampling speed for specified duration""" sample_count = 0 fail_count = 0 @@ -175,11 +233,14 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): start_time = time.perf_counter() end_time = start_time + duration_seconds total_attempts = 0 + operation_info = OPERATIONS[operation] + operation_method = getattr(unwinder, operation_info["method"]) colors = get_colors(can_colorize()) print( - f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + f"{colors.BOLD_BLUE}Benchmarking {operation_info['label']} speed " + f"for {duration_seconds} seconds...{colors.RESET}" ) try: @@ -190,8 +251,8 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): if blocking: unwinder.pause_threads() try: - stack_trace = unwinder.get_stack_trace() - if stack_trace: + sample = operation_method() + if sample: sample_count += 1 finally: if blocking: @@ -239,6 +300,7 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 ), "total_work_time": total_work_time, + "operation": operation_info["label"], "avg_work_time_us": ( (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 ), @@ -252,7 +314,7 @@ def print_benchmark_results(results): colors = get_colors(can_colorize()) print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") - print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{results['operation']} Benchmark Results{colors.RESET}") print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") # Basic statistics @@ -329,6 +391,8 @@ def parse_arguments(): %(prog)s -d 60 # Run basic benchmark for 60 seconds %(prog)s --code deep_static # Run deep static call stack benchmark %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + %(prog)s --operation async_stack_trace + %(prog)s --operation all_awaited_by Available code examples: {examples_desc} @@ -348,8 +412,15 @@ def parse_arguments(): "--code", "-c", choices=list(CODE_EXAMPLES.keys()), - default="basic", - help="Code example to benchmark (default: basic)", + default=None, + help="Code example to benchmark (default: basic, or asyncio for async operations)", + ) + + parser.add_argument( + "--operation", + choices=list(OPERATIONS.keys()), + default="stack_trace", + help="Remote unwinder operation to benchmark (default: stack_trace)", ) parser.add_argument( @@ -365,7 +436,10 @@ def parse_arguments(): help="Stop all threads before sampling for consistent snapshots", ) - return parser.parse_args() + args = parser.parse_args() + if args.code is None: + args.code = "asyncio" if args.operation != "stack_trace" else "basic" + return args def create_target_process(temp_file, code_example="basic"): @@ -420,6 +494,9 @@ def main(): print( f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" ) + print( + f"{colors.CYAN}Operation:{colors.RESET} {colors.GREEN}{OPERATIONS[args.operation]['label']}{colors.RESET}" + ) print( f"{colors.CYAN}Blocking Mode:{colors.RESET} {colors.GREEN if args.blocking else colors.YELLOW}{'enabled' if args.blocking else 'disabled'}{colors.RESET}" ) @@ -451,7 +528,12 @@ def main(): unwinder = _remote_debugging.RemoteUnwinder( process.pid, cache_frames=True, **kwargs ) - results = benchmark(unwinder, duration_seconds=args.duration, blocking=args.blocking) + results = benchmark( + unwinder, + duration_seconds=args.duration, + blocking=args.blocking, + operation=args.operation, + ) finally: cleanup_process(process, temp_file_path) diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be58..a9039d03f5f6fa1 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -13,6 +13,9 @@ <Component Id="libs_python3.lib" Directory="libs" Guid="*"> <File Id="libs_python_stable.lib" Name="python$(var.MajorVersionNumber).lib" KeyPath="yes" /> </Component> + <Component Id="libs_python3t.lib" Directory="libs" Guid="*"> + <File Id="libs_python_abi3tcompat.lib" Name="python$(var.MajorVersionNumber)t.lib" KeyPath="yes" /> + </Component> <Component Id="libs_python.lib" Directory="libs" Guid="*"> <File Id="libs_python.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber).lib" KeyPath="yes" /> </Component> @@ -24,6 +27,9 @@ <Component Id="libs_python3_d.lib" Directory="libs" Guid="*"> <File Id="libs_python_stable_d.lib" Name="python$(var.MajorVersionNumber)_d.lib" /> </Component> + <Component Id="libs_python3t_d.lib" Directory="libs" Guid="*"> + <File Id="libs_python_abi3tcompat_d.lib" Name="python$(var.MajorVersionNumber)t_d.lib" /> + </Component> <Component Id="libs_python_d.lib" Directory="libs" Guid="*"> <File Id="libs_python_d.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)_d.lib" /> </Component> diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs index 5dad7c98d4f048a..7c7784741d9178d 100644 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ b/Tools/msi/tcltk/tcltk_files.wxs @@ -10,11 +10,14 @@ <Component Id="_tkinter.lib" Directory="libs" Guid="*"> <File Name="_tkinter.lib" KeyPath="yes" /> </Component> - <Component Id="tcl86t.dll" Directory="DLLs" Guid="*"> - <File Name="tcl86t.dll" KeyPath="yes" /> + <Component Id="tcl90.dll" Directory="DLLs" Guid="*"> + <File Name="tcl90.dll" KeyPath="yes" /> </Component> - <Component Id="tk86t.dll" Directory="DLLs" Guid="*"> - <File Name="tk86t.dll" KeyPath="yes" /> + <Component Id="tcl9tk90.dll" Directory="DLLs" Guid="*"> + <File Name="tcl9tk90.dll" KeyPath="yes" /> + </Component> + <Component Id="libtommath.dll" Directory="DLLs" Guid="*"> + <File Name="libtommath.dll" KeyPath="yes" /> </Component> <Component Id="zlib1.dll" Directory="DLLs" Guid="*"> <File Name="zlib1.dll" KeyPath="yes" /> diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat index 02bcca943cf79b4..db98f690151196c 100644 --- a/Tools/msi/testrelease.bat +++ b/Tools/msi/testrelease.bat @@ -88,9 +88,7 @@ exit /B 0 ) @if not errorlevel 1 ( @echo Testing Tcl/tk - @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 - @set TCL_LIBRARY= ) @set EXITCODE=%ERRORLEVEL% diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index af5cbaa7689f33d..46381ea58a12382 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.19.1 +mypy==2.1.0 # needed for peg_generator: -types-psutil==7.2.2.20260130 -types-setuptools==82.0.0.20260210 +types-psutil==7.2.2.20260508 +types-setuptools==82.0.0.20260508 diff --git a/configure b/configure index cff7dfbfba8b9ad..7abc41648b3c451 100755 --- a/configure +++ b/configure @@ -647,6 +647,8 @@ MODULE_BLOCK JIT_SHIM_BUILD_O JIT_SHIM_O JIT_STENCILS_H +MODULE_XXLIMITED_3_13_FALSE +MODULE_XXLIMITED_3_13_TRUE MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -6643,7 +6645,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6692,7 +6694,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6724,7 +6726,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="g++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6753,7 +6755,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6802,7 +6804,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6834,7 +6836,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="c++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6863,7 +6865,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6912,7 +6914,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6944,7 +6946,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="clang++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6973,7 +6975,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7022,7 +7024,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7054,7 +7056,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="icpc" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -9864,6 +9866,61 @@ fi ;; esac +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for thread stack size" >&5 +printf %s "checking for thread stack size... " >&6; } +if test ${ac_cv_thread_stack_size+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_thread_stack_size" >&5 +printf "%s\n" "$ac_cv_thread_stack_size" >&6; } + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + case $enable_wasm_dynamic_linking in #( yes) : ac_cv_func_dlopen=yes ;; #( @@ -14619,7 +14676,7 @@ if test "x$ac_cv_have_libgcc_eh_frame_registration" = xyes then : -printf "%s\n" "#define HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h fi @@ -19483,7 +19540,7 @@ if test "x$ac_cv_efficient_builtin_shufflevector" = xyes then : -printf "%s\n" "#define HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h fi @@ -19963,12 +20020,6 @@ if test "x$ac_cv_func_dup" = xyes then : printf "%s\n" "#define HAVE_DUP 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" -if test "x$ac_cv_func_dup3" = xyes -then : - printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "execv" "ac_cv_func_execv" if test "x$ac_cv_func_execv" = xyes @@ -20443,12 +20494,6 @@ if test "x$ac_cv_func_pipe" = xyes then : printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" -if test "x$ac_cv_func_pipe2" = xyes -then : - printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "plock" "ac_cv_func_plock" if test "x$ac_cv_func_plock" = xyes @@ -21116,7 +21161,13 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" + ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" +if test "x$ac_cv_func_dup3" = xyes +then : + printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" if test "x$ac_cv_func_getentropy" = xyes then : printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h @@ -21127,6 +21178,12 @@ if test "x$ac_cv_func_getgroups" = xyes then : printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" +if test "x$ac_cv_func_pipe2" = xyes +then : + printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" if test "x$ac_cv_func_system" = xyes @@ -24582,7 +24639,7 @@ printf "%s\n" "#define HAVE_DECL_PR_SET_VMA_ANON_NAME $ac_have_decl" >>confdefs. if test $ac_have_decl = 1 then : -printf "%s\n" "#define HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h fi @@ -31944,6 +32001,7 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_xxlimited=n/a py_cv_module_xxlimited_35=n/a + py_cv_module_xxlimited_3_13=n/a py_cv_module_=n/a ;; #( @@ -35005,6 +35063,46 @@ fi printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxlimited_3_13" >&5 +printf %s "checking for stdlib extension module xxlimited_3_13... " >&6; } + if test "$py_cv_module_xxlimited_3_13" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module_xxlimited_3_13=yes +else case e in #( + e) py_cv_module_xxlimited_3_13=missing ;; +esac +fi +else case e in #( + e) py_cv_module_xxlimited_3_13=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE_XXLIMITED_3_13_STATE=$py_cv_module_xxlimited_3_13$as_nl" + if test "x$py_cv_module_xxlimited_3_13" = xyes +then : + + + + +fi + if test "$py_cv_module_xxlimited_3_13" = yes; then + MODULE_XXLIMITED_3_13_TRUE= + MODULE_XXLIMITED_3_13_FALSE='#' +else + MODULE_XXLIMITED_3_13_TRUE='#' + MODULE_XXLIMITED_3_13_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module_xxlimited_3_13" >&5 +printf "%s\n" "$py_cv_module_xxlimited_3_13" >&6; } + + # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" JIT_SHIM_O="" @@ -35518,6 +35616,10 @@ if test -z "${MODULE_XXLIMITED_35_TRUE}" && test -z "${MODULE_XXLIMITED_35_FALSE as_fn_error $? "conditional \"MODULE_XXLIMITED_35\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_XXLIMITED_3_13_TRUE}" && test -z "${MODULE_XXLIMITED_3_13_FALSE}"; then + as_fn_error $? "conditional \"MODULE_XXLIMITED_3_13\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 diff --git a/configure.ac b/configure.ac index ac3269ab765c0df..47a6e59623b8303 100644 --- a/configure.ac +++ b/configure.ac @@ -1137,10 +1137,10 @@ preset_cxx="$CXX" if test -z "$CXX" then case "$ac_cv_cc_name" in - gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; - cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + gcc) AC_PATH_TOOL([CXX], [g++], [notfound]) ;; + cc) AC_PATH_TOOL([CXX], [c++], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -2462,6 +2462,54 @@ AS_CASE([$ac_sys_system], ] ) +dnl On Linux, check the thread stack size. musl (ex: Alpine Linux) uses +dnl a default thread stack size of 128 kB, whereas the glibc uses 8 MiB. +dnl Python needs at least 1 MiB. +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + AC_CACHE_CHECK([for thread stack size], [ac_cv_thread_stack_size], [ + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ]) + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + fi +fi + AS_CASE([$enable_wasm_dynamic_linking], [yes], [ac_cv_func_dlopen=yes], [no], [ac_cv_func_dlopen=no], @@ -3863,7 +3911,7 @@ __deregister_frame(0); [ac_cv_have_libgcc_eh_frame_registration=no]) ]) AS_VAR_IF([ac_cv_have_libgcc_eh_frame_registration], [yes], [ - AC_DEFINE([HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], + AC_DEFINE([_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], [Define to 1 if libgcc __register_frame and __deregister_frame are linkable.]) ]) @@ -5163,7 +5211,7 @@ AC_LINK_IFELSE([ ]) AS_VAR_IF([ac_cv_efficient_builtin_shufflevector], [yes], [ - AC_DEFINE([HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], + AC_DEFINE([_Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], [Define if compiler supports __builtin_shufflevector with 128-bit vectors AND the target architecture has native SIMD (not just API availability)]) @@ -5421,7 +5469,7 @@ fi AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clearenv \ clock closefrom close_range confstr \ - copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dladdr dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ @@ -5431,7 +5479,7 @@ AC_CHECK_FUNCS([ \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll ppoll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ + plock poll ppoll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ @@ -5468,7 +5516,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - AC_CHECK_FUNCS([getentropy getgroups system]) + AC_CHECK_FUNCS([dup3 getentropy getgroups pipe2 system]) fi AC_CHECK_DECL([dirfd], @@ -5788,7 +5836,7 @@ AC_CHECK_DECLS([UT_NAMESIZE], # musl libc redefines struct prctl_mm_map and conflicts with linux/prctl.h AS_IF([test "$ac_cv_libc" != musl], [ AC_CHECK_DECLS([PR_SET_VMA_ANON_NAME], - [AC_DEFINE([HAVE_PR_SET_VMA_ANON_NAME], [1], + [AC_DEFINE([_Py_HAVE_PR_SET_VMA_ANON_NAME], [1], [Define if you have the 'PR_SET_VMA_ANON_NAME' constant.])], [], [@%:@include <linux/prctl.h> @@ -8047,6 +8095,7 @@ AS_CASE([$ac_sys_system], [termios], [xxlimited], [xxlimited_35], + [xxlimited_3_13], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -8438,6 +8487,7 @@ dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_3_13], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" diff --git a/pyconfig.h.in b/pyconfig.h.in index ad372255445d138..7ef83fcd0b9e0bf 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -320,10 +320,6 @@ /* Define to 1 if you have the <editline/readline.h> header file. */ #undef HAVE_EDITLINE_READLINE_H -/* Define if compiler supports __builtin_shufflevector with 128-bit vectors - AND the target architecture has native SIMD (not just API availability) */ -#undef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR - /* Define to 1 if you have the <endian.h> header file. */ #undef HAVE_ENDIAN_H @@ -701,10 +697,6 @@ /* Define to 1 if you have the 'dld' library (-ldld). */ #undef HAVE_LIBDLD -/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. - */ -#undef HAVE_LIBGCC_EH_FRAME_REGISTRATION - /* Define to 1 if you have the 'ieee' library (-lieee). */ #undef HAVE_LIBIEEE @@ -1007,9 +999,6 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ -#undef HAVE_PR_SET_VMA_ANON_NAME - /* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK @@ -2067,6 +2056,17 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if compiler supports __builtin_shufflevector with 128-bit vectors + AND the target architecture has native SIMD (not just API availability) */ +#undef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR + +/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. + */ +#undef _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION + +/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ +#undef _Py_HAVE_PR_SET_VMA_ANON_NAME + /* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ #undef _Py_STACK_GROWS_DOWN