Merge branch 'pythongh-118331-unraisable' into doctest-refactor-color…

…ize-merge

.github/workflows/reusable-docs.yml

@@ -74,7 +74,7 @@ jobs:
     - name: 'Set up Python'
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'  # known to work with Sphinx 4.2
+        python-version: '3.12'  # known to work with Sphinx 6.2.1
         cache: 'pip'
         cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt'
     - name: 'Install build dependencies'

Doc/conf.py

@@ -298,8 +298,8 @@
     'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'],
 }
 
-# Avoid a warning with Sphinx >= 2.0
-master_doc = 'contents'
+# Avoid a warning with Sphinx >= 4.0
+root_doc = 'contents'
 
 # Allow translation of index directives
 gettext_additional_targets = [

Doc/library/enum.rst

@@ -279,7 +279,7 @@ Data Types
          >>> Color.RED.value
          1
 
-      Value of the member, can be set in :meth:`~object.__new__`.
+      Value of the member, can be set in :meth:`~Enum.__new__`.
 
       .. note:: Enum member values
 
@@ -299,7 +299,7 @@ Data Types
 
    .. attribute:: Enum._value_
 
-      Value of the member, can be set in :meth:`~object.__new__`.
+      Value of the member, can be set in :meth:`~Enum.__new__`.
 
    .. attribute:: Enum._order_
 
@@ -407,8 +407,8 @@ Data Types
 
       results in the call ``int('1a', 16)`` and a value of ``17`` for the member.
 
-      ..note:: When writing a custom ``__new__``, do not use ``super().__new__`` --
-               call the appropriate ``__new__`` instead.
+      .. note:: When writing a custom ``__new__``, do not use ``super().__new__`` --
+                call the appropriate ``__new__`` instead.
 
    .. method:: Enum.__repr__(self)
 
@@ -827,7 +827,7 @@ Supported ``__dunder__`` names
 :attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member``
 items.  It is only available on the class.
 
-:meth:`~object.__new__`, if specified, must create and return the enum members;
+:meth:`~Enum.__new__`, if specified, must create and return the enum members;
 it is also a very good idea to set the member's :attr:`!_value_` appropriately.
 Once all the members are created it is no longer used.
 

Doc/requirements-oldest-sphinx.txt

@@ -7,29 +7,29 @@ blurb
 python-docs-theme>=2022.1
 
 # Generated from:
-#     pip install "Sphinx~=4.2.0"
+#     pip install "Sphinx~=6.2.1"
 #     pip freeze
 #
-# Sphinx 4.2 comes from ``needs_sphinx = '4.2'`` in ``Doc/conf.py``.
+# Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``.
 
-alabaster==0.7.13
-Babel==2.13.0
-certifi==2023.7.22
-charset-normalizer==3.3.0
-docutils==0.17.1
-idna==3.4
+alabaster==0.7.16
+Babel==2.14.0
+certifi==2024.2.2
+charset-normalizer==3.3.2
+docutils==0.19
+idna==3.7
 imagesize==1.4.1
-Jinja2==3.1.2
-MarkupSafe==2.1.3
-packaging==23.2
-Pygments==2.16.1
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+packaging==24.0
+Pygments==2.17.2
 requests==2.31.0
 snowballstemmer==2.2.0
-Sphinx==4.2.0
-sphinxcontrib-applehelp==1.0.4
-sphinxcontrib-devhelp==1.0.2
-sphinxcontrib-htmlhelp==2.0.1
+Sphinx==6.2.1
+sphinxcontrib-applehelp==1.0.8
+sphinxcontrib-devhelp==1.0.6
+sphinxcontrib-htmlhelp==2.0.5
 sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.3
-sphinxcontrib-serializinghtml==1.1.5
-urllib3==2.0.7
+sphinxcontrib-qthelp==1.0.7
+sphinxcontrib-serializinghtml==1.1.10
+urllib3==2.2.1

Doc/tools/extensions/c_annotations.py

@@ -19,7 +19,6 @@
 """
 
 from os import path
-import docutils
 from docutils import nodes
 from docutils.parsers.rst import directives
 from docutils.parsers.rst import Directive
@@ -40,16 +39,6 @@
 }
 
 
-# Monkeypatch nodes.Node.findall for forwards compatibility
-# This patch can be dropped when the minimum Sphinx version is 4.4.0
-# or the minimum Docutils version is 0.18.1.
-if docutils.__version_info__ < (0, 18, 1):
-    def findall(self, *args, **kwargs):
-        return iter(self.traverse(*args, **kwargs))
-
-    nodes.Node.findall = findall
-
-
 class RCEntry:
     def __init__(self, name):
         self.name = name

Doc/tools/extensions/peg_highlight.py

@@ -16,6 +16,7 @@ class PEGLexer(RegexLexer):
         - Rule types
         - Rule options
         - Rules named `invalid_*` or `incorrect_*`
+        - Rules with `RAISE_SYNTAX_ERROR`
     """
 
     name = "PEG"
@@ -59,6 +60,7 @@ class PEGLexer(RegexLexer):
             (r"^(\s+\|\s+.*invalid_\w+.*\n)", bygroups(None)),
             (r"^(\s+\|\s+.*incorrect_\w+.*\n)", bygroups(None)),
             (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),),
+            (r"^(\s+\|\s+.*\{[^}]*RAISE_SYNTAX_ERROR[^}]*\})\n", bygroups(None)),
         ],
         "root": [
             include("invalids"),

Doc/tools/extensions/pyspecific.py

@@ -27,13 +27,7 @@
 from sphinx.util import logging
 from sphinx.util.docutils import SphinxDirective
 from sphinx.writers.text import TextWriter, TextTranslator
-
-try:
-    # Sphinx 6+
-    from sphinx.util.display import status_iterator
-except ImportError:
-    # Deprecated in Sphinx 6.1, will be removed in Sphinx 8
-    from sphinx.util import status_iterator
+from sphinx.util.display import status_iterator
 
 
 ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s'

Include/cpython/dictobject.h

@@ -56,7 +56,11 @@ static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) {
     PyDictObject *mp;
     assert(PyDict_Check(op));
     mp = _Py_CAST(PyDictObject*, op);
+#ifdef Py_GIL_DISABLED
+    return _Py_atomic_load_ssize_relaxed(&mp->ma_used);
+#else
     return mp->ma_used;
+#endif
 }
 #define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op))
 

Include/cpython/object.h

@@ -493,7 +493,7 @@ do { \
 PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
 
 PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg);
-PyAPI_FUNC(void) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict);
+PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict);
 PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj);
 
 #define TYPE_MAX_WATCHERS 8

Include/internal/pycore_ceval.h

@@ -48,8 +48,12 @@ extern void _PyEval_SignalReceived(void);
 #define _Py_PENDING_MAINTHREADONLY 1
 #define _Py_PENDING_RAWFREE 2
 
+typedef int _Py_add_pending_call_result;
+#define _Py_ADD_PENDING_SUCCESS 0
+#define _Py_ADD_PENDING_FULL -1
+
 // Export for '_testinternalcapi' shared extension
-PyAPI_FUNC(int) _PyEval_AddPendingCall(
+PyAPI_FUNC(_Py_add_pending_call_result) _PyEval_AddPendingCall(
     PyInterpreterState *interp,
     _Py_pending_call_func func,
     void *arg,

Include/internal/pycore_ceval_state.h

@@ -14,28 +14,56 @@ extern "C" {
 
 typedef int (*_Py_pending_call_func)(void *);
 
+struct _pending_call {
+    _Py_pending_call_func func;
+    void *arg;
+    int flags;
+};
+
+#define PENDINGCALLSARRAYSIZE 32
+
+#define MAXPENDINGCALLS PENDINGCALLSARRAYSIZE
+/* For interpreter-level pending calls, we want to avoid spending too
+   much time on pending calls in any one thread, so we apply a limit. */
+#if MAXPENDINGCALLS > 100
+#  define MAXPENDINGCALLSLOOP 100
+#else
+#  define MAXPENDINGCALLSLOOP MAXPENDINGCALLS
+#endif
+
+#define MAXPENDINGCALLS_MAIN PENDINGCALLSARRAYSIZE
+/* For the main thread, we want to make sure all pending calls are
+   run at once, for the sake of prompt signal handling.  This is
+   unlikely to cause any problems since there should be very few
+   pending calls for the main thread. */
+#define MAXPENDINGCALLSLOOP_MAIN 0
+
 struct _pending_calls {
     int busy;
     PyMutex mutex;
     /* Request for running pending calls. */
-    int32_t calls_to_do;
-#define NPENDINGCALLS 32
-    struct _pending_call {
-        _Py_pending_call_func func;
-        void *arg;
-        int flags;
-    } calls[NPENDINGCALLS];
+    int32_t npending;
+    /* The maximum allowed number of pending calls.
+       If the queue fills up to this point then _PyEval_AddPendingCall()
+       will return _Py_ADD_PENDING_FULL. */
+    int32_t max;
+    /* We don't want a flood of pending calls to interrupt any one thread
+       for too long, so we keep a limit on the number handled per pass.
+       A value of 0 means there is no limit (other than the maximum
+       size of the list of pending calls). */
+    int32_t maxloop;
+    struct _pending_call calls[PENDINGCALLSARRAYSIZE];
     int first;
-    int last;
+    int next;
 };
 
+
 typedef enum {
     PERF_STATUS_FAILED = -1,  // Perf trampoline is in an invalid state
     PERF_STATUS_NO_INIT = 0,  // Perf trampoline is not initialized
     PERF_STATUS_OK = 1,       // Perf trampoline is ready to be executed
 } perf_status_t;
 
-
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 struct code_arena_st;
 
@@ -48,6 +76,7 @@ struct trampoline_api_st {
 };
 #endif
 
+
 struct _ceval_runtime_state {
     struct {
 #ifdef PY_HAVE_PERF_TRAMPOLINE
@@ -62,10 +91,15 @@ struct _ceval_runtime_state {
 #endif
     } perf;
     /* Pending calls to be made only on the main thread. */
+    // The signal machinery falls back on this
+    // so it must be especially stable and efficient.
+    // For example, we use a preallocated array
+    // for the list of pending calls.
     struct _pending_calls pending_mainthread;
     PyMutex sys_trace_profile_mutex;
 };
 
+
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 # define _PyEval_RUNTIME_PERF_INIT \
     { \

Include/internal/pycore_object.h

@@ -688,7 +688,7 @@ static inline PyDictObject *
 _PyObject_GetManagedDict(PyObject *obj)
 {
     PyManagedDictPointer *dorv = _PyObject_ManagedDictPointer(obj);
-    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_RELAXED(dorv->dict);
+    return (PyDictObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(dorv->dict);
 }
 
 static inline PyDictValues *

Include/internal/pycore_runtime_init.h

@@ -114,6 +114,10 @@ extern PyTypeObject _PyExc_MemoryError;
         .autoTSSkey = Py_tss_NEEDS_INIT, \
         .parser = _parser_runtime_state_INIT, \
         .ceval = { \
+            .pending_mainthread = { \
+                .max = MAXPENDINGCALLS_MAIN, \
+                .maxloop = MAXPENDINGCALLSLOOP_MAIN, \
+            }, \
             .perf = _PyEval_RUNTIME_PERF_INIT, \
         }, \
         .gilstate = { \
@@ -166,6 +170,10 @@ extern PyTypeObject _PyExc_MemoryError;
         .imports = IMPORTS_INIT, \
         .ceval = { \
             .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \
+            .pending = { \
+                .max = MAXPENDINGCALLS, \
+                .maxloop = MAXPENDINGCALLSLOOP, \
+            }, \
         }, \
         .gc = { \
             .enabled = 1, \