Merge branch 'main' into backoff-counter-woes

Doc/c-api/frame.rst

@@ -120,12 +120,19 @@ See also :ref:`Reflection <reflection>`.
 
 .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame)
 
-   Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`).
+   Get the *frame*'s :attr:`~frame.f_locals` attribute.
+   If the frame refers to a function or comprehension, this returns
+   a write-through proxy object that allows modifying the locals.
+   In all other cases (classes, modules) it returns the :class:`dict`
+   representing the frame locals directly.
 
    Return a :term:`strong reference`.
 
    .. versionadded:: 3.11
 
+   .. versionchanged:: 3.13
+      Return a proxy object for functions and comprehensions.
+
 
 .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame)
 

Doc/c-api/init_config.rst

@@ -1251,7 +1251,10 @@ PyConfig
       for more information.
 
       Set by :option:`-X perf <-X>` command line option and by the
-      :envvar:`PYTHONPERFSUPPORT` environment variable.
+      :envvar:`PYTHONPERFSUPPORT` environment variable for perf support
+      with stack pointers and :option:`-X perfjit <-X>` command line option
+      and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf
+      support with DWARF JIT information.
 
       Default: ``-1``.
 

Doc/howto/perf_profiling.rst

@@ -205,3 +205,36 @@ You can check if your system has been compiled with this flag by running::
 If you don't see any output it means that your interpreter has not been compiled with
 frame pointers and therefore it may not be able to show Python functions in the output
 of ``perf``.
+
+
+How to work without frame pointers
+----------------------------------
+
+If you are working with a Python interpreter that has been compiled without frame pointers
+you can still use the ``perf`` profiler but the overhead will be a bit higher because Python
+needs to generate unwinding information for every Python function call on the fly. Additionally,
+``perf`` will take more time to process the data because it will need to use the DWARF debugging
+information to unwind the stack and this is a slow process.
+
+To enable this mode, you can use the environment variable :envvar:`PYTHONPERFJITSUPPORT` or the
+:option:`-X perfjit <-X>` option, which will enable the JIT mode for the ``perf`` profiler.
+
+When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to
+call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file.
+
+    $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperfjit my_script.py
+    $ perf inject -i perf.data --jit
+    $ perf report -g -i perf.data
+
+or using the environment variable::
+
+    $ PYTHONPERFJITSUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py
+    $ perf inject -i perf.data --jit
+    $ perf report -g -i perf.data
+
+Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take snapshots of the stack of
+the process being profiled and save the information in the ``perf.data`` file. By default the size of
+the stack dump is 8192 bytes but the user can change the size by passing the size after comma like
+``--call-graph dwarf,4096``. The size of the stack dump is important because if the size is too small
+``perf`` will not be able to unwind the stack and the output will be incomplete.
+

Doc/library/bdb.rst

@@ -289,6 +289,10 @@ The :mod:`bdb` module also defines two classes:
       Start debugging from *frame*.  If *frame* is not specified, debugging
       starts from caller's frame.
 
+      .. versionchanged:: 3.13
+         :func:`set_trace` will enter the debugger immediately, rather than
+         on the next line of code to be executed.
+
    .. method:: set_continue()
 
       Stop only at breakpoints or when finished.  If there are no breakpoints,

Doc/library/cmdline.rst

@@ -36,6 +36,7 @@ The following modules have a command-line interface.
 * :mod:`pyclbr`
 * :mod:`pydoc`
 * :mod:`quopri`
+* :ref:`random <random-cli>`
 * :mod:`runpy`
 * :ref:`site <site-commandline>`
 * :ref:`sqlite3 <sqlite3-cli>`

Doc/library/itertools.rst

@@ -504,24 +504,17 @@ loops that truncate the stream.
           # islice('ABCDEFG', 2, None) → C D E F G
           # islice('ABCDEFG', 0, None, 2) → A C E G
           s = slice(*args)
-          start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1
-          it = iter(range(start, stop, step))
-          try:
-              nexti = next(it)
-          except StopIteration:
-              # Consume *iterable* up to the *start* position.
-              for i, element in zip(range(start), iterable):
-                  pass
-              return
-          try:
-              for i, element in enumerate(iterable):
-                  if i == nexti:
-                      yield element
-                      nexti = next(it)
-          except StopIteration:
-              # Consume to *stop*.
-              for i, element in zip(range(i + 1, stop), iterable):
-                  pass
+          start = 0 if s.start is None else s.start
+          stop = s.stop
+          step = 1 if s.step is None else s.step
+          if start < 0 or (stop is not None and stop < 0) or step <= 0:
+              raise ValueError
+          indices = count() if stop is None else range(max(stop, start))
+          next_i = start
+          for i, element in zip(indices, iterable):
+              if i == next_i:
+                  yield element
+                  next_i += step
 
 
 .. function:: pairwise(iterable)

Doc/library/pdb.rst

@@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command.
 
 The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode::
 
-   > ...(3)double()
-   -> return x * 2
+   > ...(2)double()
+   -> breakpoint()
    (Pdb) p x
    3
    (Pdb) continue
@@ -164,6 +164,9 @@ slightly different way:
    .. versionchanged:: 3.7
       The keyword-only argument *header*.
 
+   .. versionchanged:: 3.13
+      :func:`set_trace` will enter the debugger immediately, rather than
+      on the next line of code to be executed.
 
 .. function:: post_mortem(traceback=None)
 

Doc/library/random.rst

@@ -706,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.)
    <https://allendowney.com/research/rand/downey07randfloat.pdf>`_ a
    paper by Allen B. Downey describing ways to generate more
    fine-grained floats than normally generated by :func:`.random`.
+
+.. _random-cli:
+
+Command-line usage
+------------------
+
+.. versionadded:: 3.13
+
+The :mod:`!random` module can be executed from the command line.
+
+.. code-block:: sh
+
+   python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...]
+
+The following options are accepted:
+
+.. program:: random
+
+.. option:: -h, --help
+
+   Show the help message and exit.
+
+.. option:: -c CHOICE [CHOICE ...]
+            --choice CHOICE [CHOICE ...]
+
+   Print a random choice, using :meth:`choice`.
+
+.. option:: -i <N>
+            --integer <N>
+
+   Print a random integer between 1 and N inclusive, using :meth:`randint`.
+
+.. option:: -f <N>
+            --float <N>
+
+   Print a random floating point number between 1 and N inclusive,
+   using :meth:`uniform`.
+
+If no options are given, the output depends on the input:
+
+* String or multiple: same as :option:`--choice`.
+* Integer: same as :option:`--integer`.
+* Float: same as :option:`--float`.
+
+.. _random-cli-example:
+
+Command-line example
+--------------------
+
+Here are some examples of the :mod:`!random` command-line interface:
+
+.. code-block:: console
+
+   $ # Choose one at random
+   $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce"
+   Lobster Thermidor aux crevettes with a Mornay sauce
+
+   $ # Random integer
+   $ python -m random 6
+   6
+
+   $ # Random floating-point number
+   $ python -m random 1.8
+   1.7080016272295635
+
+   $ # With explicit arguments
+   $ python  -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce"
+   egg
+
+   $ python -m random --integer 6
+   3
+
+   $ python -m random --float 1.8
+   1.5666339105010318
+
+   $ python -m random --integer 6
+   5
+
+   $ python -m random --float 6
+   3.1942323316565915

Doc/reference/datamodel.rst

@@ -1341,7 +1341,12 @@ Special read-only attributes
 
    * - .. attribute:: frame.f_locals
      - The dictionary used by the frame to look up
-       :ref:`local variables <naming>`
+       :ref:`local variables <naming>`.
+       If the frame refers to a function or comprehension,
+       this may return a write-through proxy object.
+
+       .. versionchanged:: 3.13
+          Return a proxy for functions and comprehensions.
 
    * - .. attribute:: frame.f_globals
      - The dictionary used by the frame to look up

Doc/using/cmdline.rst

@@ -586,6 +586,15 @@ Miscellaneous options
 
      .. versionadded:: 3.12
 
+   * ``-X perfjit`` enables support for the Linux ``perf`` profiler with DWARF
+     support. When this option is provided, the ``perf`` profiler will be able
+     to report Python calls using DWARF ifnormation. This option is only available on
+     some platforms and will do nothing if is not supported on the current
+     system. The default value is "off". See also :envvar:`PYTHONPERFJITSUPPORT`
+     and :ref:`perf_profiling`.
+
+     .. versionadded:: 3.13
+
    * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`,
      :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`.
      *n* must be greater than or equal to 1.
@@ -1127,6 +1136,21 @@ conflict.
 
    .. versionadded:: 3.12
 
+.. envvar:: PYTHONPERFJITSUPPORT
+
+   If this variable is set to a nonzero value, it enables support for
+   the Linux ``perf`` profiler so Python calls can be detected by it
+   using DWARF information.
+
+   If set to ``0``, disable Linux ``perf`` profiler support.
+
+   See also the :option:`-X perfjit <-X>` command-line option
+   and :ref:`perf_profiling`.
+
+   .. versionadded:: 3.13
+
+
+
 .. envvar:: PYTHON_CPU_COUNT
 
    If this variable is set to a positive integer, it overrides the return

Doc/whatsnew/3.13.rst

@@ -87,6 +87,11 @@ Interpreter improvements:
   Performance improvements are modest -- we expect to be improving this
   over the next few releases.
 
+* :pep:`667`: :attr:`FrameType.f_locals <frame.f_locals>` when used in
+  a function now returns a write-through proxy to the frame's locals,
+  rather than a ``dict``. See the PEP for corresponding C API changes
+  and deprecations.
+
 New typing features:
 
 * :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`,
@@ -231,6 +236,11 @@ Other Language Changes
   equivalent of the :option:`-X frozen_modules <-X>` command-line option.
   (Contributed by Yilei Yang in :gh:`111374`.)
 
+* Add :ref:`support for the perf profiler <perf_profiling>` working without
+  frame pointers through the new environment variable
+  :envvar:`PYTHONPERFJITSUPPORT` and command-line option :option:`-X perfjit
+  <-X>` (Contributed by Pablo Galindo in :gh:`118518`.)
+
 * The new :envvar:`PYTHON_HISTORY` environment variable can be used to change
   the location of a ``.python_history`` file.
   (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in
@@ -710,13 +720,25 @@ pdb
 * :mod:`zipapp` is supported as a debugging target.
   (Contributed by Tian Gao in :gh:`118501`.)
 
+* ``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately
+  rather than on the next line of code to be executed. This change prevents the
+  debugger from breaking outside of the context when ``breakpoint()`` is positioned
+  at the end of the context.
+  (Contributed by Tian Gao in :gh:`118579`.)
+
 queue
 -----
 
 * Add :meth:`queue.Queue.shutdown` (along with :exc:`queue.ShutDown`) for queue
   termination.
   (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.)
 
+random
+------
+
+* Add a :ref:`command-line interface <random-cli>`.
+  (Contributed by Hugo van Kemenade in :gh:`54321`.)
+
 re
 --
 * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity.

Include/internal/pycore_ceval.h

@@ -108,6 +108,7 @@ extern int _PyIsPerfTrampolineActive(void);
 extern PyStatus _PyPerfTrampoline_AfterFork_Child(void);
 #ifdef PY_HAVE_PERF_TRAMPOLINE
 extern _PyPerf_Callbacks _Py_perfmap_callbacks;
+extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks;
 #endif
 
 static inline PyObject*

Include/internal/pycore_ceval_state.h

@@ -75,6 +75,7 @@ struct trampoline_api_st {
                         unsigned int code_size, PyCodeObject* code);
     int (*free_state)(void* state);
     void *state;
+    Py_ssize_t code_padding;
 };
 #endif
 
@@ -83,6 +84,7 @@ struct _ceval_runtime_state {
     struct {
 #ifdef PY_HAVE_PERF_TRAMPOLINE
         perf_status_t status;
+        int perf_trampoline_type;
         Py_ssize_t extra_code_index;
         struct code_arena_st *code_arena;
         struct trampoline_api_st trampoline_api;

Include/internal/pycore_importdl.h

@@ -22,6 +22,11 @@ typedef enum ext_module_kind {
     _Py_ext_module_kind_INVALID = 3,
 } _Py_ext_module_kind;
 
+typedef enum ext_module_origin {
+    _Py_ext_module_origin_CORE = 1,
+    _Py_ext_module_origin_BUILTIN = 2,
+    _Py_ext_module_origin_DYNAMIC = 3,
+} _Py_ext_module_origin;
 
 /* Input for loading an extension module. */
 struct _Py_ext_module_loader_info {
@@ -34,6 +39,7 @@ struct _Py_ext_module_loader_info {
     /* path is always a borrowed ref of name or filename,
      * depending on if it's builtin or not. */
     PyObject *path;
+    _Py_ext_module_origin origin;
     const char *hook_prefix;
     const char *newcontext;
 };
@@ -42,7 +48,11 @@ extern void _Py_ext_module_loader_info_clear(
 extern int _Py_ext_module_loader_info_init(
     struct _Py_ext_module_loader_info *info,
     PyObject *name,
-    PyObject *filename);
+    PyObject *filename,
+    _Py_ext_module_origin origin);
+extern int _Py_ext_module_loader_info_init_for_core(
+    struct _Py_ext_module_loader_info *p_info,
+    PyObject *name);
 extern int _Py_ext_module_loader_info_init_for_builtin(
     struct _Py_ext_module_loader_info *p_info,
     PyObject *name);

Include/internal/pycore_stackref.h

@@ -114,7 +114,7 @@ _Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length)
 
 #define PyStackRef_XSETREF(dst, src) \
     do { \
-        _PyStackRef *_tmp_dst_ptr = &(dst) \
+        _PyStackRef *_tmp_dst_ptr = &(dst); \
         _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \
         *_tmp_dst_ptr = (src); \
         PyStackRef_XDECREF(_tmp_old_dst); \