gh-118335: Configure Tier 2 interpreter at build time

Doc/whatsnew/3.13.rst

@@ -906,9 +906,8 @@ The internal architecture is roughly as follows.
 
 * There is a Tier 2 interpreter, but it is mostly intended for debugging
   the earlier stages of the optimization pipeline. If the JIT is not
-  enabled, the Tier 2 interpreter can be invoked by passing Python the
-  ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment
-  variable to ``1``.
+  enabled, the Tier 2 interpreter can be enabled by configuring Python
+  with the ``--enable-experimental-tier2`` option.
 
 * When the ``--enable-experimental-jit`` option is used, the optimized
   Tier 2 IR is translated to machine code, which is then executed.

Lib/dis.py

@@ -216,7 +216,7 @@ def _get_code_array(co, adaptive):
             if op == ENTER_EXECUTOR:
                 try:
                     ex = get_executor(co, i)
-                except ValueError:
+                except (ValueError, RuntimeError):
                     ex = None
 
                 if ex:

Lib/test/support/__init__.py

@@ -2540,6 +2540,7 @@ def exceeds_recursion_limit():
 def without_optimizer(func):
     try:
         import _testinternalcapi
+        from _testinternalcapi import get_optimizer, set_optimizer
     except ImportError:
         return func
     @functools.wraps(func)

Lib/test/test_capi/test_opt.py

@@ -34,6 +34,8 @@ def clear_executors(func):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 class TestOptimizerAPI(unittest.TestCase):
 
     def test_new_counter_optimizer_dealloc(self):
@@ -136,6 +138,8 @@ def get_opnames(ex):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 class TestExecutorInvalidation(unittest.TestCase):
 
     def setUp(self):
@@ -215,6 +219,8 @@ def f():
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUops(unittest.TestCase):
 
@@ -579,6 +585,8 @@ def testfunc(n):
 
 
 @requires_specialization
+@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"),
+                     "Requires optimizer infrastructure")
 @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.")
 class TestUopsOptimization(unittest.TestCase):
 

Lib/test/test_monitoring.py

@@ -1831,15 +1831,17 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase):
 
     def setUp(self):
         _testinternalcapi = import_module("_testinternalcapi")
-        self.old_opt = _testinternalcapi.get_optimizer()
-        opt = _testinternalcapi.new_counter_optimizer()
-        _testinternalcapi.set_optimizer(opt)
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            self.old_opt = _testinternalcapi.get_optimizer()
+            opt = _testinternalcapi.new_counter_optimizer()
+            _testinternalcapi.set_optimizer(opt)
         super(TestOptimizer, self).setUp()
 
     def tearDown(self):
         super(TestOptimizer, self).tearDown()
         import _testinternalcapi
-        _testinternalcapi.set_optimizer(self.old_opt)
+        if hasattr(_testinternalcapi, "get_optimizer"):
+            _testinternalcapi.set_optimizer(self.old_opt)
 
     def test_for_loop(self):
         def test_func(x):

Lib/test/test_opcache.py

@@ -16,6 +16,8 @@
 
 def disabling_optimizer(func):
     def wrapper(*args, **kwargs):
+        if not hasattr(_testinternalcapi, "get_optimizer"):
+            return func(*args, **kwargs)
         old_opt = _testinternalcapi.get_optimizer()
         _testinternalcapi.set_optimizer(None)
         try:

Lib/test/test_optimizer.py

@@ -80,6 +80,8 @@ def func(x=0):
 
 class TestOptimizerSymbols(unittest.TestCase):
 
+    @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"),
+                "requires _testinternalcapi.uop_symbols_test")
     def test_optimizer_symbols(self):
         _testinternalcapi.uop_symbols_test()
 

Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst

@@ -0,0 +1,4 @@
+Change how to use the tier 2 interpreter. Instead of running Python with
+``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this
+choice is now made at build time by configuring with
+``--enable-experimental-tier2``.

Modules/_opcode.c

@@ -367,7 +367,13 @@ _opcode_get_executor_impl(PyObject *module, PyObject *code, int offset)
                      Py_TYPE(code)->tp_name);
         return NULL;
     }
+#ifdef _Py_JIT
     return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset);
+#else
+    PyErr_Format(PyExc_RuntimeError,
+                 "Executors is not available in this build");
+    return NULL;
+#endif
 }
 
 static PyMethodDef

Modules/_testinternalcapi.c

@@ -985,6 +985,8 @@ get_co_framesize(PyObject *self, PyObject *arg)
     return PyLong_FromLong(code->co_framesize);
 }
 
+#ifdef _Py_JIT
+
 static PyObject *
 new_counter_optimizer(PyObject *self, PyObject *arg)
 {
@@ -1012,7 +1014,10 @@ set_optimizer(PyObject *self, PyObject *opt)
 static PyObject *
 get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
-    PyObject *opt = (PyObject *)PyUnstable_GetOptimizer();
+    PyObject *opt = NULL;
+#ifdef _Py_JIT
+    opt = (PyObject *)PyUnstable_GetOptimizer();
+#endif
     if (opt == NULL) {
         Py_RETURN_NONE;
     }
@@ -1045,6 +1050,8 @@ invalidate_executors(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+#endif
+
 static int _pending_callback(void *arg)
 {
     /* we assume the argument is callable object to which we own a reference */
@@ -2020,12 +2027,14 @@ static PyMethodDef module_functions[] = {
     {"iframe_getline", iframe_getline, METH_O, NULL},
     {"iframe_getlasti", iframe_getlasti, METH_O, NULL},
     {"get_co_framesize", get_co_framesize, METH_O, NULL},
+#ifdef _Py_JIT
     {"get_optimizer", get_optimizer,  METH_NOARGS, NULL},
     {"set_optimizer", set_optimizer,  METH_O, NULL},
     {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL},
     {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL},
     {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL},
     {"invalidate_executors", invalidate_executors, METH_O, NULL},
+#endif
     {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc),
      METH_VARARGS | METH_KEYWORDS},
     {"pending_identify", pending_identify, METH_VARARGS, NULL},
@@ -2072,7 +2081,9 @@ static PyMethodDef module_functions[] = {
     {"py_thread_id", get_py_thread_id, METH_NOARGS},
 #endif
     {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS},
+#ifdef _Py_JIT
     {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
+#endif
     {NULL, NULL} /* sentinel */
 };
 

Objects/codeobject.c

@@ -1496,6 +1496,8 @@ PyCode_GetFreevars(PyCodeObject *code)
     return _PyCode_GetFreevars(code);
 }
 
+#ifdef _Py_JIT
+
 static void
 clear_executors(PyCodeObject *co)
 {
@@ -1515,6 +1517,8 @@ _PyCode_Clear_Executors(PyCodeObject *code)
     clear_executors(code);
 }
 
+#endif
+
 static void
 deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions)
 {
@@ -1739,9 +1743,11 @@ code_dealloc(PyCodeObject *co)
 
         PyMem_Free(co_extra);
     }
+#ifdef _Py_JIT
     if (co->co_executors != NULL) {
         clear_executors(co);
     }
+#endif
 
     Py_XDECREF(co->co_consts);
     Py_XDECREF(co->co_names);

Objects/object.c

@@ -2281,9 +2281,11 @@ static PyTypeObject* static_types[] = {
     &_PyBufferWrapper_Type,
     &_PyContextTokenMissing_Type,
     &_PyCoroWrapper_Type,
+#ifdef _Py_JIT
     &_PyCounterExecutor_Type,
     &_PyCounterOptimizer_Type,
     &_PyDefaultOptimizer_Type,
+#endif
     &_Py_GenericAliasIterType,
     &_PyHamtItems_Type,
     &_PyHamtKeys_Type,
@@ -2304,8 +2306,10 @@ static PyTypeObject* static_types[] = {
     &_PyPositionsIterator,
     &_PyUnicodeASCIIIter_Type,
     &_PyUnion_Type,
+#ifdef _Py_JIT
     &_PyUOpExecutor_Type,
     &_PyUOpOptimizer_Type,
+#endif
     &_PyWeakref_CallableProxyType,
     &_PyWeakref_ProxyType,
     &_PyWeakref_RefType,

PCbuild/_testinternalcapi.vcxproj

@@ -108,6 +108,11 @@
       <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
     </ProjectReference>
   </ItemGroup>
+  <ItemDefinitionGroup>
+    <ClCompile>
+      <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+    </ClCompile>
+  </ItemDefinitionGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

Python/bytecodes.c

@@ -2355,6 +2355,7 @@ dummy_func(
             CHECK_EVAL_BREAKER();
             assert(oparg <= INSTR_OFFSET());
             JUMPBY(-oparg);
+            #ifdef _Py_JIT
             #if ENABLE_SPECIALIZATION
             _Py_BackoffCounter counter = this_instr[1].counter;
             if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) {
@@ -2380,6 +2381,7 @@ dummy_func(
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
             }
             #endif  /* ENABLE_SPECIALIZATION */
+            #endif /* _Py_JIT */
         }
 
         pseudo(JUMP) = {
@@ -2393,6 +2395,7 @@ dummy_func(
         };
 
         tier1 inst(ENTER_EXECUTOR, (--)) {
+            #ifdef _Py_JIT
             int prevoparg = oparg;
             CHECK_EVAL_BREAKER();
             if (this_instr->op.code != ENTER_EXECUTOR ||
@@ -2410,6 +2413,9 @@ dummy_func(
             tstate->previous_executor = Py_None;
             Py_INCREF(executor);
             GOTO_TIER_TWO(executor);
+            #else
+            assert(0);
+            #endif /* _Py_JIT */
         }
 
         replaced op(_POP_JUMP_IF_FALSE, (cond -- )) {

Python/ceval.c

@@ -755,7 +755,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     _Py_CODEUNIT *next_instr;
     PyObject **stack_pointer;
 
-#ifndef _Py_JIT
+#if _Py_JIT & 4
     /* Tier 2 interpreter state */
     _PyExecutorObject *current_executor = NULL;
     const _PyUOpInstruction *next_uop = NULL;
@@ -959,14 +959,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     goto error;
 
 
+#if _Py_JIT & 4 /* Tier 2 interpreter */
 
 // Tier 2 is also here!
 enter_tier_two:
 
-#ifdef _Py_JIT
-    assert(0);
-#else
-
 #undef LOAD_IP
 #define LOAD_IP(UNUSED) (void)0
 
@@ -1111,7 +1108,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     tstate->previous_executor = (PyObject *)current_executor;
     GOTO_TIER_TWO(exit->executor);
 
-#endif  // _Py_JIT
+#endif  /* _Py_JIT & 4 */
 
 }
 

Python/instrumentation.c

@@ -1702,10 +1702,12 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp)
         );
         return 0;
     }
+#ifdef _Py_JIT
     if (code->co_executors != NULL) {
         _PyCode_Clear_Executors(code);
     }
     _Py_Executors_InvalidateDependency(interp, code, 1);
+#endif
     int code_len = (int)Py_SIZE(code);
     /* Exit early to avoid creating instrumentation
      * data for potential statically allocated code
@@ -1943,7 +1945,9 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events)
         goto done;
     }
     set_global_version(tstate, new_version);
+#ifdef _Py_JIT
     _Py_Executors_InvalidateAll(interp, 1);
+#endif
     res = instrument_all_executing_code_objects(interp);
 done:
     _PyEval_StartTheWorld(interp);
@@ -1983,7 +1987,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent
         code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT;
     }
 
+#ifdef _Py_JIT
     _Py_Executors_InvalidateDependency(interp, code, 1);
+#endif
 
     res = instrument_lock_held(code, interp);
 

Python/optimizer.c

@@ -1,3 +1,5 @@
+#ifdef _Py_JIT
+
 #include "Python.h"
 #include "opcode.h"
 #include "pycore_interp.h"
@@ -1622,3 +1624,5 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation)
         }
     }
 }
+
+#endif /* _Py_JIT */