Merge pull request #9092 from bashtage/backport-0.14.1

Backport 0.14.1

.github/workflows/codeql.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/generate-documentation.yml

@@ -25,18 +25,18 @@ jobs:
         python-version: ["3.10"]
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Checkout statsmodels.github.io without token
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 1
         repository: statsmodels/statsmodels.github.io
         path: statsmodels.github.io
       if: ${{ github.event_name == 'pull_request' }}
     - name: Checkout statsmodels.github.io with token
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         fetch-depth: 1
         repository: statsmodels/statsmodels.github.io
@@ -52,7 +52,7 @@ jobs:
     - name: Install graphviz
       uses: ts-graphviz/setup-graphviz@v1
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         cache: "pip"

.gitignore

@@ -18,10 +18,13 @@ examples/executed
 
 #setuptools-scm generated version file
 statsmodels/_version.py
-# generated c source and built extensions
+# Generated c source and built extensions
+# and Byte-compiled / optimized / DLL files
 *.c
 *.so
-*.pyd
+*.py[cod]
+__pycache__/
+
 
 # repository directories for bzr-git
 .bzr
@@ -104,3 +107,6 @@ statsmodels/tsa/statespace/_smoothers/_classical.pyx
 # Temporary copies for packaging
 statsmodels/setup.cfg
 statsmodels/LICENSE.txt
+
+# Dataset downloads
+statsmodels/datasets/tests/*.zip

INSTALL.txt

@@ -21,7 +21,7 @@ patsy >= 0.5.2
 
     patsy.readthedocs.org
 
-cython >= 0.29.26
+cython >= 0.29.33 and < 4.0.0
 
     https://cython.org/
 

azure-pipelines.yml

@@ -16,7 +16,6 @@ variables:
   VML_NUM_THREADS: 1
   OPENBLAS_NUM_THREADS: 1
   PYTHONHASHSEED: 0 # Ensure tests are correctly gathered by xdist
-  SETUPTOOLS_USE_DISTUTILS: "stdlib"
   USE_MATPLOTLIB: true
 
 jobs:

docs/source/install.rst

@@ -134,7 +134,7 @@ The current minimum dependencies are:
 
 Cython is required to build from a git checkout but not to run or install from PyPI:
 
-* `Cython <https://cython.org/>`__ >= 0.29.26 is required to build the code from
+* `Cython <https://cython.org/>`__ >= 0.29.33 is required to build the code from
   github but not from a source distribution.
 
 Given the long release cycle, statsmodels follows a loose time-based policy for

pyproject.toml

@@ -2,14 +2,15 @@
 # These are strictly build requirements. Runtime requirements are listed in
 # INSTALL_REQUIRES in setup.py
 requires = [
-    "setuptools>=59.2.0",
-    "cython>=0.29.26,<3",  # Sync with CYTHON_MIN_VER in setup
+    "setuptools>=69.0.2; python_version>='3.12'",
+    "setuptools>=63.4.3",
+    "cython>=0.29.33,<4",  # Sync with CYTHON_MIN_VER in setup
     # Workaround for oldest supported numpy using 1.21.6, but SciPy 1.9.2+ requiring 1.22.3+
     "oldest-supported-numpy; python_version!='3.10' or platform_system!='Windows' or platform_python_implementation=='PyPy'",
     "numpy>=1.22.3,<2; python_version=='3.10' and platform_system=='Windows' and platform_python_implementation != 'PyPy'",
     "numpy<2; python_version>='3.12'",
     "scipy>=1.4",
-    "setuptools_scm[toml]>=7,<8"
+    "setuptools_scm[toml]>=8,<9"
 ]
 build-backend = "setuptools.build_meta"
 

requirements-dev.txt

@@ -1,6 +1,6 @@
 # build
-cython>=0.29.28,<3.0.0
-setuptools_scm[toml]~=7.0
+cython>=0.29.33,<4.0.0
+setuptools_scm[toml]~=8.0
 oldest-supported-numpy>=2022.4.18
 
 # run
@@ -14,6 +14,7 @@ joblib
 pytest>=7.3.0
 pytest-randomly
 pytest-xdist
+pytest-cov
 
 # Pin on Win32
 pywinpty; os_name == "nt"

requirements.txt

@@ -4,5 +4,5 @@ numpy >=1.18,<2  # released December 2019
 scipy>=1.4,!=1.9.2  # released December 2019
 scipy>=1.4,!=1.9.2; sys_platform == "win32"  # Blocked  1.9.2 due to Windows issues
 pandas>=1.0,!=2.1.0  # released January 2020, 2.1.0 blocked due to bug
-patsy>=0.5.2  # released January 2018
+patsy>=0.5.4  # released January 2018
 packaging>=21.3  # released Nov 2021

setup.cfg

@@ -23,7 +23,6 @@ filterwarnings =
     ignore:The --strict option is deprecated:pytest.PytestDeprecationWarning:
     error:genfromdta:FutureWarning
     error:StataReader:FutureWarning
-    error:Estimation of VARMA:statsmodels.tools.sm_exceptions.EstimationWarning
     error:Care should be used:UserWarning
     error::statsmodels.tools.sm_exceptions.HypothesisTestWarning
     error::statsmodels.tools.sm_exceptions.SpecificationWarning
@@ -79,6 +78,21 @@ filterwarnings =
     error:A value is trying to be set on a copy::
     error:Conversion of an array with ndim:DeprecationWarning:
     error:Series.__getitem__ treating keys:FutureWarning:
+    error:'Y' is deprecated, please use 'YE' instead:FutureWarning
+    error:'A' is deprecated, please use 'YE' instead:FutureWarning
+    error:'H' is deprecated and will:FutureWarning
+    error:'M' is deprecated, please use:FutureWarning
+    error:'Q' is deprecated, please use 'QE' instead:FutureWarning
+    error:'Q-DEC' is deprecated, please use 'QE-DEC' instead:FutureWarning
+    error:'Q-JAN' is deprecated, please use 'QE-JAN' instead:FutureWarning
+    error:'BQ-MAR' is deprecated, please use 'BQE-MAR' instead:FutureWarning
+    error:'AS-MAR' is deprecated and will be removed in a future version:FutureWarning
+    error:Downcasting object dtype arrays on:FutureWarning
+    error:The previous implementation of stack is deprecated:FutureWarning
+    error:Series.__setitem__ treating keys as positions is deprecated:FutureWarning
+    error:The provided callable:FutureWarning
+    error:divide by zero encountered in log1p:RuntimeWarning
+
 markers =
     example: mark a test that runs example code
     matplotlib: mark a test that requires matplotlib

setup.py

@@ -8,6 +8,7 @@
 from setuptools import Command, Extension, find_packages, setup
 from setuptools.dist import Distribution
 
+from packaging.version import parse
 from collections import defaultdict
 import fnmatch
 import inspect
@@ -25,15 +26,16 @@
     FORCE_C = int(os.environ.get("SM_FORCE_C", 0))
     if FORCE_C:
         raise ImportError("Force import error for testing")
-    from Cython import Tempita
+    from Cython import Tempita, __version__ as cython_version
     from Cython.Build import cythonize
     from Cython.Distutils import build_ext
 
     HAS_CYTHON = True
+    CYTHON_3 = parse(cython_version) >= parse("3.0")
 except ImportError:
     from setuptools.command.build_ext import build_ext
 
-    HAS_CYTHON = False
+    HAS_CYTHON = CYTHON_3 = False
 
 try:
     import numpy  # noqa: F401
@@ -57,7 +59,7 @@
     for line in req.readlines():
         DEVELOP_REQUIRES.append(line.split("#")[0].strip())
 
-CYTHON_MIN_VER = "0.29.26"  # released 2020
+CYTHON_MIN_VER = "0.29.33"  # released January 2023
 
 EXTRAS_REQUIRE = {
     "build": ["cython>=" + CYTHON_MIN_VER],
@@ -346,8 +348,9 @@ def process_tempita(source_name):
     for extension in extensions:
         requires_math = extension.name in EXT_REQUIRES_NUMPY_MATH_LIBS
         update_extension(extension, requires_math=requires_math)
-
 if HAS_CYTHON:
+    if CYTHON_3:
+        COMPILER_DIRECTIVES["cpow"] = True
     extensions = cythonize(
         extensions,
         compiler_directives=COMPILER_DIRECTIVES,

statsmodels/base/tests/test_optimize.py

@@ -85,7 +85,13 @@ def test_full_output_false(reset_randomstate):
 
         else:
             xopt, retvals = func(
-                dummy_func, dummy_score, [1], (), {}, full_output=False, disp=0
+                dummy_func,
+                dummy_score,
+                [1.0],
+                (),
+                {},
+                full_output=False,
+                disp=0
             )
         assert_(retvals is None)
         if method == "powell" and SP_LT_15:
@@ -102,7 +108,7 @@ def test_full_output(reset_randomstate):
             xopt, retvals = func(
                 dummy_func,
                 dummy_score,
-                [1],
+                [1.0],
                 (),
                 {},
                 hess=dummy_hess,
@@ -112,7 +118,13 @@ def test_full_output(reset_randomstate):
 
         else:
             xopt, retvals = func(
-                dummy_func, dummy_score, [1], (), {}, full_output=True, disp=0
+                dummy_func,
+                dummy_score,
+                [1.0],
+                (),
+                {},
+                full_output=True,
+                disp=0
             )
 
         assert_(retvals is not None)
@@ -130,7 +142,7 @@ def test_minimize_scipy_slsqp():
     xopt, _ = func(
         dummy_bounds_constraint_func,
         None,
-        (2, 0),
+        (2.0, 0.0),
         (),
         {
             "min_method": "SLSQP",

statsmodels/compat/pandas.py

@@ -32,11 +32,17 @@
     "call_cached_func",
     "PD_LT_1_4",
     "PD_LT_2",
+    "MONTH_END",
+    "QUARTER_END",
+    "YEAR_END",
+    "FUTURE_STACK",
 ]
 
 version = parse(pd.__version__)
 
-PD_LT_1_0_0 = version < Version("1.0.0")
+PD_LT_2_2_0 = version < Version("2.1.99")
+PD_LT_2_1_0 = version < Version("2.0.99")
+PD_LT_1_0_0 = version < Version("0.99.0")
 PD_LT_1_4 = version < Version("1.3.99")
 PD_LT_2 = version < Version("1.9.99")
 
@@ -174,3 +180,9 @@ def call_cached_func(cached_prop, *args, **kwargs):
 
 def get_cached_doc(cached_prop) -> Optional[str]:
     return get_cached_func(cached_prop).__doc__
+
+
+MONTH_END = "M" if PD_LT_2_2_0 else "ME"
+QUARTER_END = "Q" if PD_LT_2_2_0 else "QE"
+YEAR_END = "Y" if PD_LT_2_2_0 else "YE"
+FUTURE_STACK = {} if PD_LT_2_1_0 else {"future_stack": True}

statsmodels/discrete/tests/test_discrete.py

@@ -2586,16 +2586,15 @@ def test_cov_confint_pandas():
     assert isinstance(ci.index, pd.MultiIndex)
 
 
-def test_t_test():
+def test_mlogit_t_test():
     # GH669, check t_test works in multivariate model
-    data = load_anes96()
+    data = sm.datasets.anes96.load()
     exog = sm.add_constant(data.exog, prepend=False)
     res1 = sm.MNLogit(data.endog, exog).fit(disp=0)
     r = np.ones(res1.cov_params().shape[0])
     t1 = res1.t_test(r)
     f1 = res1.f_test(r)
 
-    data = sm.datasets.anes96.load()
     exog = sm.add_constant(data.exog, prepend=False)
     endog, exog = np.asarray(data.endog), np.asarray(exog)
     res2 = sm.MNLogit(endog, exog).fit(disp=0)
@@ -2604,3 +2603,20 @@ def test_t_test():
 
     assert_allclose(t1.effect, t2.effect)
     assert_allclose(f1.statistic, f2.statistic)
+
+    tt = res1.t_test(np.eye(np.size(res2.params)))
+    assert_allclose(tt.tvalue.reshape(6,6, order="F"), res1.tvalues.to_numpy())
+    tt = res2.t_test(np.eye(np.size(res2.params)))
+    assert_allclose(tt.tvalue.reshape(6,6, order="F"), res2.tvalues)
+
+    wt = res1.wald_test(np.eye(np.size(res2.params))[0], scalar=True)
+    assert_allclose(wt.pvalue, res1.pvalues.to_numpy()[0, 0])
+
+
+    tt = res1.t_test("y1_logpopul")
+    wt = res1.wald_test("y1_logpopul", scalar=True)
+    assert_allclose(tt.pvalue, wt.pvalue)
+
+    wt = res1.wald_test("y1_logpopul, y2_logpopul", scalar=True)
+    # regression test
+    assert_allclose(wt.statistic, 5.68660562, rtol=1e-8)

statsmodels/discrete/tests/test_sandwich_cov.py

@@ -9,6 +9,8 @@
 import os
 import numpy as np
 import pandas as pd
+import pytest
+
 import statsmodels.discrete.discrete_model as smd
 from statsmodels.genmod.generalized_linear_model import GLM
 from statsmodels.genmod import families
@@ -499,6 +501,21 @@ def test_score_test(self):
         res_lm2 = res2.score_test(exog_extra, cov_type='nonrobust')
         assert_allclose(np.hstack(res_lm1), np.hstack(res_lm2), rtol=5e-7)
 
+    def test_margeff(self):
+        if (isinstance(self.res2.model, OLS) or
+                hasattr(self.res1.model, "offset")):
+            pytest.skip("not available yet")
+
+        marg1 = self.res1.get_margeff()
+        marg2 = self.res2.get_margeff()
+        assert_allclose(marg1.margeff, marg2.margeff, rtol=1e-10)
+        assert_allclose(marg1.margeff_se, marg2.margeff_se, rtol=1e-10)
+
+        marg1 = self.res1.get_margeff(count=True, dummy=True)
+        marg2 = self.res2.get_margeff(count=True, dummy=True)
+        assert_allclose(marg1.margeff, marg2.margeff, rtol=1e-10)
+        assert_allclose(marg1.margeff_se, marg2.margeff_se, rtol=1e-10)
+
 
 class TestGLMPoisson(CheckDiscreteGLM):