diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e2d48582d14..34a71cdbc7d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Versions are `MAJOR.PATCH`. # Changelog + Salt 3004.1 (2022-02-16) ======================== @@ -207,6 +208,36 @@ Added - Allow a user to use the aptpkg.py module without installing python-apt. (#60818) +Salt 3003.5 (2022-07-05) +======================== + +Fixed +----- + +- Update Markup and contextfunction imports for jinja versions >=3.1. (#61848) +- Fix bug in tcp transport (#61865) +- Make sure the correct key is being used when verifying or validating communication, eg. when a Salt syndic is involved use syndic_master.pub and when a Salt minion is involved use minion_master.pub. (#61868) + + +Security +-------- + +- Fixed PAM auth to reject auth attempt if user account is locked. (cve-2022-22967) + + +Salt 3003.4 (2022-02-25) +======================== + +Security +-------- + +- Sign authentication replies to prevent MiTM (cve-2022-22935) +- Prevent job and fileserver replays (cve-2022-22936) +- Sign pillar data to prevent MiTM attacks. (cve-2202-22934) +- Fixed targeting bug, especially visible when using syndic and user auth. (CVE-2022-22941) (#60413) +- Fix denial of service in junos ifconfig output parsing. + + Salt 3003.3 (2021-08-20) ======================== @@ -427,6 +458,37 @@ Added metadata for a package by extracting library requirement information from the binary ELF files in the package. (#59569) + +Salt 3002.9 (2022-05-25) +======================== + +Fixed +----- + +- Fixed an error when running on CentOS Stream 8. (#59161) +- Fix bug in tcp transport (#61865) +- Make sure the correct key is being used when verifying or validating communication, eg. when a Salt syndic is involved use syndic_master.pub and when a Salt minion is involved use minion_master.pub. (#61868) + + +Security +-------- + +- Fixed PAM auth to reject auth attempt if user account is locked. (cve-2022-22967) + + +Salt 3002.8 (2022-02-25) +======================== + +Security +-------- + +- Sign authentication replies to prevent MiTM (cve-2020-22935) +- Sign pillar data to prevent MiTM attacks. (cve-2022-22934) +- Prevent job and fileserver replays (cve-2022-22936) +- Fixed targeting bug, especially visible when using syndic and user auth. (CVE-2022-22941) (#60413) + + + Salt 3002.7 (2021-08-20) ======================== @@ -443,6 +505,7 @@ Security Additionally, an audit and a tool was put in place, ``bandit``, to address similar issues througout the code base, and prevent them. (CVE-2021-31607) - Ensure that sourced file is cached using its hash name (cve-2021-21996) + Salt 3002.6 (2021-03-10) ======================== @@ -451,6 +514,7 @@ Changed - Store git sha in salt/_version.py when installing from a tag so it can be found if needed later. (#59137) + Fixed ----- diff --git a/doc/man/salt-api.1 b/doc/man/salt-api.1 index fa9c708c7837..ad179ebc87f9 100644 --- a/doc/man/salt-api.1 +++ b/doc/man/salt-api.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-API" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-API" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-api \- salt-api Command . diff --git a/doc/man/salt-call.1 b/doc/man/salt-call.1 index a0aadbdb8ba3..baa2527310cc 100644 --- a/doc/man/salt-call.1 +++ b/doc/man/salt-call.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-CALL" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-CALL" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-call \- salt-call Documentation . diff --git a/doc/man/salt-cloud.1 b/doc/man/salt-cloud.1 index b12faa75d186..7bc7ea020b47 100644 --- a/doc/man/salt-cloud.1 +++ b/doc/man/salt-cloud.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-CLOUD" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-CLOUD" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-cloud \- Salt Cloud Command . diff --git a/doc/man/salt-cp.1 b/doc/man/salt-cp.1 index debd293f2be6..249b311c4778 100644 --- a/doc/man/salt-cp.1 +++ b/doc/man/salt-cp.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-CP" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-CP" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-cp \- salt-cp Documentation . diff --git a/doc/man/salt-key.1 b/doc/man/salt-key.1 index 25f364a3c159..b5769e92e50e 100644 --- a/doc/man/salt-key.1 +++ b/doc/man/salt-key.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-KEY" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-KEY" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-key \- salt-key Documentation . diff --git a/doc/man/salt-master.1 b/doc/man/salt-master.1 index e3251582714f..3169db7bb518 100644 --- a/doc/man/salt-master.1 +++ b/doc/man/salt-master.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-MASTER" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-MASTER" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-master \- salt-master Documentation . diff --git a/doc/man/salt-minion.1 b/doc/man/salt-minion.1 index b25bf8d86762..25eeb950eaa6 100644 --- a/doc/man/salt-minion.1 +++ b/doc/man/salt-minion.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-MINION" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-MINION" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-minion \- salt-minion Documentation . diff --git a/doc/man/salt-proxy.1 b/doc/man/salt-proxy.1 index 740f441ecfe4..01ebb8726106 100644 --- a/doc/man/salt-proxy.1 +++ b/doc/man/salt-proxy.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-PROXY" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-PROXY" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-proxy \- salt-proxy Documentation . diff --git a/doc/man/salt-run.1 b/doc/man/salt-run.1 index da5985e2ad26..1d65a41209af 100644 --- a/doc/man/salt-run.1 +++ b/doc/man/salt-run.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-RUN" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-RUN" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-run \- salt-run Documentation . diff --git a/doc/man/salt-ssh.1 b/doc/man/salt-ssh.1 index ae34c6ea7129..ff81e43f0a83 100644 --- a/doc/man/salt-ssh.1 +++ b/doc/man/salt-ssh.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-SSH" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-SSH" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-ssh \- salt-ssh Documentation . diff --git a/doc/man/salt-syndic.1 b/doc/man/salt-syndic.1 index 5ccb6fb3a603..84d6d42d59f1 100644 --- a/doc/man/salt-syndic.1 +++ b/doc/man/salt-syndic.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT-SYNDIC" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT-SYNDIC" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt-syndic \- salt-syndic Documentation . diff --git a/doc/man/salt.1 b/doc/man/salt.1 index 108d38ac0943..2d97b1b7a802 100644 --- a/doc/man/salt.1 +++ b/doc/man/salt.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME salt \- salt . diff --git a/doc/man/salt.7 b/doc/man/salt.7 index 618c63080eda..c7dd9e110abd 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SALT" "7" "Feb 16, 2022" "3004.1" "Salt" +.TH "SALT" "7" "May 12, 2022" "3004.2" "Salt" .SH NAME salt \- Salt Documentation . diff --git a/doc/man/spm.1 b/doc/man/spm.1 index d0f9e78929bf..f5be1daf70aa 100644 --- a/doc/man/spm.1 +++ b/doc/man/spm.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "SPM" "1" "Feb 16, 2022" "3004.1" "Salt" +.TH "SPM" "1" "May 12, 2022" "3004.2" "Salt" .SH NAME spm \- Salt Package Manager Command . diff --git a/doc/topics/releases/3002.8.rst b/doc/topics/releases/3002.8.rst new file mode 100644 index 000000000000..eddef98ff200 --- /dev/null +++ b/doc/topics/releases/3002.8.rst @@ -0,0 +1,34 @@ +.. _release-3002-8: + +======================== +Salt 3002.8 (2022-02-25) +======================== + +Version 3002.8 is a CVE security fix release for :ref:`3002 `. + + +Important notice about upgrading +-------------------------------- + +Version 3002.8 is a security release. 3002.8 minions are not able to +communicate with masters older than 3002.8. You must upgrade your masters +before upgrading minions. + + +Minion authentication security +------------------------------ + +Authentication between masters and minions rely on public/private key +encryption and message signing. To secure minion authentication before you must +pre-seed the master's public key on minions. To pre-seed the minions' master +key, place a copy of the master's public key in the minion's pki directory as +``minion_master.pub``. + + +Security +-------- + +- Sign authentication replies to prevent MiTM (cve-2020-22935) +- Sign pillar data to prevent MiTM attacks. (cve-2022-22934) +- Prevent job and fileserver replays (cve-2022-22936) +- Fixed targeting bug, especially visible when using syndic and user auth. (CVE-2022-22941) (#60413) diff --git a/doc/topics/releases/3002.9.rst b/doc/topics/releases/3002.9.rst new file mode 100644 index 000000000000..2de692db4a35 --- /dev/null +++ b/doc/topics/releases/3002.9.rst @@ -0,0 +1,21 @@ +.. _release-3002-9: + + +======================== +Salt 3002.9 (2022-05-25) +======================== + +Version 3002.9 is a CVE security fix release for :ref:`3002 `. + +Fixed +----- + +- Fixed an error when running on CentOS Stream 8. (#59161) +- Fix bug in tcp transport (#61865) +- Make sure the correct key is being used when verifying or validating communication, eg. when a Salt syndic is involved use syndic_master.pub and when a Salt minion is involved use minion_master.pub. (#61868) + + +Security +-------- + +- Fixed PAM auth to reject auth attempt if user account is locked. (cve-2022-22967) diff --git a/doc/topics/releases/3003.4.rst b/doc/topics/releases/3003.4.rst new file mode 100644 index 000000000000..a01da65f643e --- /dev/null +++ b/doc/topics/releases/3003.4.rst @@ -0,0 +1,35 @@ +.. _release-3003-4: + +======================== +Salt 3003.4 (2022-02-25) +======================== + +Version 3003.4 is a CVE security fix release for :ref:`3003 `. + + +Important notice about upgrading +-------------------------------- + +Version 3003.4 is a security release. 3003.4 minions are not able to +communicate with masters older than 3003.4. You must upgrade your masters +before upgrading minions. + + +Minion authentication security +------------------------------ + +Authentication between masters and minions rely on public/private key +encryption and message signing. To secure minion authentication before you must +pre-seed the master's public key on minions. To pre-seed the minions' master +key, place a copy of the master's public key in the minion's pki directory as +``minion_master.pub``. + + +Security +-------- + +- Sign authentication replies to prevent MiTM (cve-2022-22935) +- Prevent job and fileserver replays (cve-2022-22936) +- Sign pillar data to prevent MiTM attacks. (cve-2202-22934) +- Fixed targeting bug, especially visible when using syndic and user auth. (CVE-2022-22941) (#60413) +- Fix denial of service in junos ifconfig output parsing. diff --git a/doc/topics/releases/3003.5.rst b/doc/topics/releases/3003.5.rst new file mode 100644 index 000000000000..2e2462c79b2b --- /dev/null +++ b/doc/topics/releases/3003.5.rst @@ -0,0 +1,21 @@ +.. _release-3003-5: + +======================== +Salt 3003.5 (2022-07-05) +======================== + +Version 3003.5 is a CVE security fix release for :ref:`3003 `. + +Fixed +----- + +- Update Markup and contextfunction imports for jinja versions >=3.1. (#61848) +- Fix bug in tcp transport (#61865) +- Make sure the correct key is being used when verifying or validating communication, eg. when a Salt syndic is involved use syndic_master.pub and when a Salt minion is involved use minion_master.pub. (#61868) + + +Security +-------- + +- Fixed PAM auth to reject auth attempt if user account is locked. (cve-2022-22967) + diff --git a/doc/topics/releases/3004.2.rst b/doc/topics/releases/3004.2.rst new file mode 100644 index 000000000000..bc7909f21cdd --- /dev/null +++ b/doc/topics/releases/3004.2.rst @@ -0,0 +1,20 @@ +.. _release-3004-2: + +========================= +Salt 3004.2 Release Notes +========================= + +Version 3004.2 is a CVE security fix release for :ref:`3004 `. + +Fixed +----- + +- Expand environment variables in the root_dir registry key (#61445) +- Update Markup and contextfunction imports for jinja versions >=3.1. (#61848) +- Fix bug in tcp transport (#61865) +- Make sure the correct key is being used when verifying or validating communication, eg. when a Salt syndic is involved use syndic_master.pub and when a Salt minion is involved use minion_master.pub. (#61868) + +Security +-------- + +- Fixed PAM auth to reject auth attempt if user account is locked. (cve-2022-22967) diff --git a/pkg/osx/pkg-scripts/postinstall b/pkg/osx/pkg-scripts/postinstall index 74898ca215a5..90fff0755516 100755 --- a/pkg/osx/pkg-scripts/postinstall +++ b/pkg/osx/pkg-scripts/postinstall @@ -29,6 +29,7 @@ BIN_DIR="$INSTALL_DIR/bin" CONFIG_DIR="/etc/salt" TEMP_DIR="/tmp" SBIN_DIR="/usr/local/sbin" +PY_DOT_VERSION="3.7.12" ############################################################################### # Set up logging and error handling diff --git a/pkg/osx/sign_binaries.sh b/pkg/osx/sign_binaries.sh index 8b5fc55f8a20..d1b3f5aa3b05 100755 --- a/pkg/osx/sign_binaries.sh +++ b/pkg/osx/sign_binaries.sh @@ -78,6 +78,14 @@ install_name_tool $INSTALL_DIR/bin/python${PY_VERSION}m \ -add_rpath $INSTALL_DIR/.pyenv/versions/$PY_DOT_VERSION/lib \ -add_rpath $INSTALL_DIR/.pyenv/versions/$PY_DOT_VERSION/openssl/lib || echo "already present" +################################################################################ +# Add rpath to the Python binaries before signing +################################################################################ +echo "**** Setting rpath in binaries" +install_name_tool $INSTALL_DIR/bin/python3.7m \ + -add_rpath $INSTALL_DIR/.pyenv/versions/3.7.12/lib \ + -add_rpath $INSTALL_DIR/.pyenv/versions/3.7.12/openssl/lib || echo "already present" + ################################################################################ # Sign python binaries in `bin` and `lib` ################################################################################ diff --git a/requirements/static/ci/py3.7/windows.txt b/requirements/static/ci/py3.7/windows.txt index 9ee7d156177f..53bfe5e5b28e 100644 --- a/requirements/static/ci/py3.7/windows.txt +++ b/requirements/static/ci/py3.7/windows.txt @@ -403,6 +403,7 @@ typing-extensions==3.10.0.0 # yarl urllib3==1.26.6 # via + # -r requirements/windows.txt # botocore # kubernetes # python-etcd diff --git a/requirements/static/ci/py3.8/windows.txt b/requirements/static/ci/py3.8/windows.txt index 93ff5533514b..76c6e91c6b94 100644 --- a/requirements/static/ci/py3.8/windows.txt +++ b/requirements/static/ci/py3.8/windows.txt @@ -389,6 +389,7 @@ typing-extensions==4.2.0 # pytest-system-statistics urllib3==1.26.6 # via + # -r requirements/windows.txt # botocore # kubernetes # python-etcd diff --git a/requirements/static/ci/py3.9/windows.txt b/requirements/static/ci/py3.9/windows.txt index bad09cf2b0f8..8b8deabc8c7c 100644 --- a/requirements/static/ci/py3.9/windows.txt +++ b/requirements/static/ci/py3.9/windows.txt @@ -389,6 +389,7 @@ typing-extensions==4.2.0 # pytest-system-statistics urllib3==1.26.6 # via + # -r requirements/windows.txt # botocore # kubernetes # python-etcd diff --git a/requirements/static/pkg/py3.7/windows.txt b/requirements/static/pkg/py3.7/windows.txt index c31fe29e5ac6..0dbc68c658de 100644 --- a/requirements/static/pkg/py3.7/windows.txt +++ b/requirements/static/pkg/py3.7/windows.txt @@ -130,7 +130,9 @@ typing-extensions==3.10.0.0 # gitpython # importlib-metadata urllib3==1.26.6 - # via requests + # via + # -r requirements/windows.txt + # requests wheel==0.36.2 # via -r requirements/windows.txt wmi==1.5.1 diff --git a/requirements/static/pkg/py3.8/windows.txt b/requirements/static/pkg/py3.8/windows.txt index af0db64c89dd..8ca7c0877874 100644 --- a/requirements/static/pkg/py3.8/windows.txt +++ b/requirements/static/pkg/py3.8/windows.txt @@ -126,7 +126,9 @@ tempora==4.1.1 timelib==0.2.5 # via -r requirements/windows.txt urllib3==1.26.6 - # via requests + # via + # -r requirements/windows.txt + # requests wheel==0.36.2 # via -r requirements/windows.txt wmi==1.5.1 diff --git a/requirements/static/pkg/py3.9/windows.txt b/requirements/static/pkg/py3.9/windows.txt index c5dcb96d97b8..c03852e0b690 100644 --- a/requirements/static/pkg/py3.9/windows.txt +++ b/requirements/static/pkg/py3.9/windows.txt @@ -126,7 +126,9 @@ tempora==4.1.1 timelib==0.2.5 # via -r requirements/windows.txt urllib3==1.26.6 - # via requests + # via + # -r requirements/windows.txt + # requests wheel==0.36.2 # via -r requirements/windows.txt wmi==1.5.1 diff --git a/requirements/windows.txt b/requirements/windows.txt index 7c8c032b3033..bc364d83c64c 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -27,6 +27,7 @@ python-gnupg>=0.4.7 requests>=2.25.1 setproctitle timelib>=0.2.5 +urllib3>=1.26.5 # Watchdog pulls in a GPL-3 package, argh, which cannot be shipped on the # windows distribution package. # diff --git a/salt/auth/pam.py b/salt/auth/pam.py index a9dde951498b..d91883b74326 100644 --- a/salt/auth/pam.py +++ b/salt/auth/pam.py @@ -209,7 +209,7 @@ def my_conv(n_messages, messages, p_response, app_data): retval = PAM_AUTHENTICATE(handle, 0) if retval == 0: - PAM_ACCT_MGMT(handle, 0) + retval = PAM_ACCT_MGMT(handle, 0) PAM_END(handle, 0) return retval == 0 diff --git a/salt/beacons/watchdog.py b/salt/beacons/watchdog.py index a1feea589741..01c3f7f82072 100644 --- a/salt/beacons/watchdog.py +++ b/salt/beacons/watchdog.py @@ -13,9 +13,12 @@ import salt.utils.beacons try: + # pylint: disable=no-name-in-module from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler + # pylint: enable=no-name-in-module + HAS_WATCHDOG = True except ImportError: HAS_WATCHDOG = False diff --git a/salt/channel/client.py b/salt/channel/client.py index ad1e00937dcb..c92aba4b6584 100644 --- a/salt/channel/client.py +++ b/salt/channel/client.py @@ -135,6 +135,9 @@ def __init__(self, opts, transport, auth, **kwargs): self.opts = dict(opts) self.transport = transport self.auth = auth + self.master_pubkey_path = None + if self.auth: + self.master_pubkey_path = os.path.join(self.opts["pki_dir"], self.auth.mpub) self._closing = False @property @@ -188,10 +191,7 @@ def crypted_transfer_decode_dictentry( signed_msg = pcrypt.loads(ret[dictkey]) # Validate the master's signature. - master_pubkey_path = os.path.join(self.opts["pki_dir"], "minion_master.pub") - if not salt.crypt.verify_signature( - master_pubkey_path, signed_msg["data"], signed_msg["sig"] - ): + if not self.verify_signature(signed_msg["data"], signed_msg["sig"]): raise salt.crypt.AuthenticationError( "Pillar payload signature failed to validate." ) @@ -206,6 +206,9 @@ def crypted_transfer_decode_dictentry( raise salt.crypt.AuthenticationError("Pillar nonce verification failed.") raise salt.ext.tornado.gen.Return(data["pillar"]) + def verify_signature(self, data, sig): + return salt.crypt.verify_signature(self.master_pubkey_path, data, sig) + @salt.ext.tornado.gen.coroutine def _crypted_transfer(self, load, timeout=60, raw=False): """ @@ -367,6 +370,7 @@ def __init__(self, opts, transport, auth, io_loop=None): self._closing = False self._reconnected = False self.event = salt.utils.event.get_event("minion", opts=self.opts, listen=False) + self.master_pubkey_path = os.path.join(self.opts["pki_dir"], self.auth.mpub) @property def crypt(self): @@ -379,6 +383,7 @@ def connect(self): """ try: if not self.auth.authenticated: + log.error("WTF %r %r", self.auth.authenticated, self.auth.authenticate) yield self.auth.authenticate() # if this is changed from the default, we assume it was intentional if int(self.opts.get("publish_port", 4506)) != 4506: @@ -536,9 +541,8 @@ def _verify_master_signature(self, payload): ) # Verify that the signature is valid - master_pubkey_path = os.path.join(self.opts["pki_dir"], "minion_master.pub") if not salt.crypt.verify_signature( - master_pubkey_path, payload["load"], payload.get("sig") + self.master_pubkey_path, payload["load"], payload.get("sig") ): raise salt.crypt.AuthenticationError( "Message signature failed to validate." diff --git a/salt/engines/docker_events.py b/salt/engines/docker_events.py index bc9669caf77c..66b95c46b0bb 100644 --- a/salt/engines/docker_events.py +++ b/salt/engines/docker_events.py @@ -9,8 +9,8 @@ import salt.utils.json try: - import docker # pylint: disable=import-error - import docker.utils # pylint: disable=import-error + import docker # pylint: disable=import-error,no-name-in-module + import docker.utils # pylint: disable=import-error,no-name-in-module HAS_DOCKER_PY = True except ImportError: diff --git a/salt/master.py b/salt/master.py index dad2d838652e..9c06a52c1cd7 100644 --- a/salt/master.py +++ b/salt/master.py @@ -737,7 +737,10 @@ def start(self): # must be after channels log.info("Creating master maintenance process") self.process_manager.add_process( - Maintenance, args=(self.opts,), name="Maintenance" + Maintenance, + args=(self.opts,), + kwargs={"master_secrets": SMaster.secrets}, + name="Maintenance", ) if self.opts.get("event_return"): diff --git a/salt/modules/ddns.py b/salt/modules/ddns.py index f360812c7032..943940b795ab 100644 --- a/salt/modules/ddns.py +++ b/salt/modules/ddns.py @@ -32,8 +32,8 @@ try: import dns.query - import dns.update - import dns.tsigkeyring + import dns.update # pylint: disable=no-name-in-module + import dns.tsigkeyring # pylint: disable=no-name-in-module dns_support = True except ImportError as e: diff --git a/salt/modules/kubernetesmod.py b/salt/modules/kubernetesmod.py index 87d09fb9e563..aa3a463c4e91 100644 --- a/salt/modules/kubernetesmod.py +++ b/salt/modules/kubernetesmod.py @@ -59,6 +59,7 @@ import salt.utils.yaml from salt.exceptions import CommandExecutionError, TimeoutError +# pylint: disable=import-error,no-name-in-module try: import kubernetes # pylint: disable=import-self import kubernetes.client @@ -78,6 +79,7 @@ HAS_LIBS = True except ImportError: HAS_LIBS = False +# pylint: enable=import-error,no-name-in-module log = logging.getLogger(__name__) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 485bb9dfdc1d..6d083968427d 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -5778,7 +5778,6 @@ def get_hypervisor(): def _is_bhyve_hyper(): - sysctl_cmd = "sysctl hw.vmm.create" vmm_enabled = False try: stdout = subprocess.Popen( diff --git a/salt/runners/bgp.py b/salt/runners/bgp.py index 2a1eee09b6aa..4e23f576308c 100644 --- a/salt/runners/bgp.py +++ b/salt/runners/bgp.py @@ -103,7 +103,10 @@ from netaddr import IPNetwork from netaddr import IPAddress - from napalm.base import helpers as napalm_helpers # pylint: disable=unused-import + # pylint: disable=unused-import,no-name-in-module + from napalm.base import helpers as napalm_helpers + + # pylint: enable=unused-import,no-name-in-module HAS_NAPALM = True except ImportError: diff --git a/salt/runners/ddns.py b/salt/runners/ddns.py index ebabfaa82263..96fef7fad65e 100644 --- a/salt/runners/ddns.py +++ b/salt/runners/ddns.py @@ -19,8 +19,8 @@ HAS_LIBS = False try: import dns.query - import dns.update - import dns.tsigkeyring + import dns.update # pylint: disable=no-name-in-module + import dns.tsigkeyring # pylint: disable=no-name-in-module HAS_LIBS = True except ImportError: diff --git a/salt/runners/net.py b/salt/runners/net.py index 166a8ee25706..d8dbd679f198 100644 --- a/salt/runners/net.py +++ b/salt/runners/net.py @@ -74,8 +74,12 @@ try: from netaddr import IPNetwork # netaddr is already required by napalm from netaddr.core import AddrFormatError + + # pylint: disable=no-name-in-module from napalm.base import helpers as napalm_helpers + # pylint: enable=no-name-in-module + HAS_NAPALM = True except ImportError: HAS_NAPALM = False diff --git a/salt/states/netntp.py b/salt/states/netntp.py index 3ca7c28eff65..27a8b09713f3 100644 --- a/salt/states/netntp.py +++ b/salt/states/netntp.py @@ -38,7 +38,7 @@ HAS_NETADDR = False try: - import dns.resolver + import dns.resolver # pylint: disable=no-name-in-module HAS_DNSRESOLVER = True except ImportError: diff --git a/salt/utils/dns.py b/salt/utils/dns.py index 0aa5342385a8..dc0d680c7739 100644 --- a/salt/utils/dns.py +++ b/salt/utils/dns.py @@ -33,7 +33,7 @@ # Integrations try: - import dns.resolver + import dns.resolver # pylint: disable=no-name-in-module HAS_DNSPYTHON = True except ImportError: diff --git a/salt/utils/mako.py b/salt/utils/mako.py index 4cc4e25c1d8c..c45b8a3932dd 100644 --- a/salt/utils/mako.py +++ b/salt/utils/mako.py @@ -3,10 +3,10 @@ """ try: - from mako.lookup import ( - TemplateCollection, - TemplateLookup, - ) # pylint: disable=import-error,3rd-party-module-not-gated + # pylint: disable=import-error,3rd-party-module-not-gated,no-name-in-module + from mako.lookup import TemplateCollection, TemplateLookup + + # pylint: enable=import-error,3rd-party-module-not-gated,no-name-in-module HAS_MAKO = True except ImportError: diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 60796c1d4913..5459962b632a 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -488,6 +488,11 @@ def _realpath_windows(path): base = os.path.abspath(os.path.sep.join([base, part])) else: base = part + # Python 3.8 added support for directory junctions which prefixes the + # return with `\\?\`. We need to strip that off. + # https://docs.python.org/3/library/os.html#os.readlink + if base.startswith("\\\\?\\"): + base = base[4:] return base diff --git a/tests/pytests/functional/states/test_pkgrepo.py b/tests/pytests/functional/states/test_pkgrepo.py new file mode 100644 index 000000000000..82a6f093983f --- /dev/null +++ b/tests/pytests/functional/states/test_pkgrepo.py @@ -0,0 +1,40 @@ +import platform + +import pytest +import salt.utils.files + + +@pytest.mark.skipif( + not any([x for x in ["ubuntu", "debian"] if x in platform.platform()]), + reason="Test only for debian based platforms", +) +def test_adding_repo_file(states, tmp_path): + """ + test adding a repo file using pkgrepo.managed + """ + repo_file = str(tmp_path / "stable-binary.list") + repo_content = "deb http://www.deb-multimedia.org stable main" + ret = states.pkgrepo.managed(name=repo_content, file=repo_file, clean_file=True) + with salt.utils.files.fopen(repo_file, "r") as fp: + file_content = fp.read() + assert file_content.strip() == repo_content + + +@pytest.mark.skipif( + not any([x for x in ["ubuntu", "debian"] if x in platform.platform()]), + reason="Test only for debian based platforms", +) +def test_adding_repo_file_arch(states, tmp_path): + """ + test adding a repo file using pkgrepo.managed + and setting architecture + """ + repo_file = str(tmp_path / "stable-binary.list") + repo_content = "deb [arch=amd64 ] http://www.deb-multimedia.org stable main" + ret = states.pkgrepo.managed(name=repo_content, file=repo_file, clean_file=True) + with salt.utils.files.fopen(repo_file, "r") as fp: + file_content = fp.read() + assert ( + file_content.strip() + == "deb [arch=amd64] http://www.deb-multimedia.org stable main" + ) diff --git a/tests/pytests/functional/transport/server/test_req_channel.py b/tests/pytests/functional/transport/server/test_req_channel.py index 99253432b2bc..685b02b9dc0d 100644 --- a/tests/pytests/functional/transport/server/test_req_channel.py +++ b/tests/pytests/functional/transport/server/test_req_channel.py @@ -8,6 +8,7 @@ import salt.config import salt.exceptions import salt.ext.tornado.gen +import salt.log.setup import salt.master import salt.transport.client import salt.transport.server diff --git a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py index 9b6511c90d70..d72f87755f58 100644 --- a/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py +++ b/tests/pytests/functional/transport/zeromq/test_pub_server_channel.py @@ -1,6 +1,7 @@ import ctypes import logging import multiprocessing +import socket import time from concurrent.futures.thread import ThreadPoolExecutor @@ -11,8 +12,14 @@ import salt.exceptions import salt.ext.tornado.gen import salt.ext.tornado.ioloop +import salt.log.setup import salt.master +import salt.transport.client +import salt.transport.server +import salt.transport.tcp import salt.transport.zeromq +import salt.utils.msgpack +import salt.utils.platform import salt.utils.process import salt.utils.stringutils import zmq @@ -29,13 +36,22 @@ ] +class RecvError(Exception): + """ + Raised by the Collector's _recv method when there is a problem + getting publishes from to the publisher. + """ + + class Collector(salt.utils.process.SignalHandlingProcess): def __init__( - self, minion_config, pub_uri, aes_key, timeout=30, zmq_filtering=False + self, minion_config, interface, port, aes_key, timeout=300, zmq_filtering=False ): super().__init__() self.minion_config = minion_config - self.pub_uri = pub_uri + self.interface = interface + self.port = port + self.aes_key = aes_key self.timeout = timeout self.aes_key = aes_key self.hard_timeout = time.time() + timeout + 30 @@ -45,6 +61,16 @@ def __init__( self.stopped = multiprocessing.Event() self.started = multiprocessing.Event() self.running = multiprocessing.Event() + if salt.utils.msgpack.version >= (0, 5, 2): + # Under Py2 we still want raw to be set to True + msgpack_kwargs = {"raw": False} + else: + msgpack_kwargs = {"encoding": "utf-8"} + self.unpacker = salt.utils.msgpack.Unpacker(**msgpack_kwargs) + + @property + def transport(self): + return self.minion_config["transport"] def _rotate_secrets(self, now=None): salt.master.SMaster.secrets["aes"] = { @@ -61,46 +87,101 @@ def _rotate_secrets(self, now=None): "rotate_master_key": self._rotate_secrets, } - def run(self): - """ - Gather results until then number of seconds specified by timeout passes - without receiving a message - """ - ctx = zmq.Context() - sock = ctx.socket(zmq.SUB) - sock.setsockopt(zmq.LINGER, -1) - sock.setsockopt(zmq.SUBSCRIBE, b"") - sock.connect(self.pub_uri) + def _setup_listener(self): + if self.transport == "zeromq": + ctx = zmq.Context() + self.sock = ctx.socket(zmq.SUB) + self.sock.setsockopt(zmq.LINGER, -1) + self.sock.setsockopt(zmq.SUBSCRIBE, b"") + pub_uri = "tcp://{}:{}".format(self.interface, self.port) + self.sock.connect(pub_uri) + else: + end = time.time() + 60 + while True: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.connect((self.interface, self.port)) + except ConnectionRefusedError: + if time.time() >= end: + raise + time.sleep(1) + else: + break + self.sock = salt.ext.tornado.iostream.IOStream(sock) + + @salt.ext.tornado.gen.coroutine + def _recv(self): + if self.transport == "zeromq": + # test_zeromq_filtering requires catching the + # SaltDeserializationError in order to pass. + try: + payload = self.sock.recv(zmq.NOBLOCK) + serial_payload = salt.payload.loads(payload) + raise salt.ext.tornado.gen.Return(serial_payload) + except (zmq.ZMQError, salt.exceptions.SaltDeserializationError): + raise RecvError("ZMQ Error") + else: + for msg in self.unpacker: + raise salt.ext.tornado.gen.Return(msg["body"]) + byts = yield self.sock.read_bytes(8096, partial=True) + self.unpacker.feed(byts) + for msg in self.unpacker: + raise salt.ext.tornado.gen.Return(msg["body"]) + raise RecvError("TCP Error") + + @salt.ext.tornado.gen.coroutine + def _run(self, loop): + try: + self._setup_listener() + except Exception: # pylint: disable=broad-except + self.started.set() + log.exception("Failed to start listening") + return + self.started.set() last_msg = time.time() + serial = salt.payload.Serial(self.minion_config) crypticle = salt.crypt.Crypticle(self.minion_config, self.aes_key) - self.started.set() while True: curr_time = time.time() if time.time() > self.hard_timeout: + log.error("Hard timeout reaced in test collector!") break if curr_time - last_msg >= self.timeout: + log.error("Receive timeout reaced in test collector!") break try: - payload = sock.recv(zmq.NOBLOCK) - except zmq.ZMQError: - time.sleep(0.1) + payload = yield self._recv() + except RecvError: + time.sleep(0.01) else: try: - serial_payload = salt.payload.loads(payload) - payload = crypticle.loads(serial_payload["load"]) + payload = crypticle.loads(payload["load"]) if not payload: continue if "start" in payload: + log.info("Collector started") self.running.set() continue if "stop" in payload: + log.info("Collector stopped") break last_msg = time.time() self.results.append(payload["jid"]) except salt.exceptions.SaltDeserializationError: + log.error("Deserializer Error") if not self.zmq_filtering: log.exception("Failed to deserialize...") break + loop.stop() + + def run(self): + """ + Gather results until then number of seconds specified by timeout passes + without receiving a message + """ + loop = salt.ext.tornado.ioloop.IOLoop() + loop.add_callback(self._run, loop) + loop.start() def __enter__(self): self.manager.__enter__() @@ -152,7 +233,11 @@ def __init__(self, master_config, minion_config, **collector_kwargs): self.queue = multiprocessing.Queue() self.stopped = multiprocessing.Event() self.collector = Collector( - self.minion_config, self.pub_uri, self.aes_key, **self.collector_kwargs + self.minion_config, + self.master_config["interface"], + self.master_config["publish_port"], + self.aes_key, + **self.collector_kwargs ) def run(self): @@ -179,7 +264,8 @@ def close(self): if self.process_manager is None: return self.process_manager.terminate() - self.pub_server_channel.close() + if hasattr(self.pub_server_channel, "pub_close"): + self.pub_server_channel.pub_close() # Really terminate any process still left behind for pid in self.process_manager._process_map: terminate_process(pid=pid, kill_children=True, slow_stop=False) @@ -191,7 +277,7 @@ def publish(self, payload): def __enter__(self): self.start() self.collector.__enter__() - attempts = 30 + attempts = 300 while attempts > 0: self.publish({"tgt_type": "glob", "tgt": "*", "jid": -1, "start": True}) if self.collector.running.wait(1) is True: @@ -218,16 +304,24 @@ def __exit__(self, *args): log.info("The PubServerChannelProcess has terminated") +@pytest.fixture(params=["tcp", "zeromq"]) +def transport(request): + yield request.param + + @pytest.mark.skip_on_windows @pytest.mark.slow_test -def test_publish_to_pubserv_ipc(salt_master, salt_minion): +def test_publish_to_pubserv_ipc(salt_master, salt_minion, transport): """ Test sending 10K messags to ZeroMQPubServerChannel using IPC transport ZMQ's ipc transport not supported on Windows """ - opts = dict(salt_master.config.copy(), ipc_mode="ipc", pub_hwm=0) - with PubServerChannelProcess(opts, salt_minion.config.copy()) as server_channel: + opts = dict( + salt_master.config.copy(), ipc_mode="ipc", pub_hwm=0, transport=transport + ) + minion_opts = dict(salt_minion.config.copy(), transport=transport) + with PubServerChannelProcess(opts, minion_opts) as server_channel: send_num = 10000 expect = [] for idx in range(send_num): @@ -269,7 +363,6 @@ def _send_large(opts, sid, num=10, size=250000 * 3): } server_channel.publish(load) time.sleep(0.3) - time.sleep(3) server_channel.close_pub() opts = dict(salt_master.config.copy(), ipc_mode="tcp", pub_hwm=0) diff --git a/tests/pytests/integration/ssh/test_state.py b/tests/pytests/integration/ssh/test_state.py index ebacd170b04c..dfd0fdeea34b 100644 --- a/tests/pytests/integration/ssh/test_state.py +++ b/tests/pytests/integration/ssh/test_state.py @@ -121,8 +121,8 @@ def test_state_with_import_from_dir(salt_ssh_cli, nested_state_tree): ret = salt_ssh_cli.run( "--extra-filerefs=salt://foo/map.jinja", "state.apply", "foo" ) - assert ret.returncode == 0 - assert ret.data + assert ret.exitcode == 0 + assert ret.json @pytest.mark.slow_test diff --git a/tests/pytests/scenarios/compat/conftest.py b/tests/pytests/scenarios/compat/conftest.py index 69abb05e6d9f..45c49a352518 100644 --- a/tests/pytests/scenarios/compat/conftest.py +++ b/tests/pytests/scenarios/compat/conftest.py @@ -15,14 +15,16 @@ from tests.support.runtests import RUNTIME_VARS from tests.support.sminion import create_sminion -try: - import docker - from docker.errors import DockerException -except ImportError: - docker = None - - class DockerException(Exception): - pass +docker = pytest.importorskip("docker") +# pylint: disable=3rd-party-module-not-gated,no-name-in-module +from docker.errors import DockerException # isort:skip + +# pylint: enable=3rd-party-module-not-gated,no-name-in-module + +pytestmark = [ + pytest.mark.slow_test, + pytest.mark.skip_if_binaries_missing("docker"), +] log = logging.getLogger(__name__) diff --git a/tests/pytests/unit/auth/test_pam.py b/tests/pytests/unit/auth/test_pam.py new file mode 100644 index 000000000000..04457c79cb6a --- /dev/null +++ b/tests/pytests/unit/auth/test_pam.py @@ -0,0 +1,36 @@ +import pytest +import salt.auth.pam +from tests.support.mock import patch + +pytestmark = [ + pytest.mark.skip_on_windows, +] + + +@pytest.fixture +def configure_loader_modules(): + return {salt.auth.pam: {}} + + +@pytest.fixture +def mock_pam(): + with patch("salt.auth.pam.CALLOC", autospec=True), patch( + "salt.auth.pam.pointer", autospec=True + ), patch("salt.auth.pam.PamHandle", autospec=True), patch( + "salt.auth.pam.PAM_START", autospec=True, return_value=0 + ), patch( + "salt.auth.pam.PAM_AUTHENTICATE", autospec=True, return_value=0 + ), patch( + "salt.auth.pam.PAM_END", autospec=True + ): + yield + + +def test_cve_if_pam_acct_mgmt_returns_nonzero_authenticate_should_be_false(mock_pam): + with patch("salt.auth.pam.PAM_ACCT_MGMT", autospec=True, return_value=42): + assert salt.auth.pam.authenticate(username="fnord", password="fnord") is False + + +def test_if_pam_acct_mgmt_returns_zero_authenticate_should_be_true(mock_pam): + with patch("salt.auth.pam.PAM_ACCT_MGMT", autospec=True, return_value=0): + assert salt.auth.pam.authenticate(username="fnord", password="fnord") is True diff --git a/tests/pytests/unit/transport/test_tcp.py b/tests/pytests/unit/transport/test_tcp.py index d536751e0ebc..9688b562ffbd 100644 --- a/tests/pytests/unit/transport/test_tcp.py +++ b/tests/pytests/unit/transport/test_tcp.py @@ -1,3 +1,4 @@ +import os import socket import attr @@ -7,7 +8,45 @@ import salt.ext.tornado import salt.transport.tcp from pytestshellutils.utils import ports -from tests.support.mock import MagicMock, patch +from tests.support.mock import MagicMock, PropertyMock, patch + + +@pytest.fixture +def fake_keys(): + with patch("salt.crypt.AsyncAuth.get_keys", autospec=True): + yield + + +@pytest.fixture +def fake_crypto(): + with patch("salt.transport.tcp.PKCS1_OAEP", create=True) as fake_crypto: + yield fake_crypto + + +@pytest.fixture +def fake_authd(): + @salt.ext.tornado.gen.coroutine + def return_nothing(): + raise salt.ext.tornado.gen.Return() + + with patch( + "salt.crypt.AsyncAuth.authenticated", new_callable=PropertyMock + ) as mock_authed, patch( + "salt.crypt.AsyncAuth.authenticate", + autospec=True, + return_value=return_nothing(), + ), patch( + "salt.crypt.AsyncAuth.gen_token", autospec=True, return_value=42 + ): + mock_authed.return_value = False + yield + + +@pytest.fixture +def fake_crypticle(): + with patch("salt.crypt.Crypticle") as fake_crypticle: + fake_crypticle.generate_key_string.return_value = "fakey fake" + yield fake_crypticle @attr.s(frozen=True, slots=True) @@ -335,3 +374,58 @@ def connect(*args, **kwargs): client.io_loop.run_sync(client.connect) finally: client.close() + + +async def test_when_async_req_channel_with_syndic_role_should_use_syndic_master_pub_file_to_verify_master_sig( + fake_keys, fake_crypto, fake_crypticle +): + # Syndics use the minion pki dir, but they also create a syndic_master.pub + # file for comms with the Salt master + expected_pubkey_path = os.path.join("/etc/salt/pki/minion", "syndic_master.pub") + fake_crypto.new.return_value.decrypt.return_value = "decrypted_return_value" + mockloop = MagicMock() + opts = { + "master_uri": "tcp://127.0.0.1:4506", + "interface": "127.0.0.1", + "ret_port": 4506, + "ipv6": False, + "sock_dir": ".", + "pki_dir": "/etc/salt/pki/minion", + "id": "syndic", + "__role": "syndic", + "keysize": 4096, + "transport": "tcp", + "acceptance_wait_time": 30, + "acceptance_wait_time_max": 30, + } + client = salt.channel.client.ReqChannel.factory(opts, io_loop=mockloop) + assert client.master_pubkey_path == expected_pubkey_path + with patch("salt.crypt.verify_signature") as mock: + client.verify_signature("mockdata", "mocksig") + assert mock.call_args_list[0][0][0] == expected_pubkey_path + + +async def test_mixin_should_use_correct_path_when_syndic( + fake_keys, fake_authd, fake_crypticle +): + mockloop = MagicMock() + expected_pubkey_path = os.path.join("/etc/salt/pki/minion", "syndic_master.pub") + opts = { + "master_uri": "tcp://127.0.0.1:4506", + "interface": "127.0.0.1", + "ret_port": 4506, + "ipv6": False, + "sock_dir": ".", + "pki_dir": "/etc/salt/pki/minion", + "id": "syndic", + "__role": "syndic", + "keysize": 4096, + "sign_pub_messages": True, + "transport": "tcp", + } + client = salt.channel.client.AsyncPubChannel.factory(opts, io_loop=mockloop) + client.master_pubkey_path = expected_pubkey_path + payload = {"sig": "abc", "load": {"foo": "bar"}} + with patch("salt.crypt.verify_signature") as mock: + client._verify_master_signature(payload) + assert mock.call_args_list[0][0][0] == expected_pubkey_path diff --git a/tests/pytests/unit/transport/test_zeromq.py b/tests/pytests/unit/transport/test_zeromq.py index e7abcf36c2d2..de2cc98e09ec 100644 --- a/tests/pytests/unit/transport/test_zeromq.py +++ b/tests/pytests/unit/transport/test_zeromq.py @@ -708,6 +708,7 @@ async def test_req_chan_decode_data_dict_entry_v2(pki_dir): auth = client.auth auth._crypticle = salt.crypt.Crypticle(opts, AES_KEY) client.auth = MagicMock() + client.auth.mpub = auth.mpub client.auth.authenticated = True client.auth.get_keys = auth.get_keys client.auth.crypticle.dumps = auth.crypticle.dumps @@ -772,6 +773,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_nonce(pki_dir): auth = client.auth auth._crypticle = salt.crypt.Crypticle(opts, AES_KEY) client.auth = MagicMock() + client.auth.mpub = auth.mpub client.auth.authenticated = True client.auth.get_keys = auth.get_keys client.auth.crypticle.dumps = auth.crypticle.dumps @@ -835,6 +837,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_signature(pki_dir): auth = client.auth auth._crypticle = salt.crypt.Crypticle(opts, AES_KEY) client.auth = MagicMock() + client.auth.mpub = auth.mpub client.auth.authenticated = True client.auth.get_keys = auth.get_keys client.auth.crypticle.dumps = auth.crypticle.dumps @@ -914,6 +917,7 @@ async def test_req_chan_decode_data_dict_entry_v2_bad_key(pki_dir): auth = client.auth auth._crypticle = salt.crypt.Crypticle(opts, AES_KEY) client.auth = MagicMock() + client.auth.mpub = auth.mpub client.auth.authenticated = True client.auth.get_keys = auth.get_keys client.auth.crypticle.dumps = auth.crypticle.dumps diff --git a/tests/support/case.py b/tests/support/case.py index a172c30e7c5e..c919272e3aa4 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -737,6 +737,7 @@ def run_function( "time.sleep", "grains.delkey", "grains.delval", + "sdb.get", ) if "f_arg" in kwargs: kwargs["arg"] = kwargs.pop("f_arg")