Replace boulder tests with pebble (#9918)

Pebble 2.5.1 supports OCSP stapling, so we can finally replace all boulder tests/harnesses with the much simpler pebble setup.

Closes #9898

* Remove unused `--acme-server` argument

Since this argument is never set and always defaults to 'pebble', just
remove it to simplify assumptions about which test server's being used.

* Remove boulder option from integration tests

Now that pebble supports all of our test cases, we can move off of
the much more complicated boulder test harness.

* pebble_artifacts: bump to latest pebble release

* pebble_artifacts: fix download path

* certbot-ci: unzip pebble assets

* CI: rip out windows tests/jobs

* tox.ini: rm outdated Windows comment

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* ci: rm redundant integration test

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

* acme_server: raise error if proxy and http-01 port are both set

* acme_server: rm vestigial preterimate commands stuff

---------

Co-authored-by: Brad Warren <bmw@users.noreply.github.com>

.azure-pipelines/templates/jobs/extended-tests-jobs.yml

@@ -19,34 +19,26 @@ jobs:
           TOXENV: py311
         linux-isolated:
           TOXENV: 'isolated-acme,isolated-certbot,isolated-apache,isolated-cloudflare,isolated-digitalocean,isolated-dnsimple,isolated-dnsmadeeasy,isolated-gehirn,isolated-google,isolated-linode,isolated-luadns,isolated-nsone,isolated-ovh,isolated-rfc2136,isolated-route53,isolated-sakuracloud,isolated-nginx'
-        linux-boulder-v2-integration-certbot-oldest:
+        linux-integration-certbot-oldest:
           PYTHON_VERSION: 3.8
           TOXENV: integration-certbot-oldest
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-integration-nginx-oldest:
+        linux-integration-nginx-oldest:
           PYTHON_VERSION: 3.8
           TOXENV: integration-nginx-oldest
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-py38-integration:
-          PYTHON_VERSION: 3.8
-          TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-py39-integration:
+        # python 3.8 integration tests are not run here because they're run as    
+        # part of the standard test suite 
+        linux-py39-integration:
           PYTHON_VERSION: 3.9
           TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-py310-integration:
+        linux-py310-integration:
           PYTHON_VERSION: 3.10
           TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-py311-integration:
+        linux-py311-integration:
           PYTHON_VERSION: 3.11
           TOXENV: integration
-          ACME_SERVER: boulder-v2
-        linux-boulder-v2-py312-integration:
+        linux-py312-integration:
           PYTHON_VERSION: 3.12
           TOXENV: integration
-          ACME_SERVER: boulder-v2
         nginx-compat:
           TOXENV: nginx_compat
         linux-integration-rfc2136:

.azure-pipelines/templates/jobs/packaging-jobs.yml

@@ -55,65 +55,6 @@ jobs:
       - bash: |
           set -e && tools/docker/test.sh $(dockerTag) $DOCKER_ARCH
         displayName: Run integration tests for Docker images
-  - job: installer_build
-    pool:
-      vmImage: windows-2019
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.9
-          architecture: x64
-          addToPath: true
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pip_install.py -e windows-installer
-        displayName: Prepare Windows installer build environment
-      - script: |
-          venv\Scripts\construct-windows-installer
-        displayName: Build Certbot installer
-      - task: CopyFiles@2
-        inputs:
-          sourceFolder: $(System.DefaultWorkingDirectory)/windows-installer/build/nsis
-          contents: '*.exe'
-          targetFolder: $(Build.ArtifactStagingDirectory)
-      - task: PublishPipelineArtifact@1
-        inputs:
-          path: $(Build.ArtifactStagingDirectory)
-          # If we change the artifact's name, it should also be changed in tools/create_github_release.py
-          artifact: windows-installer
-        displayName: Publish Windows installer
-  - job: installer_run
-    dependsOn: installer_build
-    strategy:
-      matrix:
-        win2019:
-          imageName: windows-2019
-    pool:
-      vmImage: $(imageName)
-    steps:
-      - task: UsePythonVersion@0
-        inputs:
-          versionSpec: 3.9
-          addToPath: true
-      - task: DownloadPipelineArtifact@2
-        inputs:
-          artifact: windows-installer
-          path: $(Build.SourcesDirectory)/bin
-        displayName: Retrieve Windows installer
-      - script: |
-          python -m venv venv
-          venv\Scripts\python tools\pip_install.py -e certbot-ci
-        env:
-          PIP_NO_BUILD_ISOLATION: no
-        displayName: Prepare Certbot-CI
-      - script: |
-          set PATH=%ProgramFiles%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\windows_installer_integration_tests --allow-persistent-changes --installer-path $(Build.SourcesDirectory)\bin\certbot-beta-installer-win_amd64.exe
-        displayName: Run windows installer integration tests
-      - script: |
-          set PATH=%ProgramFiles%\Certbot\bin;%PATH%
-          venv\Scripts\python -m pytest certbot-ci\certbot_integration_tests\certbot_tests -n 4
-        displayName: Run certbot integration tests
   - job: snaps_build
     pool:
       vmImage: ubuntu-22.04

.azure-pipelines/templates/jobs/standard-tests-jobs.yml

@@ -16,18 +16,6 @@ jobs:
           TOXENV: cover
           # See explanation under macos-py38-cover.
           PIP_USE_PEP517: "true"
-        windows-py38:
-          IMAGE_NAME: windows-2019
-          PYTHON_VERSION: 3.8
-          TOXENV: py-win
-        windows-py39-cover:
-          IMAGE_NAME: windows-2019
-          PYTHON_VERSION: 3.9
-          TOXENV: cover-win
-        windows-integration-certbot:
-          IMAGE_NAME: windows-2019
-          PYTHON_VERSION: 3.9
-          TOXENV: integration-certbot
         linux-oldest:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.8
@@ -49,7 +37,6 @@ jobs:
           IMAGE_NAME: ubuntu-22.04
           PYTHON_VERSION: 3.8
           TOXENV: integration
-          ACME_SERVER: pebble
         apache-compat:
           IMAGE_NAME: ubuntu-22.04
           TOXENV: apache_compat

certbot-apache/certbot_apache/_internal/tests/apache-conf-files/apache-conf-test-pebble.py

@@ -14,7 +14,7 @@
 
 def main() -> int:
     args = sys.argv[1:]
-    with acme_server.ACMEServer('pebble', [], False) as acme_xdist:
+    with acme_server.ACMEServer([], False) as acme_xdist:
         environ = os.environ.copy()
         environ['SERVER'] = acme_xdist['directory_url']
         command = [os.path.join(SCRIPT_DIRNAME, 'apache-conf-test')]

certbot-ci/certbot_integration_tests/certbot_tests/context.py

@@ -23,7 +23,6 @@ def __init__(self, request: pytest.FixtureRequest) -> None:
             self.worker_id = 'primary'
             acme_xdist = request.config.acme_xdist  # type: ignore[attr-defined]
 
-        self.acme_server = acme_xdist['acme_server']
         self.directory_url = acme_xdist['directory_url']
         self.tls_alpn_01_port = acme_xdist['https_port'][self.worker_id]
         self.http_01_port = acme_xdist['http_port'][self.worker_id]

certbot-ci/certbot_integration_tests/certbot_tests/test_main.py

@@ -7,7 +7,6 @@
 import subprocess
 import time
 from typing import Generator
-from typing import Iterable
 from typing import Tuple
 from typing import Type
 
@@ -82,11 +81,9 @@ def test_registration_override(context: IntegrationTestsContext) -> None:
     context.certbot(['update_account', '--email', 'ex1@domain.org,ex2@domain.org'])
     stdout2, _ = context.certbot(['show_account'])
 
-    # https://github.com/letsencrypt/boulder/issues/6144
-    if context.acme_server != 'boulder-v2':
-        assert 'example@domain.org' in stdout1, "New email should be present"
-        assert 'example@domain.org' not in stdout2, "Old email should not be present"
-        assert 'ex1@domain.org, ex2@domain.org' in stdout2, "New emails should be present"
+    assert 'example@domain.org' in stdout1, "New email should be present"
+    assert 'example@domain.org' not in stdout2, "Old email should not be present"
+    assert 'ex1@domain.org, ex2@domain.org' in stdout2, "New emails should be present"
 
 
 def test_prepare_plugins(context: IntegrationTestsContext) -> None:
@@ -566,19 +563,15 @@ def test_default_rsa_size(context: IntegrationTestsContext) -> None:
     assert_rsa_key(key1, 2048)
 
 
-@pytest.mark.parametrize('curve,curve_cls,skip_servers', [
+@pytest.mark.parametrize('curve,curve_cls', [
     # Curve name, Curve class, ACME servers to skip
-    ('secp256r1', SECP256R1, []),
-    ('secp384r1', SECP384R1, []),
-    ('secp521r1', SECP521R1, ['boulder-v2'])]
+    ('secp256r1', SECP256R1),
+    ('secp384r1', SECP384R1),
+    ('secp521r1', SECP521R1)]
 )
-def test_ecdsa_curves(context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve],
-                      skip_servers: Iterable[str]) -> None:
+def test_ecdsa_curves(context: IntegrationTestsContext, curve: str,
+                      curve_cls: Type[EllipticCurve]) -> None:
     """Test issuance for each supported ECDSA curve"""
-    if context.acme_server in skip_servers:
-        pytest.skip('ACME server {} does not support ECDSA curve {}'
-                    .format(context.acme_server, curve))
-
     domain = context.get_domain('curve')
     context.certbot([
         'certonly',
@@ -640,9 +633,6 @@ def test_renew_with_ec_keys(context: IntegrationTestsContext) -> None:
 
 def test_ocsp_must_staple(context: IntegrationTestsContext) -> None:
     """Test that OCSP Must-Staple is correctly set in the generated certificate."""
-    if context.acme_server == 'pebble':
-        pytest.skip('Pebble does not support OCSP Must-Staple.')
-
     certname = context.get_domain('must-staple')
     context.certbot(['auth', '--must-staple', '--domains', certname])
 
@@ -710,17 +700,14 @@ def test_revoke_and_unregister(context: IntegrationTestsContext) -> None:
     assert cert3 in stdout
 
 
-@pytest.mark.parametrize('curve,curve_cls,skip_servers', [
-    ('secp256r1', SECP256R1, []),
-    ('secp384r1', SECP384R1, []),
-    ('secp521r1', SECP521R1, ['boulder-v2'])]
+@pytest.mark.parametrize('curve,curve_cls', [
+    ('secp256r1', SECP256R1),
+    ('secp384r1', SECP384R1),
+    ('secp521r1', SECP521R1)]
 )
 def test_revoke_ecdsa_cert_key(
-    context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve],
-    skip_servers: Iterable[str]) -> None:
+    context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve]) -> None:
     """Test revoking a certificate """
-    if context.acme_server in skip_servers:
-        pytest.skip(f'ACME server {context.acme_server} does not support ECDSA curve {curve}')
     cert: str = context.get_domain('curve')
     context.certbot([
         'certonly',
@@ -738,17 +725,14 @@ def test_revoke_ecdsa_cert_key(
     assert stdout.count('INVALID: REVOKED') == 1, 'Expected {0} to be REVOKED'.format(cert)
 
 
-@pytest.mark.parametrize('curve,curve_cls,skip_servers', [
-    ('secp256r1', SECP256R1, []),
-    ('secp384r1', SECP384R1, []),
-    ('secp521r1', SECP521R1, ['boulder-v2'])]
+@pytest.mark.parametrize('curve,curve_cls', [
+    ('secp256r1', SECP256R1),
+    ('secp384r1', SECP384R1),
+    ('secp521r1', SECP521R1)]
 )
 def test_revoke_ecdsa_cert_key_delete(
-    context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve],
-    skip_servers: Iterable[str]) -> None:
+    context: IntegrationTestsContext, curve: str, curve_cls: Type[EllipticCurve]) -> None:
     """Test revoke and deletion for each supported curve type"""
-    if context.acme_server in skip_servers:
-        pytest.skip(f'ACME server {context.acme_server} does not support ECDSA curve {curve}')
     cert: str = context.get_domain('curve')
     context.certbot([
         'certonly',
@@ -913,7 +897,7 @@ def test_dry_run_deactivate_authzs(context: IntegrationTestsContext) -> None:
 def test_preferred_chain(context: IntegrationTestsContext) -> None:
     """Test that --preferred-chain results in the correct chain.pem being produced"""
     try:
-        issuers = misc.get_acme_issuers(context)
+        issuers = misc.get_acme_issuers()
     except NotImplementedError:
         pytest.skip('This ACME server does not support alternative issuers.')
 

certbot-ci/certbot_integration_tests/conftest.py

@@ -8,7 +8,6 @@
 See https://docs.pytest.org/en/latest/reference.html#hook-reference
 """
 import contextlib
-import subprocess
 import sys
 
 from certbot_integration_tests.utils import acme_server as acme_lib
@@ -20,10 +19,6 @@ def pytest_addoption(parser):
     Standard pytest hook to add options to the pytest parser.
     :param parser: current pytest parser that will be used on the CLI
     """
-    parser.addoption('--acme-server', default='pebble',
-                     choices=['boulder-v2', 'pebble'],
-                     help='select the ACME server to use (boulder-v2, pebble), '
-                          'defaulting to pebble')
     parser.addoption('--dns-server', default='challtestsrv',
                      choices=['bind', 'challtestsrv'],
                      help='select the DNS server to use (bind, challtestsrv), '
@@ -80,23 +75,6 @@ def _setup_primary_node(config):
 
     :param config: Configuration of the pytest primary node. Is modified by this function.
     """
-    # Check for runtime compatibility: some tools are required to be available in PATH
-    if 'boulder' in config.option.acme_server:
-        try:
-            subprocess.check_output(['docker', '-v'], stderr=subprocess.STDOUT)
-        except (subprocess.CalledProcessError, OSError):
-            raise ValueError('Error: docker is required in PATH to launch the integration tests on'
-                             'boulder, but is not installed or not available for current user.')
-
-        try:
-            subprocess.check_output(['docker', 'compose', 'ls'], stderr=subprocess.STDOUT)
-        except (subprocess.CalledProcessError, OSError):
-            raise ValueError(
-                'Error: A version of Docker with the "compose" subcommand '
-                'is required in PATH to launch the integration tests, '
-                'but is not installed or not available for current user.'
-            )
-
     # Parameter numprocesses is added to option by pytest-xdist
     workers = ['primary'] if not config.option.numprocesses\
         else ['gw{0}'.format(i) for i in range(config.option.numprocesses)]
@@ -116,8 +94,7 @@ def _setup_primary_node(config):
 
     # By calling setup_acme_server we ensure that all necessary acme server instances will be
     # fully started. This runtime is reflected by the acme_xdist returned.
-    acme_server = acme_lib.ACMEServer(config.option.acme_server, workers,
-                                      dns_server=acme_dns_server)
+    acme_server = acme_lib.ACMEServer(workers, dns_server=acme_dns_server)
     config.add_cleanup(acme_server.stop)
     print('ACME xdist config:\n{0}'.format(acme_server.acme_xdist))
     acme_server.start()