Merge branch 'apache:main' into feat/dbt-retry-from-failure

.asf.yaml

@@ -110,6 +110,8 @@ github:
         required_approving_review_count: 1
       required_linear_history: true
   collaborators:
+    # Max 10 collaborators allowed
+    # https://github.com/apache/infrastructure-asfyaml/blob/main/README.md#assigning-the-github-triage-role-to-external-collaborators
     - RNHTTR
     - aritra24
     - dirrao
@@ -118,6 +120,7 @@ github:
     - sunank200
     - vatsrahul1001
     - cmarteepants
+    - romsharon98
 
 notifications:
   jobs: jobs@airflow.apache.org

.github/workflows/additional-prod-image-tests.yml

@@ -93,7 +93,7 @@ jobs:
 
   test-examples-of-prod-image-building:
     timeout-minutes: 60
-    name: "Test examples of POD image building"
+    name: "Test examples of PROD image building"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}

.github/workflows/build-images.yml

@@ -91,8 +91,25 @@ jobs:
       - name: Discover PR merge commit
         id: discover-pr-merge-commit
         run: |
+          # Sometimes target-commit-sha cannot be
           TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
-          echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV}
+          if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
+            # Sometimes retrieving the merge commit SHA from PR fails. We retry it once. Otherwise we
+            # fall-back to github.event.pull_request.head.sha
+            echo
+            echo "Could not retrieve merge commit SHA from PR, waiting for 3 seconds and retrying."
+            echo
+            sleep 3
+            TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}' --jq .merge_commit_sha)"
+            if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
+              echo
+              echo "Could not retrieve merge commit SHA from PR, falling back to PR head SHA."
+              echo
+              TARGET_COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
+            fi
+          fi
+          echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}"
+          echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}" >> ${GITHUB_ENV}
           echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
         if: github.event_name == 'pull_request_target'
       # The labels in the event aren't updated when re-triggering the job, So lets hit the API to get

.github/workflows/finalize-tests.yml

@@ -191,9 +191,20 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           path: ./artifacts
+      - name: "Setup python"
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.default-python-version }}
       - name: "Summarize all warnings"
         run: |
-          ls -R ./artifacts/
-          cat ./artifacts/test-warnings*/* | sort | uniq
-          echo
-          echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l)
+          ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \
+            --pattern "**/warnings-*.txt" \
+            --output ./files
+      - name: "Upload artifact for summarized warnings"
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summarized-warnings
+          path: ./files/warn-summary-*.txt
+          retention-days: 7
+          if-no-files-found: ignore
+          overwrite: true

.github/workflows/k8s-tests.yml

@@ -97,7 +97,7 @@ jobs:
             k8s-env-${{ steps.breeze.outputs.host-python-version }}-\
             ${{ hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','pyproject.toml') }}"
       - name: Run complete K8S tests ${{ inputs.kubernetes-combos-list-as-string }}
-        run: breeze k8s run-complete-tests --run-in-parallel --upgrade
+        run: breeze k8s run-complete-tests --run-in-parallel --upgrade --no-copy-local-sources
         env:
           PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }}
           KUBERNETES_VERSIONS: ${{ inputs.kubernetes-versions-list-as-string }}

.github/workflows/run-unit-tests.yml

@@ -175,6 +175,8 @@ jobs:
             breeze testing tests --test-type "All-Quarantined" || true
           elif [[ "${{ inputs.test-scope }}" == "ARM collection" ]]; then
             breeze testing tests --collect-only --remove-arm-packages
+          elif [[ "${{ inputs.test-scope }}" == "System" ]]; then
+            breeze testing tests tests/system/example_empty.py --system core
           else
             echo "Unknown test scope: ${{ inputs.test-scope }}"
             exit 1

.github/workflows/special-tests.yml

@@ -230,3 +230,24 @@ jobs:
       include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
       run-coverage: ${{ inputs.run-coverage }}
       debug-resources: ${{ inputs.debug-resources }}
+
+  tests-system:
+    name: "System test"
+    uses: ./.github/workflows/run-unit-tests.yml
+    permissions:
+      contents: read
+      packages: read
+    secrets: inherit
+    with:
+      runs-on-as-json-default: ${{ inputs.runs-on-as-json-default }}
+      test-name: "SystemTest"
+      test-scope: "System"
+      backend: "postgres"
+      image-tag: ${{ inputs.image-tag }}
+      python-versions: "['${{ inputs.default-python-version }}']"
+      backend-versions: "['${{ inputs.default-postgres-version }}']"
+      excludes: "[]"
+      parallel-test-types-list-as-string: ${{ inputs.parallel-test-types-list-as-string }}
+      include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }}
+      run-coverage: ${{ inputs.run-coverage }}
+      debug-resources: ${{ inputs.debug-resources }}

.gitignore

@@ -16,6 +16,7 @@ airflow/www/*.log
 airflow-webserver.pid
 standalone_admin_password.txt
 warnings.txt
+warn-summary-*.txt
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.pre-commit-config.yaml

@@ -335,7 +335,7 @@ repos:
         types_or: [python, pyi]
         args: [--fix]
         require_serial: true
-        additional_dependencies: ["ruff==0.3.5"]
+        additional_dependencies: ["ruff==0.3.6"]
         exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py
       - id: ruff-format
         name: Run 'ruff format' for extremely fast Python formatting
@@ -345,7 +345,7 @@ repos:
         types_or: [python, pyi]
         args: []
         require_serial: true
-        additional_dependencies: ["ruff==0.3.5"]
+        additional_dependencies: ["ruff==0.3.6"]
         exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^airflow/contrib/
       - id: replace-bad-characters
         name: Replace bad characters
@@ -437,21 +437,21 @@ repos:
         files: ^docs/apache-airflow/extra-packages-ref\.rst$|^hatch_build.py
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_extra_packages_ref.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.22.5', 'tabulate']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2', 'tabulate']
       - id: check-hatch-build-order
         name: Check order of dependencies in hatch_build.py
         language: python
         files: ^hatch_build.py$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/check_order_hatch_build.py
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.22.5']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2']
       - id: update-extras
         name: Update extras in documentation
         entry: ./scripts/ci/pre_commit/insert_extras.py
         language: python
         files: ^contributing-docs/12_airflow_dependencies_and_extras.rst$|^INSTALL$|^airflow/providers/.*/provider\.yaml$|^Dockerfile.*
         pass_filenames: false
-        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.22.5']
+        additional_dependencies: ['rich>=12.4.4', 'hatchling==1.24.2']
       - id: check-extras-order
         name: Check order of extras in Dockerfile
         entry: ./scripts/ci/pre_commit/check_order_dockerfile_extras.py
@@ -596,6 +596,7 @@ repos:
           ^airflow/www/static/css/bootstrap-theme.css$|
           ^airflow/www/static/js/types/api-generated.ts$|
           ^airflow/www/templates/appbuilder/flash.html$|
+          ^chart/values.schema.json$|
           ^.*/.*_vendor/|
           ^dev/|
           ^docs/README.rst$|
@@ -1000,6 +1001,13 @@ repos:
         pass_filenames: true
         files: ^airflow/.*\.py$
         exclude: ^.*/.*_vendor/
+      - id: check-code-deprecations
+        name: Check deprecations categories in decorators
+        entry: ./scripts/ci/pre_commit/check_deprecations.py
+        language: python
+        pass_filenames: true
+        files: ^airflow/.*\.py$
+        exclude: ^.*/.*_vendor/
       - id: lint-chart-schema
         name: Lint chart/values.schema.json file
         entry: ./scripts/ci/pre_commit/chart_schema.py

.rat-excludes

@@ -127,6 +127,7 @@ newsfragments/*
 
 # Warning file generated
 warnings.txt
+warn-summary-*.txt
 
 # Dev stuff
 tests/*

Dockerfile

@@ -50,7 +50,7 @@ ARG AIRFLOW_VERSION="2.9.0"
 ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm"
 
 ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.1.29
+ARG AIRFLOW_UV_VERSION=0.1.35
 ARG AIRFLOW_USE_UV="false"
 ARG UV_HTTP_TIMEOUT="300"
 ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
@@ -780,6 +780,10 @@ function install_airflow_and_providers_from_docker_context_files(){
                 ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \
                 "${install_airflow_package[@]}" "${installing_providers_packages[@]}"
             set +x
+            echo
+            echo "${COLOR_BLUE}Copying ${local_constraints_file} to ${HOME}/constraints.txt${COLOR_RESET}"
+            echo
+            cp "${local_constraints_file}" "${HOME}/constraints.txt"
         else
             echo
             echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}"

Dockerfile.ci

@@ -1222,7 +1222,7 @@ ARG DEFAULT_CONSTRAINTS_BRANCH="constraints-main"
 ARG AIRFLOW_CI_BUILD_EPOCH="10"
 ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.1.29
+ARG AIRFLOW_UV_VERSION=0.1.35
 ARG AIRFLOW_USE_UV="true"
 # Setup PIP
 # By default PIP install run without cache to make image smaller
@@ -1246,7 +1246,7 @@ ARG AIRFLOW_VERSION=""
 ARG ADDITIONAL_PIP_INSTALL_FLAGS=""
 
 ARG AIRFLOW_PIP_VERSION=24.0
-ARG AIRFLOW_UV_VERSION=0.1.29
+ARG AIRFLOW_UV_VERSION=0.1.35
 ARG AIRFLOW_USE_UV="true"
 
 ENV AIRFLOW_REPO=${AIRFLOW_REPO}\

INSTALL

@@ -137,7 +137,7 @@ or install all Python versions that are used in Airflow:
 Airflow has some pre-defined virtualenvs that you can use to develop and test airflow.
 You can see the list of available envs with:
 
-    hatch show env
+    hatch env show
 
 This is what it shows currently:
 
@@ -154,7 +154,7 @@ This is what it shows currently:
 ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤
 │ airflow-311 │ virtual │ Environment with Python 3.11. No devel installed              │
 ├─────────────┼─────────┼───────────────────────────────────────────────────────────────┤
-│ airflow-312 │ virtual │ Environment with Python 3.11. No devel installed              │
+│ airflow-312 │ virtual │ Environment with Python 3.12. No devel installed              │
 └─────────────┴─────────┴───────────────────────────────────────────────────────────────┘
 
 The default env (if you have not used one explicitly) is `default` and it is a Python 3.8
@@ -262,9 +262,9 @@ Those extras are available as regular core airflow extras - they install optiona
 
 # START CORE EXTRAS HERE
 
-aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, deprecated-api, github-enterprise,
-google-auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic, rabbitmq, s3fs,
-saml, sentry, statsd, uv, virtualenv
+aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, deprecated-api, github-
+enterprise, google-auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic,
+rabbitmq, s3fs, saml, sentry, statsd, uv, virtualenv
 
 # END CORE EXTRAS HERE
 

INTHEWILD.md

@@ -218,6 +218,7 @@ Currently, **officially** using Airflow:
 1. [Gentner Lab](https://github.com/gentnerlab) [[@neuromusic](https://github.com/neuromusic)]
 1. [Get Simpl](https://getsimpl.com/) [[@rootcss](https://github.com/rootcss)]
 1. [Getir](https://www.getir.com/) [[@mpolatcan](https://github.com/mpolatcan)]
+1. [Giant Steps Capital](https://www.gscap.com.br/) [[@flaviokr](https://github.com/flaviokr) & [@giant-steps](https://github.com/giant-steps) & [@mateuslatrova](https://github.com/mateuslatrova) & [@NataliaGodot](https://github.com/NataliaGodot)]
 1. [GitLab](https://about.gitlab.com/) [[@tayloramurphy](https://gitlab.com/tayloramurphy) & [@m_walker](https://gitlab.com/m_walker)]
 1. [Glassdoor](https://github.com/Glassdoor) [[@syvineckruyk](https://github.com/syvineckruyk) & [@sid88in](https://github.com/sid88in)]
 1. [Global Fashion Group](http://global-fashion-group.com) [[@GFG](https://github.com/GFG)]

PROVIDERS.rst

@@ -144,8 +144,8 @@ Airflow version to the next MINOR release, when 12 months passed since the first
 MINOR version of Airflow.
 
 For example this means that by default we upgrade the minimum version of Airflow supported by providers
-to 2.7.0 in the first Provider's release after 27th of April 2024. 27th of April 2023 is the date when the
-first ``PATCHLEVEL`` of 2.6 (2.6.0) has been released.
+to 2.8.0 in the first Provider's release after 18th of August 2024. 18th of August 2023 is the date when the
+first ``PATCHLEVEL`` of 2.7 (2.7.0) has been released.
 
 When we increase the minimum Airflow version, this is not a reason to bump ``MAJOR`` version of the providers
 (unless there are other breaking changes in the provider). The reason for that is that people who use

README.md

@@ -100,9 +100,9 @@ Apache Airflow is tested with:
 
 |             | Main version (dev)         | Stable version (2.9.0)      |
 |-------------|----------------------------|-----------------------------|
-| Python      | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11        |
+| Python      | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11, 3.12  |
 | Platform    | AMD64/ARM64(\*)            | AMD64/ARM64(\*)             |
-| Kubernetes  | 1.26, 1.27, 1.28, 1.29     | 1.25, 1.26, 1.27, 1.28, 1.29|
+| Kubernetes  | 1.26, 1.27, 1.28, 1.29     | 1.26, 1.27, 1.28, 1.29      |
 | PostgreSQL  | 12, 13, 14, 15, 16         | 12, 13, 14, 15, 16          |
 | MySQL       | 8.0, Innovation            | 8.0, Innovation             |
 | SQLite      | 3.15.0+                    | 3.15.0+                     |

RELEASE_NOTES.rst

@@ -111,6 +111,14 @@ Xcom table column ``value`` type has changed from ``blob`` to ``longblob``. This
 
 To downgrade from revision: ``b4078ac230a1``, ensure that you don't have Xcom values larger than 65,535 bytes. Otherwise, you'll need to clean those rows or run ``airflow db clean xcom`` to clean the Xcom table.
 
+Stronger validation for key parameter defaults in taskflow context variables (#38015)
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+As for the taskflow implementation in conjunction with context variable defaults invalid parameter orders can be
+generated, it is now not accepted anymore (and validated) that taskflow functions are defined with defaults
+other than ``None``. If you have done this before you most likely will see a broken DAG and a error message like
+``Error message: Context key parameter my_param can't have a default other than None``.
+
 New Features
 """"""""""""
 - Allow users to write dag_id and task_id in their national characters, added display name for dag / task (v2) (#38446)
@@ -127,7 +135,6 @@ New Features
 - Introduce mechanism to support multiple executor configuration (#37635)
 - Add color formatting for ANSI chars in logs from task executions (#37985)
 - Add the dataset_expression as part of DagModel and DAGDetailSchema (#37826)
-- Add TaskFail entries to Gantt chart (#37918)
 - Allow longer rendered_map_index (#37798)
 - Inherit the run_ordering from DatasetTriggeredTimetable for DatasetOrTimeSchedule (#37775)
 - Implement AIP-60 Dataset URI formats (#37005)
@@ -150,7 +157,6 @@ New Features
 - Create new Metrics with Tagging (#36528)
 - Add support for openlineage to AFS and common.io (#36410)
 - Introduce ``@task.bash`` TaskFlow decorator (#30176, #37875)
-- Added functionality to automatically ingest custom airflow.cfg file upon startup (#36289)
 
 Improvements
 """"""""""""

airflow/__init__.py

@@ -21,6 +21,7 @@
 
 import os
 import sys
+import warnings
 
 if os.environ.get("_AIRFLOW_PATCH_GEVENT"):
     # If you are using gevents and start airflow webserver, you might want to run gevent monkeypatching
@@ -30,6 +31,17 @@
 
     patch_all()
 
+if sys.platform == "win32":
+    warnings.warn(
+        "Airflow currently can be run on POSIX-compliant Operating Systems. For development, "
+        "it is regularly tested on fairly modern Linux Distros and recent versions of macOS. "
+        "On Windows you can run it via WSL2 (Windows Subsystem for Linux 2) or via Linux Containers. "
+        "The work to add Windows support is tracked via https://github.com/apache/airflow/issues/10388, "
+        "but it is not a high priority.",
+        category=RuntimeWarning,
+        stacklevel=1,
+    )
+
 # The configuration module initializes and validates the conf object as a side effect the first
 # time it is imported. If it is not imported before importing the settings module, the conf
 # object will then be initted/validated as a side effect of it being imported in settings,
@@ -76,19 +88,16 @@ def __getattr__(name: str):
     module_path, attr_name, deprecated = __lazy_imports.get(name, ("", "", False))
     if not module_path:
         if name.startswith("PY3") and (py_minor := name[3:]) in ("6", "7", "8", "9", "10", "11", "12"):
-            import warnings
-
             warnings.warn(
                 f"Python version constraint {name!r} is deprecated and will be removed in the future. "
                 f"Please get version info from the 'sys.version_info'.",
                 DeprecationWarning,
+                stacklevel=2,
             )
             return sys.version_info >= (3, int(py_minor))
 
         raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
     elif deprecated:
-        import warnings
-
         warnings.warn(
             f"Import {name!r} directly from the airflow module is deprecated and "
             f"will be removed in the future. Please import it from 'airflow{module_path}.{attr_name}'.",

airflow/api/common/delete_dag.py

@@ -27,6 +27,7 @@
 from airflow import models
 from airflow.exceptions import AirflowException, DagNotFound
 from airflow.models import DagModel, TaskFail
+from airflow.models.errors import ParseImportError
 from airflow.models.serialized_dag import SerializedDagModel
 from airflow.utils.db import get_sqla_model_classes
 from airflow.utils.session import NEW_SESSION, provide_session
@@ -99,8 +100,8 @@ def delete_dag(dag_id: str, keep_records_in_log: bool = True, session: Session =
     # Delete entries in Import Errors table for a deleted DAG
     # This handles the case when the dag_id is changed in the file
     session.execute(
-        delete(models.ImportError)
-        .where(models.ImportError.filename == dag.fileloc)
+        delete(ParseImportError)
+        .where(ParseImportError.filename == dag.fileloc)
         .execution_options(synchronize_session="fetch")
     )