feat!: Upgrade dependencies to Airflow 2.1.1 (#135)

* revised dependendices for Airflow 2.1.1

* use proper variables file as required by Airflow 2

* changed README to use Airflow 2 requirement

* uncommented deploy_dag_versioning check

* feat: Upgrade `usa_names` pipeline to usse Airflow 2 operators and environment (#136)

Pipfile

@@ -7,7 +7,9 @@ name = "pypi"
 beautifulsoup4 = "==4.9.3"
 
 [dev-packages]
-apache-airflow = {version = "==1.10.15", extras = ["google"]}
+apache-airflow = "==2.1.1"
+apache-airflow-providers-google = "*"
+apache-airflow-providers-cncf-kubernetes = "*"
 black = "==20.8b1"
 flake8 = "==3.9.2"
 isort = "*"
@@ -16,8 +18,8 @@ pandas-gbq = "==0.14.1"
 pytest-mock = "*"
 pytest = "*"
 "ruamel.yaml" = "==0.17.10"
-Jinja2 = "*"
-SQLAlchemy = "==1.3.15"
+Jinja2 = "==2.11.3"
+SQLAlchemy = "==1.3.18"
 
 [requires]
 python_version = "3.8"

README.md

@@ -8,11 +8,11 @@ Cloud-native, data pipeline architecture for onboarding public datasets to [Data
 
 # Requirements
 - Python `>=3.6.10,<3.9`. We currently use `3.8`. For more info, see the [Cloud Composer version list](https://cloud.google.com/composer/docs/concepts/versioning/composer-versions).
-- Familiarity with [Apache Airflow](https://airflow.apache.org/docs/apache-airflow/1.10.15/concepts.html) (>=v1.10.15)
+- Familiarity with [Apache Airflow](https://airflow.apache.org/docs/apache-airflow/stable/concepts/index.html) (>=v2.1)
 - [pipenv](https://pipenv-fork.readthedocs.io/en/latest/install.html#installing-pipenv) for creating similar Python environments via `Pipfile.lock`
 - [gcloud](https://cloud.google.com/sdk/gcloud) command-line tool with Google Cloud Platform credentials configured. Instructions can be found [here](https://cloud.google.com/sdk/docs/initializing).
 - [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) `>=v0.15.1`
-- [Google Cloud Composer](https://cloud.google.com/composer/docs/concepts/overview) environment running [Apache Airflow](https://airflow.apache.org/docs/apache-airflow/1.10.15/concepts.html) `>=v1.10.15,<2.0`. To create a new Cloud Composer environment, see [this guide](https://cloud.google.com/composer/docs/how-to/managing/creating).
+- [Google Cloud Composer](https://cloud.google.com/composer/docs/concepts/overview) environment running [Apache Airflow](https://airflow.apache.org/docs/apache-airflow/stable/concepts.html) `>=2.0`. To create a new Cloud Composer environment, see [this guide](https://cloud.google.com/composer/docs/how-to/managing/creating).
 
 # Environment Setup
 

datasets/usa_names/usa_1910_current/pipeline.yaml

@@ -22,6 +22,7 @@ resources:
       source: http://www.ssa.gov/OACT/babynames/limits.html
 
 dag:
+  airflow_version: 2
   initialize:
     dag_id: "usa_1910_current"
     default_args:

datasets/usa_names/usa_1910_current/usa_1910_current_dag.py

@@ -14,8 +14,8 @@
 
 
 from airflow import DAG
-from airflow.contrib.operators import gcs_to_bq
-from airflow.operators import bash_operator
+from airflow.operators import bash
+from airflow.providers.google.cloud.transfers import gcs_to_bigquery
 
 default_args = {
     "owner": "Google",
@@ -34,7 +34,7 @@
 ) as dag:
 
     # Task to copy `namesbystate.zip` from Social Security Administration to GCS
-    download_and_process_source_zip_file = bash_operator.BashOperator(
+    download_and_process_source_zip_file = bash.BashOperator(
         task_id="download_and_process_source_zip_file",
         bash_command="mkdir -p $data_dir/{{ ds }}\ncurl -o $data_dir/{{ ds }}/namesbystate.zip -L $zip_source_url\nunzip $data_dir/{{ ds }}/namesbystate.zip -d $data_dir/{{ ds }}\ncat $data_dir/{{ ds }}/*.TXT \u003e\u003e $data_dir/{{ ds }}/data.csv\n",
         env={
@@ -44,7 +44,7 @@
     )
 
     # Task to load the data from Airflow data folder to BigQuery
-    load_csv_file_to_bq_table = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
+    load_csv_file_to_bq_table = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="load_csv_file_to_bq_table",
         bucket="{{ var.json.shared.composer_bucket }}",
         source_objects=["data/usa_names/usa_1910_current/{{ ds }}/data.csv"],

requirements-dev.txt

@@ -6,156 +6,192 @@
 #
 
 -i https://pypi.org/simple
-alembic==1.6.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-apache-airflow[google]==1.10.14
-apispec[yaml]==1.3.3
+alembic==1.6.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+anyio==3.3.0; python_full_version >= '3.6.2'
+apache-airflow-providers-cncf-kubernetes==2.0.1
+apache-airflow-providers-ftp==2.0.0; python_version ~= '3.6'
+apache-airflow-providers-google==5.0.0
+apache-airflow-providers-imap==2.0.0; python_version ~= '3.6'
+apache-airflow-providers-sqlite==2.0.0; python_version ~= '3.6'
+apache-airflow==2.1.1
+apispec[yaml]==3.3.2; python_full_version >= '3.5.0'
 appdirs==1.4.4
 argcomplete==1.12.3
 attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 babel==2.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+backports.entry-points-selectable==1.1.0; python_version >= '2.7'
 black==20.8b1
-cached-property==1.5.2
+blinker==1.4
 cachetools==4.2.2; python_version ~= '3.5'
-cattrs==1.6.0; python_version >= '3.7'
-certifi==2020.12.5
-cffi==1.14.5
-chardet==3.0.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+cattrs==1.5.0; python_version > '3.6'
+certifi==2021.5.30
+cffi==1.14.6
+charset-normalizer==2.0.4; python_version >= '3'
 click==7.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+clickclick==20.10.2
 colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-colorlog==4.0.2
-configparser==3.5.3; python_version >= '2.6'
-croniter==0.3.37; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-cryptography==3.4.7; python_version >= '3.0'
+colorlog==4.8.0
+commonmark==0.9.1
+croniter==1.0.15; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+cryptography==3.4.7; python_version >= '3.6'
 defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-dill==0.3.3; python_version >= '2.6' and python_version != '3.0'
+dill==0.3.4; python_version >= '2.7' and python_version != '3.0'
+distlib==0.3.2
 dnspython==2.1.0; python_version >= '3.6'
-docutils==0.17.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-email-validator==1.1.2
-flake8==3.8.4
-flask-admin==1.5.4
-flask-appbuilder==2.3.4; python_version >= '3.6'
+docutils==0.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+email-validator==1.1.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+filelock==3.0.12
+flake8==3.9.2
+flask-appbuilder==3.3.2; python_version ~= '3.6'
 flask-babel==1.0.0
-flask-caching==1.3.3
-flask-jwt-extended==3.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
+flask-caching==1.10.1; python_full_version >= '3.5.0'
+flask-jwt-extended==3.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'
 flask-login==0.4.1
 flask-openid==1.2.5
 flask-sqlalchemy==2.5.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-flask-swagger==0.2.14
 flask-wtf==0.14.3
 flask==1.1.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
-funcsigs==1.0.2
-future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-google-api-core[grpc,grpcgcp]==1.26.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-google-api-python-client==1.12.8
+google-ads==13.0.0; python_version >= '3.7'
+google-api-core[grpc,grpcgcp]==1.31.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+google-api-python-client==1.12.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 google-auth-httplib2==0.1.0
-google-auth-oauthlib==0.4.4; python_version >= '3.6'
-google-auth==1.30.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-google-cloud-bigquery-storage==2.4.0
-google-cloud-bigquery[bqstorage,pandas]==2.16.1; python_version < '3.10' and python_version >= '3.6'
-google-cloud-bigtable==1.7.0
-google-cloud-container==1.0.1
-google-cloud-core==1.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-google-cloud-dlp==1.0.0
-google-cloud-language==1.3.0
-google-cloud-secret-manager==1.0.0
-google-cloud-spanner==1.19.1
-google-cloud-speech==1.3.2
-google-cloud-storage==1.38.0
-google-cloud-texttospeech==1.0.1
-google-cloud-translate==1.7.0
-google-cloud-videointelligence==1.16.1
-google-cloud-vision==1.0.0
+google-auth-oauthlib==0.4.5; python_version >= '3.6'
+google-auth==1.34.0; python_version >= '3.6'
+google-cloud-appengine-logging==0.1.4; python_version >= '3.6'
+google-cloud-audit-log==0.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-automl==2.4.2; python_version >= '3.6'
+google-cloud-bigquery-datatransfer==3.3.1; python_version >= '3.6'
+google-cloud-bigquery-storage==2.6.2
+google-cloud-bigquery[bqstorage,pandas]==2.23.2; python_version < '3.10' and python_version >= '3.6'
+google-cloud-bigtable==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-container==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-core==1.7.2; python_version >= '3.6'
+google-cloud-datacatalog==3.4.0
+google-cloud-dataproc==2.5.0; python_version >= '3.6'
+google-cloud-dlp==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-kms==2.4.3; python_version >= '3.6'
+google-cloud-language==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-logging==2.6.0; python_version >= '3.6'
+google-cloud-memcache==1.0.0; python_version >= '3.6'
+google-cloud-monitoring==2.4.2; python_version >= '3.6'
+google-cloud-os-login==2.3.1; python_version >= '3.6'
+google-cloud-pubsub==2.7.0; python_version >= '3.6'
+google-cloud-redis==2.2.2; python_version >= '3.6'
+google-cloud-secret-manager==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-spanner==1.19.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-speech==1.3.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-storage==1.41.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+google-cloud-tasks==2.5.1; python_version >= '3.6'
+google-cloud-texttospeech==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-translate==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-videointelligence==1.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-vision==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+google-cloud-workflows==1.2.1; python_version >= '3.6'
 google-crc32c==1.1.2; python_version >= '3.5'
-google-resumable-media==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+google-resumable-media==1.3.3; python_version >= '3.6'
 googleapis-common-protos[grpc]==1.53.0; python_version >= '3.6'
-graphviz==0.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+graphviz==0.17; python_version >= '3.6'
 grpc-google-iam-v1==0.12.3
 grpcio-gcp==0.2.2
-grpcio==1.37.1
-gunicorn==20.1.0; python_version >= '3.5'
+grpcio==1.39.0
+gunicorn==20.1.0; python_full_version >= '3.5.0'
+h11==0.12.0; python_version >= '3.6'
+httpcore==0.13.6; python_version >= '3.6'
 httplib2==0.19.1
-idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+httpx==0.18.2; python_version >= '3.6'
+idna==3.2
+importlib-metadata==1.7.0; python_version < '3.9'
 importlib-resources==1.5.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+inflection==0.5.1; python_full_version >= '3.5.0'
 iniconfig==1.1.1
-iso8601==0.1.14
-isort==5.8.0
+iso8601==0.1.16
+isodate==0.6.0
+isort==5.9.3
 itsdangerous==1.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 jinja2==2.11.3
 json-merge-patch==0.2
 jsonschema==3.2.0
-kubernetes==17.17.0
-lazy-object-proxy==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+kubernetes==11.0.0
+lazy-object-proxy==1.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
 libcst==0.3.19; python_version >= '3.6'
 lockfile==0.12.2
 mako==1.1.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-markdown==2.6.11
-markupsafe==2.0.0; python_version >= '3.6'
+markdown==3.3.4; python_version >= '3.6'
+markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 marshmallow-enum==1.5.1
+marshmallow-oneofschema==3.0.1; python_version >= '3.6'
 marshmallow-sqlalchemy==0.23.1; python_version >= '3.6'
-marshmallow==2.21.0
+marshmallow==3.13.0; python_full_version >= '3.5.0'
 mccabe==0.6.1
 mypy-extensions==0.4.3
-natsort==7.1.1; python_version >= '3.4'
-numpy==1.20.3; python_version >= '3.7'
-oauthlib==3.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-packaging==20.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+nox==2020.12.31; python_version >= '3.6'
+numpy==1.21.1; python_version >= '3.7'
+oauthlib==3.1.1; python_version >= '3.6'
+openapi-schema-validator==0.1.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+openapi-spec-validator==0.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+packaging==21.0; python_version >= '3.6'
 pandas-gbq==0.14.1
-pandas==1.2.4
-pathspec==0.8.1
-pendulum==1.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pandas==1.3.1
+pathspec==0.9.0
+pendulum==2.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+platformdirs==2.2.0; python_version >= '3.6'
 pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 prison==0.1.3
-proto-plus==1.18.1; python_version >= '3.6'
-protobuf==3.17.0
+proto-plus==1.19.0; python_version >= '3.6'
+protobuf==3.17.3
 psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
 py==1.10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyarrow==4.0.0
+pyarrow==5.0.0
 pyasn1-modules==0.2.8
 pyasn1==0.4.8
-pycodestyle==2.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pycodestyle==2.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 pydata-google-auth==1.2.0
-pyflakes==2.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pygments==2.9.0; python_version >= '3.5'
+pyflakes==2.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+pygments==2.9.0; python_full_version >= '3.5.0'
 pyjwt==1.7.1
-pyopenssl==20.0.1
-pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyrsistent==0.17.3; python_version >= '3.5'
+pyopenssl==20.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+pyrsistent==0.18.0; python_version >= '3.6'
 pytest-mock==3.6.1
 pytest==6.2.4
 python-daemon==2.3.0
-python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
 python-editor==1.0.4
 python-nvd3==0.15.0
 python-slugify==4.0.1
 python3-openid==3.2.0
 pytz==2021.1
 pytzdata==2020.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pyyaml==5.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-regex==2021.4.4
+pyyaml==5.4.1
+regex==2021.8.3
 requests-oauthlib==1.3.0
-requests==2.23.0; python_version >= '3.0'
+requests==2.26.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+rfc3986[idna2008]==1.5.0
+rich==10.7.0; python_version >= '3.6' and python_version < '4.0'
 rsa==4.7.2; python_version >= '3.6'
-ruamel.yaml.clib==0.2.2; python_version < '3.9' and platform_python_implementation == 'CPython'
-ruamel.yaml==0.16.12
+ruamel.yaml.clib==0.2.6; platform_python_implementation == 'CPython' and python_version < '3.10'
+ruamel.yaml==0.17.10
 setproctitle==1.2.2; python_version >= '3.6'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-sqlalchemy-jsonfield==0.9.0; python_version >= '3.5'
-sqlalchemy-utils==0.37.3; python_version ~= '3.4'
-sqlalchemy==1.3.15
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+sniffio==1.2.0; python_full_version >= '3.5.0'
+sqlalchemy-jsonfield==1.0.0; python_full_version >= '3.5.0'
+sqlalchemy-utils==0.37.8; python_version ~= '3.4'
+sqlalchemy==1.3.18
+swagger-ui-bundle==0.0.8
 tabulate==0.8.9
-tenacity==4.12.0
+tenacity==6.2.0
+termcolor==1.1.0
 text-unidecode==1.3
-thrift==0.13.0
-toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
 typed-ast==1.4.3
 typing-extensions==3.10.0.0
-typing-inspect==0.6.0
-tzlocal==1.5.1
+typing-inspect==0.7.1
 unicodecsv==0.14.1
 uritemplate==3.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'
-websocket-client==0.59.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
-werkzeug==0.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'
+virtualenv==20.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+websocket-client==1.1.1; python_version >= '3.6'
+werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
 wtforms==2.3.3
-zope.deprecation==4.4.0
+zipp==3.5.0; python_version >= '3.6'

scripts/deploy_dag.py

@@ -58,13 +58,15 @@ def main(
     else:
         pipelines = list_subdirs(env_path / "datasets" / dataset_id)
 
-    #  if local:
-    #      runtime_airflow_version = local_airflow_version()
-    #  else:
-    #      runtime_airflow_version = composer_airflow_version(composer_env, composer_region)
+    if local:
+        runtime_airflow_version = local_airflow_version()
+    else:
+        runtime_airflow_version = composer_airflow_version(
+            composer_env, composer_region
+        )
 
     for pipeline_path in pipelines:
-        #  check_airflow_version_compatibility(pipeline_path, runtime_airflow_version)
+        check_airflow_version_compatibility(pipeline_path, runtime_airflow_version)
 
         copy_custom_callables_to_airflow_dags_folder(
             local,

scripts/generate_dag.py

@@ -143,9 +143,11 @@ def generate_task_contents(task: dict, airflow_version: str) -> str:
 
 
 def generate_shared_variables_file(env: str) -> None:
-    pathlib.Path(
+    shared_variables_file = pathlib.Path(
         PROJECT_ROOT / f".{env}" / "datasets" / "shared_variables.json"
-    ).touch()
+    )
+    shared_variables_file.touch()
+    shared_variables_file.write_text("{}", encoding="utf-8")
 
 
 def dag_init(config: dict) -> dict: