From 119f8fb9bcc747b3f544150864f3a22164da708e Mon Sep 17 00:00:00 2001 From: Adler Santos Date: Wed, 1 Sep 2021 18:53:31 -0400 Subject: [PATCH] feat: Support Dataflow operator and job requirements (#153) --- Pipfile | 6 +- Pipfile.lock | 683 ++++++++++++++++++++++++++------------- requirements-dev.txt | 115 ++++--- samples/pipeline.yaml | 136 +++++++- scripts/dag_imports.json | 4 + 5 files changed, 664 insertions(+), 280 deletions(-) diff --git a/Pipfile b/Pipfile index f33422f89..af44ed1ff 100644 --- a/Pipfile +++ b/Pipfile @@ -8,8 +8,10 @@ beautifulsoup4 = "==4.9.3" [dev-packages] apache-airflow = "==2.1.1" -apache-airflow-providers-google = "*" -apache-airflow-providers-cncf-kubernetes = "*" +apache-airflow-providers-apache-beam = "3.0.0" +apache-airflow-providers-google = "5.0.0" +apache-airflow-providers-cncf-kubernetes = "2.0.2" +apache-beam = "2.32.0" black = "==20.8b1" flake8 = "==3.9.2" isort = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 71d242033..f2bec1b1a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9185ec4574d9e0cfc336ffb1d9b26f9575ccf2c642ed204e244d028b95c3c71d" + "sha256": "c15a67db6145f722c66d80d412a1b1cf68b21d450bf4f3f4dd2f740842f18fb9" }, "pipfile-spec": 6, "requires": { @@ -37,11 +37,11 @@ "develop": { "alembic": { "hashes": [ - "sha256:a21fedebb3fb8f6bbbba51a11114f08c78709377051384c9c5ead5705ee93a51", - "sha256:e78be5b919f5bb184e3e0e2dd1ca986f2362e29a2bc933c446fe89f39dbe4e9c" + "sha256:25f996b7408b11493d6a2d669fd9d2ff8d87883fe7434182bc7669d6caa526ab", + "sha256:aea964d3dcc9c205b8759e4e9c1c3935ea3afeee259bffd7ed8414f8085140fb" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.6.5" + "markers": "python_version >= '3.6'", + "version": "==1.7.1" }, "anyio": { "hashes": [ @@ -59,13 +59,21 @@ "index": "pypi", "version": "==2.1.1" }, + "apache-airflow-providers-apache-beam": { + "hashes": [ + "sha256:84b2f0bf6bcdd7feee78d58a6fbd04bf1eba8976d1915090203e35b4a3476d64", + "sha256:aa981a565d6d7dcee6518a82a83d322680a2f97ba4961a212850f89d2bd5067a" + ], + "index": "pypi", + "version": "==3.0.0" + }, "apache-airflow-providers-cncf-kubernetes": { "hashes": [ - "sha256:4efecd18a9f0daa1e37f09c2d952f114b10f0507cafb8dd23630dca0092bf8aa", - "sha256:81ca71025a68f18ebba79de8f005798a10bc1ed75d60ef60fefe44704ec0fa89" + "sha256:6d2fd623161e46971b5842f51e27d4b055ca285589c81185b2de7012a7cbc62b", + "sha256:ff9b702201c148ecb26d01f6dbb0cb151de2229c88fc22ce1ac77bbaf982048d" ], "index": "pypi", - "version": "==2.0.1" + "version": "==2.0.2" }, "apache-airflow-providers-ftp": { "hashes": [ @@ -99,6 +107,37 @@ "markers": "python_version ~= '3.6'", "version": "==2.0.0" }, + "apache-beam": { + "hashes": [ + "sha256:03f3897cf492c8b468fa8b160065dd4f98a9ab4e288ed6d0022b60a200b61da5", + "sha256:19111b3679764051a2383675967579eff22d503f79e77482a7617a58a7e44e21", + "sha256:2faf26a10c37e7fa96ea0efbd50acace2cb2d66a70e8a480902944d81ed2335f", + "sha256:468e5403e4628309a492de2cc5d1162119a1bf1ebd5c67cf91c6ef83bc512f18", + "sha256:58c0404f8f2c28cd899bfd5060ac594a78a0345aefd43e8e134be3bdbed36bf7", + "sha256:676bdc610d4ff32c3216a60755105f70ef552cadf6c2f2ab8a0fa7ffee7c05ba", + "sha256:7ae2268577c582c10beedaed399a5d21752e633e8e2c86867354697c5ca4fbb1", + "sha256:7d851b59800cdddf1410cd1a865c96cc6c450c416874cefe622ad4585737b630", + "sha256:83d5b63302cf49ed823658d4fc5e2ddfca762e7233fe48ae50841e90eb86c7cd", + "sha256:8bbce31dda6cc675b7c9d5ef27f78902500cbb3a40f8b5506e899d2325a4d96f", + "sha256:906d4f10f4cd6609648071820d3928fa8ecf6559b3316d953f3694f7e5eb6c30", + "sha256:9831714f92ca3657497e665761f2bd9db93b020cbdb2ee05ec1099ec7d12eff9", + "sha256:9c5329428e03fd2086437b232efe1e1734f2ab3677fc6187c60e13a2f144ddcd", + "sha256:ad8626a8d321f9a757ed747e792a87d9f870f95521fc59002eb10dad8940c356", + "sha256:c38ca4cd77efb6037e7e010e1a6227f0d9a2c62a345f4ac9e37b5598138187e6", + "sha256:cb43ba5d1d0678457f201149d6ff34f93ab4af79c4bc4ac2a97d6d4953255223", + "sha256:cbe5dbc515e5dba0aef40118433491b30f797bdee248d2f9caaf07c61fa2c892", + "sha256:cdb22c0e8f9db071e9e0357379e7bec9c738e0ddf696c1f91e5cbafefe6d3278", + "sha256:d310395861980dfd495f8c82526004143ca38be314e23441caf89de759b0cb4d", + "sha256:de479e86c49a80380af36f7196ed4c3f51b5864b7557c0aae0e3e6807c2be61e", + "sha256:e877c5d3b0de3568f59c7a96dea5c1a2e328ac8967ca7159b8d5c85c4ddb4014", + "sha256:eec887207b5ec19fdbd826a7102ac9d29c70368e5825fb75164e2eb29283bddb", + "sha256:f0bc4debf97e3120297bdc2dd983fe20a9487499dabb2059ff0f331277c79352", + "sha256:f5d31d18e68237d0512b2256e38913dfe01ceec8a9e8efe09547e4f7df6a2eee", + "sha256:fecab6549be7a309cde2c5711f0cac4e69ff3826880e82d6f8120e6c8dfbef30" + ], + "index": "pypi", + "version": "==2.32.0" + }, "apispec": { "extras": [ "yaml" @@ -132,6 +171,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.3.0" }, + "avro-python3": { + "hashes": [ + "sha256:ca1e77a3da5ac98e8833588f71fb2e170b38e34787ee0e04920de0e9470b7d32" + ], + "markers": "python_version >= '3.5'", + "version": "==1.9.2.1" + }, "babel": { "hashes": [ "sha256:ab49e12b91d937cd11f0b67cb259a57ab4ad2b59ac7a3b41d6c06c0ac5b0def9", @@ -279,6 +325,15 @@ ], "version": "==0.9.1" }, + "crcmod": { + "hashes": [ + "sha256:50586ab48981f11e5b117523d97bb70864a2a1af246cf6e4f5c4a21ef4611cd1", + "sha256:69a2e5c6c36d0f096a7beb4cd34e5f882ec5fd232efb710cdb85d4ff196bd52e", + "sha256:737fb308fa2ce9aed2e29075f0d5980d4a89bfbec48a368c607c5c63b3efb90e", + "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e" + ], + "version": "==1.7" + }, "croniter": { "hashes": [ "sha256:0f97b361fe343301a8f66f852e7d84e4fb7f21379948f71e1bbfe10f5d015fbd", @@ -289,23 +344,26 @@ }, "cryptography": { "hashes": [ - "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d", - "sha256:1e056c28420c072c5e3cb36e2b23ee55e260cb04eee08f702e0edfec3fb51959", - "sha256:240f5c21aef0b73f40bb9f78d2caff73186700bf1bc6b94285699aff98cc16c6", - "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873", - "sha256:37340614f8a5d2fb9aeea67fd159bfe4f5f4ed535b1090ce8ec428b2f15a11f2", - "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713", - "sha256:3d8427734c781ea5f1b41d6589c293089704d4759e34597dce91014ac125aad1", - "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177", - "sha256:8e56e16617872b0957d1c9742a3f94b43533447fd78321514abbe7db216aa250", - "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586", - "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3", - "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca", - "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d", - "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9" + "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e", + "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b", + "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7", + "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085", + "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc", + "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a", + "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498", + "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9", + "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c", + "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7", + "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb", + "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14", + "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af", + "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e", + "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5", + "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06", + "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7" ], "markers": "python_version >= '3.6'", - "version": "==3.4.7" + "version": "==3.4.8" }, "defusedxml": { "hashes": [ @@ -317,11 +375,10 @@ }, "dill": { "hashes": [ - "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f", - "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675" + "sha256:42d8ef819367516592a825746a18073ced42ca169ab1f5f4044134703e7a049c" ], - "markers": "python_version >= '2.7' and python_version != '3.0'", - "version": "==0.3.4" + "markers": "python_version >= '2.6' and python_version != '3.0'", + "version": "==0.3.1.1" }, "distlib": { "hashes": [ @@ -338,6 +395,12 @@ "markers": "python_version >= '3.6'", "version": "==2.1.0" }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, "docutils": { "hashes": [ "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", @@ -354,6 +417,29 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.1.3" }, + "fastavro": { + "hashes": [ + "sha256:0a3e3fa7288db81c1ad521321ced7a895c6a7928511b27393c3f418d6e5f6946", + "sha256:16fcc82844913804b05f28481972b850580ff3103b48f36c021d3b99019f9e3f", + "sha256:24802411fa8557a6703ad1ccdea892f6b72730fc68ecb80214d7208f49a37ebc", + "sha256:30acd5c296155959f371bc4ccf7e76461fef5923c738fa34211b5a65c855aef2", + "sha256:4f1e0093a5ea7f5b72997c91abfa7838cb536c9a1dca9df4c8b7f233e853e0ac", + "sha256:52b9fd197319a1b2eaeb9bfeb7313fa45ef92b49c3884c138b3ab90740232a66", + "sha256:6446204cf55299243c9d43b1c5e355c74e6980e61c9bb58138ea9f3a6c99d826", + "sha256:76aa6545443e1f1d6406c52876f3f7157076b91441943db300bc1c0f0daad6aa", + "sha256:7844fc4f55da7b1948234a7e5a50230fa0f27ff74cdb5efbb107c5cd5db0f49a", + "sha256:836399e5ac80ef8a19e3a0484585c58e52268c87d77b3681615761a9439032ed", + "sha256:a8391357b6857f80b850bb4708c4b8db0f3d1c1e83ab2ea888958d89f46efd26", + "sha256:b53816eec734ca34518c7e748d6d538aca0d3c0111ec726ad9808a098bfdd209", + "sha256:b85f2569102f3b54a3973dd12f5bf5049c125488e70959360dd5cbb8c71d47c7", + "sha256:bb49ece8865c8515f34ff8e307799b8cdb32b6db31acf4174c09b5c8e7fc5280", + "sha256:bfcff703061a9ecce06fc69ac8f994fd1ddb2d55a6ff585fe25a7b45bbda4011", + "sha256:c74f8b48d4e4b36a9013ddb2cbaac68504cfdc48cdfe4753edfd017b5156e18a", + "sha256:e03b80a9fb52b753d948788b0048b2a3f7551ba7f8584e60e90a1b3b5071fdd0" + ], + "markers": "python_version >= '3.6'", + "version": "==1.4.4" + }, "filelock": { "hashes": [ "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", @@ -404,7 +490,7 @@ "hashes": [ "sha256:bbf4467f41c56cf1fd8a5870d2556f419c572aad2b4085757581c3f9b4d7767a" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==3.25.1" }, "flask-login": { @@ -434,12 +520,19 @@ ], "version": "==0.14.3" }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18.2" + }, "google-ads": { "hashes": [ - "sha256:172b25ffabd2f4926f4b300dff74f64692230609f8ca21a9e8780295ab23500b" + "sha256:c6879f1f84c408547e88906e3f021172eda48d9c7d678e7877c618efe5e6de5a" ], "markers": "python_version >= '3.7'", - "version": "==13.0.0" + "version": "==14.0.0" }, "google-api-core": { "extras": [ @@ -478,11 +571,11 @@ }, "google-auth-oauthlib": { "hashes": [ - "sha256:4ab58e6c3dc6ccf112f921fcced40e5426fba266768986ea502228488276eaba", - "sha256:b5a1ce7c617d247ccb2dfbba9d4bfc734b41096803d854a2c52592ae80150a67" + "sha256:3f2a6e802eebbb6fb736a370fbf3b055edcb6b52878bf2f26330b5e041316c73", + "sha256:a90a072f6993f2c327067bf65270046384cda5a8ecb20b94ea9a687f1f233a7a" ], "markers": "python_version >= '3.6'", - "version": "==0.4.5" + "version": "==0.4.6" }, "google-cloud-appengine-logging": { "hashes": [ @@ -494,11 +587,11 @@ }, "google-cloud-audit-log": { "hashes": [ - "sha256:5bf5a53c641b13828154ab21fb209669be69d71cd462f5d6456bf87722fc0eeb", - "sha256:a17064203b985ffa9219937d339cab3583548f07d38e96d67a498aba69c1387d" + "sha256:33a8e5fdc9d98c1c0b4b0502d723941215a586c04ccc44558ec63c16e9194152", + "sha256:a87fdf3c393b830b35c8f7db09094790d0d7babb35068736bea64e1618d286fe" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.1.0" + "version": "==0.1.1" }, "google-cloud-automl": { "hashes": [ @@ -514,19 +607,19 @@ "pandas" ], "hashes": [ - "sha256:1d5a48933556b511d13793d626d955812c38184f56342e7e910845bccc4899a9", - "sha256:44a3ae2732d820ea4295e8b1c1bde50206ca078de2016cec2059f00362b6826b" + "sha256:15ca1f9c8165b584c2d593620354cd3b17c007e87cfd1abd1eebd2c08eb5d109", + "sha256:8431da4c3f9b6a5aca1b811ef64e452d53536160b26252e4d132969ef3e4f173" ], "markers": "python_version < '3.10' and python_version >= '3.6'", - "version": "==2.24.0" + "version": "==2.26.0" }, "google-cloud-bigquery-datatransfer": { "hashes": [ - "sha256:a0292db7cc0060a7b0cfa19ae26206572d085a89d74ea42f68ee4c4f83241cc6", - "sha256:a83578f3c60356b95500aa1a8167989877890a0f08a04412d89f1364e96c96fa" + "sha256:9e421486b4ced687c29f54758a3b9589d32b8563bbf014a3d6cd8d3bcdb2b3e7", + "sha256:fcb71ebe5c5b232d24fe7d666b65709e4fc8db43263c8182e5ed8e5a52abefec" ], "markers": "python_version >= '3.6'", - "version": "==3.3.1" + "version": "==3.3.2" }, "google-cloud-bigquery-storage": { "hashes": [ @@ -561,10 +654,10 @@ }, "google-cloud-datacatalog": { "hashes": [ - "sha256:2faca51e974c46203c09fd4cb2c03fd6e82cd572cc06a2bbc3b401aa419cb09f", - "sha256:c70d7ad720ba7d23d316b7468b6fc0c52e4a1824b7f1b946b8855f6964953562" + "sha256:24c3476d3af68b878d306baa81b326934d57e99935ac19cd845f153dc6524528", + "sha256:ad1bf9991bdee2a2fee44d19e54790a6eb900652841a5d7a32aa1c468a196f49" ], - "version": "==3.4.0" + "version": "==3.4.1" }, "google-cloud-dataproc": { "hashes": [ @@ -584,11 +677,11 @@ }, "google-cloud-kms": { "hashes": [ - "sha256:3aebe99c825e1be3bed937a8a6af056093872328b557745d966244abf7c54b13", - "sha256:9d2d67747a6fc0b69a9d7a5dae98ca2b24de925c24500fb134fb8b39a042166a" + "sha256:0210020813b2f1d8187ac58d95d47d070b747a5745dc974d9fb5fc08cd0fb61b", + "sha256:44be4673ad0a327ea6447967216e124fcd4c7487ca886f000446f6db209988af" ], "markers": "python_version >= '3.6'", - "version": "==2.5.0" + "version": "==2.6.0" }, "google-cloud-language": { "hashes": [ @@ -616,11 +709,11 @@ }, "google-cloud-monitoring": { "hashes": [ - "sha256:3d08d302a4b38c204f1f9f2764701ce8e6b5d8d48356fb761346477b04c565f6", - "sha256:c80ed69ae2ce94de85bbb0b2b90678094599a76e96ed558523dd40d2f2ae71e4" + "sha256:d3d436fab7f50f69bdd47655871ac923e4dcbaf8538808bbe31cf0a3cbdf3514", + "sha256:ddb1e98cecc6eeaedb0227fa8ad8c23e0e31365155bab428d5c9b1bb3fc8da76" ], "markers": "python_version >= '3.6'", - "version": "==2.4.2" + "version": "==2.5.0" }, "google-cloud-os-login": { "hashes": [ @@ -632,11 +725,11 @@ }, "google-cloud-pubsub": { "hashes": [ - "sha256:1788ad25a765ea83499d8f4410d4c8ba026597cb82db46d2f7217f1c2525ad65", - "sha256:7b49095623de343f2ad91a066e1bd59fe196ab8e088c6f11351fb54ca7b1303e" + "sha256:c5cb4a7d02fe0979d45586575fea574a3201d6c24d93b6720bdbfb4ef8eaa329", + "sha256:d52d386617c110c35043f6ff37ccb50d9f37c75b1e5586409ed64a3e8ae61038" ], "markers": "python_version >= '3.6'", - "version": "==2.7.0" + "version": "==2.7.1" }, "google-cloud-redis": { "hashes": [ @@ -728,46 +821,45 @@ }, "google-crc32c": { "hashes": [ - "sha256:0ae3cf54e0d4d83c8af1afe96fc0970fbf32f1b29275f3bfd44ce25c4b622a2b", - "sha256:0dd9b61d0c63043b013349c9ec8a83ec2b05c96410c5bc257da5d0de743fc171", - "sha256:110157fb19ab5db15603debfaf5fcfbac9627576787d9caf8618ff96821a7a1f", - "sha256:1dc6904c0d958f43102c85d70792cca210d3d051ddbeecd0eff10abcd981fdfa", - "sha256:298a9a922d35b123a73be80233d0f19c6ea01f008743561a8937f9dd83fb586b", - "sha256:34a97937f164147aefa53c3277364fd3bfa7fd244cbebbd5a976fa8325fb496b", - "sha256:364eb36e8d9d34542c17b0c410035b0557edd4300a92ed736b237afaa0fd6dae", - "sha256:49838ede42592154f9fcd21d07c7a43a67b00a36e252f82ae72542fde09dc51f", - "sha256:51f4aa06125bf0641f65fb83268853545dbeb36b98ccfec69ef57dcb6b73b176", - "sha256:6789db0b12aab12a0f04de22ed8412dfa5f6abd5a342ea19f15355064e1cc387", - "sha256:78cf5b1bd30f3a6033b41aa4ce8c796870bc4645a15d3ef47a4b05d31b0a6dc1", - "sha256:7c5138ed2e815189ba524756e027ac5833365e86115b1c2e6d9e833974a58d82", - "sha256:80abca603187093ea089cd1215c3779040dda55d3cdabc0cd5ea0e10df7bff99", - "sha256:8ed8f6dc4f55850cba2eb22b78902ad37f397ee02692d3b8e00842e9af757321", - "sha256:91ad96ee2958311d0bb75ffe5c25c87fb521ef547c09e04a8bb6143e75fb1367", - "sha256:92ed6062792b989e84621e07a5f3d37da9cc3153b77d23a582921f14863af31d", - "sha256:9372211acbcc207f63ffaffea1d05f3244a21311e4710721ffff3e8b7a0d24d0", - "sha256:a64e0e8ed6076a8d867fc4622ad821c55eba8dff1b48b18f56b7c2392e22ab9d", - "sha256:a6c8a712ffae56c805ca732b735af02860b246bed2c1acb38ea954a8b2dc4581", - "sha256:ab2b31395fbeeae6d15c98bd7f8b9fb76a18f18f87adc11b1f6dbe8f90d8382f", - "sha256:ae7b9e7e2ca1b06c3a68b6ef223947a52c30ffae329b1a2be3402756073f2732", - "sha256:b5ea1055fe470334ced844270e7c808b04fe31e3e6394675daa77f6789ca9eff", - "sha256:d0630670d27785d7e610e72752dc8087436d00d2c7115e149c0a754babb56d3e", - "sha256:d4a0d4fb938c2c3c0076445c9bd1215a3bd3df557b88d8b05ec2889ca0c92f8d", - "sha256:dff5bd1236737f66950999d25de7a78144548ebac7788d30ada8c1b6ead60b27", - "sha256:e5af77656e8d367701f40f80a91c985ca43319f322f0a36ba9f93909d0bc4cb2", - "sha256:e6458c41236d37cb982120b070ebcc115687c852bee24cdd18792da2640cf44d", - "sha256:ea170341a4a9078a067b431044cd56c73553425833a7c2bb81734777a230ad4b", - "sha256:ef2ed6d0ac4de4ac602903e203eccd25ec8e37f1446fe1a3d2953a658035e0a5" + "sha256:013524fca74673d8ded31f2236d8b6c017408ce170fd2979708a5b362a065811", + "sha256:03672ae1c1cb41b79fc6c2379aeb61e2b657fcaa4ce11871c87b22af70af9a30", + "sha256:0484ddc264080122f1de32c765ce34f8358fd9004c8b0c4763aade57bad36350", + "sha256:0a4ed1488a100495fef0bd0e9dc6358298e2e3cff4b7fa1b583203f0c5441619", + "sha256:0e4c2b20f2e59bd089f2bbdb5b49e9aeba9151afad487dfc28ad28d0b1741309", + "sha256:2032e9be71b0248e56b035675b287811f2efd96ef81660c2e97f40b670a26458", + "sha256:292c4a24c7f3d45ac053e471c2e36b900d13af8b35d843dc9b50f77f6adfb9f9", + "sha256:2c67aafb77e1c7fc37bffc8a7fe20b39181c4c7c2d7563ad8cf9e0e6e165922e", + "sha256:3ad1d5e5cff49e929b7e567c5d6e17d7b8f1092e90b5ee2b434451239d048832", + "sha256:519d76ab82878c649e0618b05fc2adabe79b0491bde9ca428da06ea377594936", + "sha256:56358f82ca95c815aea6ffe7db2f72427a509098740429a23ba726627eb5cbe3", + "sha256:58246c72a4d623823b93026bfb868623e1287b1364c2ba93d307891e0282b373", + "sha256:5bb34e72448cfbdd41dc291eccf6359171ee72d44b0ce31198b672df03c76829", + "sha256:8827f2116d17cce236dddad82c38c5d4ea99f24b0d938dcc73475e46988117cf", + "sha256:949f2e0eace8e5878c7d63a9aeef137d35fac161afe4960a929d1400a7e6941b", + "sha256:94c6e590260057d7a73f1b7d474e2e5d3b4367f68cfc0feeff6633534d01011d", + "sha256:9c41440ad996a88f1ec7717ea7f7ba4a93066e35f06766d6c5de1c5ad23cab87", + "sha256:b69605a56e0aaa0205f69d8dd256bbda2a52094b40d619ad27c4ac520d2b2fa7", + "sha256:bbb775b81b9a4cad1c19ea42ad4082f0dae5670ad47f80a39d7ef170faec8b81", + "sha256:c33add3f566cf8f39026bbbd9e49cf8044d1471bce0819844280b25dac6545a4", + "sha256:c5cdf5121079db2b33baddffebf5f1c5985757029284b5ccc88ed7114d179d63", + "sha256:d04cb334da5f05ac1003ccb01cc99f40c138ed43288a2ca139b07aaeaacf6635", + "sha256:d4d19af1f3e73c1f690bfb2a592b1ff24a1dc8dcfeb7cf3d7783b20570de1940", + "sha256:da570ac61913bdfa16d21a5471471fe0b5ad155f77f0542d89242f95c4b7b788", + "sha256:e3fec4f2c39c98a3d33483edcce223dc47aa2d89f87085c3d4ff2bde42b84997", + "sha256:e47f918025018b4f5d34778d378312d4b99b9ac4cf52460110767809143f5cff", + "sha256:f009b5826341687748e47d10fc4ab29271999ff9bc206c8f0602514dc4d4fbac", + "sha256:ff524539ec7c79a3ff04312f2fb9d780ec3741604968609e4f0e1b32cc85ee03" ], - "markers": "python_version >= '3.5'", - "version": "==1.1.2" + "markers": "python_version >= '3.6'", + "version": "==1.1.3" }, "google-resumable-media": { "hashes": [ - "sha256:092f39153cd67a4e409924edf08129f43cc72e630a1eb22abec93e80155df4ba", - "sha256:ce38555d250bd70b0c2598bf61e99003cb8c569b0176ec0e3f38b86f9ffff581" + "sha256:5dff19b008e08f307b4ae9f3f1fdb8e072698b27111710ca3867c9e63540bdf9", + "sha256:cac55be7802e3424b8f022d8a572a8349327e7ce8494eee5e0f4df02458b1813" ], "markers": "python_version >= '3.6'", - "version": "==1.3.3" + "version": "==2.0.1" }, "googleapis-common-protos": { "extras": [ @@ -873,6 +965,13 @@ "markers": "python_version >= '3.6'", "version": "==0.12.0" }, + "hdfs": { + "hashes": [ + "sha256:05912125cfc68075387f271654dac185dc1aba8b347519f6a14d1395e39d7749", + "sha256:bc92ce4347f106d48b541f756caa930476998cfd3eed477ffbd63ae9ad1cdc22" + ], + "version": "==2.6.0" + }, "httpcore": { "hashes": [ "sha256:b0d16f0012ec88d8cc848f5a55f8a03158405f4bca02ee49bc4ca2c1fda49f3e", @@ -890,11 +989,11 @@ }, "httpx": { "hashes": [ - "sha256:979afafecb7d22a1d10340bafb403cf2cb75aff214426ff206521fc79d26408c", - "sha256:9f99c15d33642d38bce8405df088c1c4cfd940284b4290cacbfb02e64f4877c6" + "sha256:92ecd2c00c688b529eda11cedb15161eaf02dee9116712f621c70d9a40b2cdd0", + "sha256:9bd728a6c5ec0a9e243932a9983d57d3cc4a87bb4f554e1360fce407f78f9435" ], "markers": "python_version >= '3.6'", - "version": "==0.18.2" + "version": "==0.19.0" }, "idna": { "hashes": [ @@ -916,7 +1015,7 @@ "sha256:6f87df66833e1942667108628ec48900e02a4ab4ad850e25fbf07cb17cf734ca", "sha256:85dc0b9b325ff78c8bef2e4ff42616094e16b98ebd5e3b50fe7e2f0bbcdcde49" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "markers": "python_version < '3.9'", "version": "==1.5.0" }, "inflection": { @@ -1038,11 +1137,11 @@ }, "mako": { "hashes": [ - "sha256:17831f0b7087c313c0ffae2bcbbd3c1d5ba9eeac9c38f2eb7b50e8c99fe9d5ab", - "sha256:aea166356da44b9b830c8023cd9b557fa856bd8b4035d6de771ca027dfc5cc6e" + "sha256:169fa52af22a91900d852e937400e79f535496191c63712e3b9fda5a9bed6fc3", + "sha256:6804ee66a7f6a6416910463b00d76a7b25194cd27f1918500c5bd7be2a088a23" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.1.4" + "version": "==1.1.5" }, "markdown": { "hashes": [ @@ -1165,39 +1264,40 @@ }, "numpy": { "hashes": [ - "sha256:09858463db6dd9f78b2a1a05c93f3b33d4f65975771e90d2cf7aadb7c2f66edf", - "sha256:209666ce9d4a817e8a4597cd475b71b4878a85fa4b8db41d79fdb4fdee01dde2", - "sha256:298156f4d3d46815eaf0fcf0a03f9625fc7631692bd1ad851517ab93c3168fc6", - "sha256:30fc68307c0155d2a75ad19844224be0f2c6f06572d958db4e2053f816b859ad", - "sha256:423216d8afc5923b15df86037c6053bf030d15cc9e3224206ef868c2d63dd6dc", - "sha256:426a00b68b0d21f2deb2ace3c6d677e611ad5a612d2c76494e24a562a930c254", - "sha256:466e682264b14982012887e90346d33435c984b7fead7b85e634903795c8fdb0", - "sha256:51a7b9db0a2941434cd930dacaafe0fc9da8f3d6157f9d12f761bbde93f46218", - "sha256:52a664323273c08f3b473548bf87c8145b7513afd63e4ebba8496ecd3853df13", - "sha256:550564024dc5ceee9421a86fc0fb378aa9d222d4d0f858f6669eff7410c89bef", - "sha256:5de64950137f3a50b76ce93556db392e8f1f954c2d8207f78a92d1f79aa9f737", - "sha256:640c1ccfd56724f2955c237b6ccce2e5b8607c3bc1cc51d3933b8c48d1da3723", - "sha256:7fdc7689daf3b845934d67cb221ba8d250fdca20ac0334fea32f7091b93f00d3", - "sha256:805459ad8baaf815883d0d6f86e45b3b0b67d823a8f3fa39b1ed9c45eaf5edf1", - "sha256:92a0ab128b07799dd5b9077a9af075a63467d03ebac6f8a93e6440abfea4120d", - "sha256:9f2dc79c093f6c5113718d3d90c283f11463d77daa4e83aeeac088ec6a0bda52", - "sha256:a5109345f5ce7ddb3840f5970de71c34a0ff7fceb133c9441283bb8250f532a3", - "sha256:a55e4d81c4260386f71d22294795c87609164e22b28ba0d435850fbdf82fc0c5", - "sha256:a9da45b748caad72ea4a4ed57e9cd382089f33c5ec330a804eb420a496fa760f", - "sha256:b160b9a99ecc6559d9e6d461b95c8eec21461b332f80267ad2c10394b9503496", - "sha256:b342064e647d099ca765f19672696ad50c953cac95b566af1492fd142283580f", - "sha256:b5e8590b9245803c849e09bae070a8e1ff444f45e3f0bed558dd722119eea724", - "sha256:bf75d5825ef47aa51d669b03ce635ecb84d69311e05eccea083f31c7570c9931", - "sha256:c01b59b33c7c3ba90744f2c695be571a3bd40ab2ba7f3d169ffa6db3cfba614f", - "sha256:d96a6a7d74af56feb11e9a443150216578ea07b7450f7c05df40eec90af7f4a7", - "sha256:dd0e3651d210068d13e18503d75aaa45656eef51ef0b261f891788589db2cc38", - "sha256:e167b9805de54367dcb2043519382be541117503ce99e3291cc9b41ca0a83557", - "sha256:e42029e184008a5fd3d819323345e25e2337b0ac7f5c135b7623308530209d57", - "sha256:f545c082eeb09ae678dd451a1b1dbf17babd8a0d7adea02897a76e639afca310", - "sha256:fde50062d67d805bc96f1a9ecc0d37bfc2a8f02b937d2c50824d186aa91f2419" + "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010", + "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd", + "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43", + "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9", + "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df", + "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400", + "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2", + "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4", + "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a", + "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6", + "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8", + "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b", + "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8", + "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb", + "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2", + "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f", + "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4", + "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a", + "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16", + "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f", + "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69", + "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65", + "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17", + "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48" ], "markers": "python_version >= '3.7'", - "version": "==1.21.2" + "version": "==1.20.3" + }, + "oauth2client": { + "hashes": [ + "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac", + "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6" + ], + "version": "==4.1.3" }, "oauthlib": { "hashes": [ @@ -1225,6 +1325,33 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.3.1" }, + "orjson": { + "hashes": [ + "sha256:084de43ca9b19ad58c618c9f1ff93784e0190df2d88a02ae24c3cdebe9f2e9f7", + "sha256:1014a6f514b39dc414fce60568c9e7f635de97a1f1f5972ebc38f88a6160944a", + "sha256:353cc079cedfe990ea2d2186306f766e0d47bba63acd072e22d6df96c67be993", + "sha256:39aa7d42c9760fba36c37adb1d9c6752696ce9443c5dcb65222dd0994b5735e1", + "sha256:4606907b9aaec9fea6159ac14f838dbd2851f18b05fb414c4b3143bff9f2bb0d", + "sha256:4c702c78c33416fc8a138c5ec36eef5166ecfe8990c8f99c97551cd37c396e4d", + "sha256:4ebb464b8b557a1401a03da6f41761544886db95b52280e60d25549da7427453", + "sha256:5eb9d7f2f45e12cbc7500da4176f2d3221a73891b4be505fe79c52cbb800e872", + "sha256:5f78ed46b179585272a5670537f2203dbb7b3e2f8e4db1be72839cc423e2daef", + "sha256:720a7d7ba1dcf32bbd8fb380370b1fdd06ed916caea48403edd64f2ccf7883c1", + "sha256:7936bef5589c9955ebee3423df51709d5f3b37ef54b830239bddb9fa5ead99f4", + "sha256:82e3afbf404cb91774f894ed7bf52fd83bb1cc6bd72221711f4ce4e7774f0560", + "sha256:8a5e46418f51f03060f91d743b59aed70c8d02a5012428365cfa20b7f670e903", + "sha256:8d4430e0cc390c1d745aea3827fd0c6fd7aa5f0690de30a2fe25c406aa5efa20", + "sha256:8f105e9290f901a618a0ced87f785fce2fcf6ab753699de081d82ee05c90f038", + "sha256:9e4a26212851ea8ff81dee7e4e0da7e1e63b5b4f4330a8b4f27e99f1ba3f758b", + "sha256:a99f310960e3acdda72ba1e98df8bf8c9145d90a0f72719786f43f4ea6937846", + "sha256:b68a601f49c0328bf16498309e56ab87c1d6c2bb0287abf70329eb958d565c62", + "sha256:be79e0ddea7f3a47332ec9573365c0b8a8cce4357e9682050f53c1bc75c1571f", + "sha256:c3beff02a339f194274ec1fcf03e2c1563e84f297b568eb3d45751722454a52e", + "sha256:fce5ada0f8dd7c9e16c675626a29dfc5cc766e1eb67d8021b1e77d0861e4e850" + ], + "markers": "python_version >= '3.7'", + "version": "==3.6.3" + }, "packaging": { "hashes": [ "sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7", @@ -1301,34 +1428,34 @@ }, "platformdirs": { "hashes": [ - "sha256:4666d822218db6a262bdfdc9c39d21f23b4cfdb08af331a81e92751daf6c866c", - "sha256:632daad3ab546bd8e6af0537d09805cec458dce201bccfe23012df73332e181e" + "sha256:15b056538719b1c94bdaccb29e5f81879c7f7f0f4a153f46086d155dffcd4f0f", + "sha256:8003ac87717ae2c7ee1ea5a84a1a61e87f3fbd16eb5aadba194ea30a9019f648" ], "markers": "python_version >= '3.6'", - "version": "==2.2.0" + "version": "==2.3.0" }, "pluggy": { "hashes": [ - "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", - "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.13.1" + "markers": "python_version >= '3.6'", + "version": "==1.0.0" }, "prison": { "hashes": [ - "sha256:58e972709e9e2feabc1aa06cb6493f9c43b55a8afd7d653cf0aa82e3923fbd35", - "sha256:a02b55719d033c4e7efc1d1660ed884a73743d89effb48205c5743c3c658b799" + "sha256:e6cd724044afcb1a8a69340cad2f1e3151a5839fd3a8027fd1357571e797c599", + "sha256:f90bab63fca497aa0819a852f64fb21a4e181ed9f6114deaa5dc04001a7555c5" ], - "version": "==0.1.3" + "version": "==0.2.1" }, "proto-plus": { "hashes": [ - "sha256:ce6695ce804383ad6f392c4bb1874c323896290a1f656560de36416ba832d91e", - "sha256:df7c71c08dc06403bdb0fba58cf9bf5f217198f6488c26b768f81e03a738c059" + "sha256:600e2793ec1a0bf2b9e5ba18cd9eccbc1bc690a03c73b571bbe59789fbaaeecc", + "sha256:cfc45474c7eda0fe3c4b9eca2542124f2a0ff5543242bec61e8d08bce0f5bd48" ], "markers": "python_version >= '3.6'", - "version": "==1.19.0" + "version": "==1.18.1" }, "protobuf": { "hashes": [ @@ -1406,36 +1533,33 @@ }, "pyarrow": { "hashes": [ - "sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923", - "sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614", - "sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922", - "sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820", - "sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4", - "sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231", - "sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce", - "sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2", - "sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc", - "sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf", - "sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c", - "sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7", - "sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54", - "sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae", - "sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02", - "sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d", - "sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d", - "sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af", - "sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b", - "sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90", - "sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf", - "sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d", - "sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03", - "sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161", - "sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6", - "sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe", - "sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c", - "sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406" + "sha256:04be0f7cb9090bd029b5b53bed628548fef569e5d0b5c6cd7f6d0106dbbc782d", + "sha256:0fde9c7a3d5d37f3fe5d18c4ed015e8f585b68b26d72a10d7012cad61afe43ff", + "sha256:11517f0b4f4acbab0c37c674b4d1aad3c3dfea0f6b1bb322e921555258101ab3", + "sha256:150db335143edd00d3ec669c7c8167d401c4aa0a290749351c80bbf146892b2e", + "sha256:24040a20208e9b16ba7b284624ebfe67e40f5c40b5dc8d874da322ac0053f9d3", + "sha256:33c457728a1ce825b80aa8c8ed573709f1efe72003d45fa6fdbb444de9cc0b74", + "sha256:423cd6a14810f4e40cb76e13d4240040fc1594d69fe1c4f2c70be00ad512ade5", + "sha256:5387db80c6a7b5598884bf4df3fc546b3373771ad614548b782e840b71704877", + "sha256:5a76ec44af838862b23fb5cfc48765bc7978f7b58a181c96ad92856280de548b", + "sha256:5f2660f59dfcfd34adac7c08dc7f615920de703f191066ed6277628975f06878", + "sha256:6b7bd8f5aa327cc32a1b9b02a76502851575f5edb110f93c59a45c70211a5618", + "sha256:72cf3477538bd8504f14d6299a387cc335444f7a188f548096dfea9533551f02", + "sha256:76b75a9cfc572e890a1e000fd532bdd2084ec3f1ee94ee51802a477913a21072", + "sha256:a81adbfbe2f6528d4593b5a8962b2751838517401d14e9d4cab6787478802693", + "sha256:a968375c66e505f72b421f5864a37f51aad5da61b6396fa283f956e9f2b2b923", + "sha256:afd4f7c0a225a326d2c0039cdc8631b5e8be30f78f6b7a3e5ce741cf5dd81c72", + "sha256:b05bdd513f045d43228247ef4d9269c88139788e2d566f4cb3e855e282ad0330", + "sha256:c2733c9bcd00074ce5497dd0a7b8a10c91d3395ddce322d7021c7fdc4ea6f610", + "sha256:d0f080b2d9720bec42624cb0df66f60ae66b84a2ccd1fe2c291322df915ac9db", + "sha256:dcd20ee0240a88772eeb5691102c276f5cdec79527fb3a0679af7f93f93cb4bd", + "sha256:e1351576877764fb4d5690e4721ce902e987c85f4ab081c70a34e1d24646586e", + "sha256:e44dfd7e61c9eb6dda59bc49ad69e77945f6d049185a517c130417e3ca0494d8", + "sha256:ee3d87615876550fee9a523307dd4b00f0f44cf47a94a32a07793da307df31a0", + "sha256:fa7b165cfa97158c1e6d15c68428317b4f4ae786d1dc2dbab43f1328c1eb43aa", + "sha256:fe976695318560a97c6d31bba828eeca28c44c6f6401005e54ba476a28ac0a10" ], - "version": "==5.0.0" + "version": "==4.0.1" }, "pyasn1": { "hashes": [ @@ -1496,6 +1620,14 @@ ], "version": "==1.2.0" }, + "pydot": { + "hashes": [ + "sha256:248081a39bcb56784deb018977e428605c1c758f10897a339fce1dd728ff007d", + "sha256:66c98190c65b8d2e2382a441b4c0edfdb4f4c025ef9cb9874de478fb0793a451" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.4.2" + }, "pyflakes": { "hashes": [ "sha256:7893783d01b8a89811dd72d7dfd4d84ff098e5eed95cfa8905b22bbffe52efc3", @@ -1519,6 +1651,109 @@ ], "version": "==1.7.1" }, + "pymongo": { + "hashes": [ + "sha256:02dc0b0f48ed3cd06c13b7e31b066bf91e00dac5f8147b0a0a45f9009bfab857", + "sha256:053b4ebf91c7395d1fcd2ce6a9edff0024575b7b2de6781554a4114448a8adc9", + "sha256:070a4ef689c9438a999ec3830e69b208ff0d12251846e064d947f97d819d1d05", + "sha256:072ba7cb65c8aa4d5c5659bf6722ee85781c9d7816dc00679b8b6f3dff1ddafc", + "sha256:0b6055e0ef451ff73c93d0348d122a0750dddf323b9361de5835dac2f6cf7fc1", + "sha256:11f9e0cfc84ade088a38df2708d0b958bb76360181df1b2e1e1a41beaa57952b", + "sha256:18290649759f9db660972442aa606f845c368db9b08c4c73770f6da14113569b", + "sha256:186104a94d39b8412f8e3de385acd990a628346a4402d4f3a288a82b8660bd22", + "sha256:1970cfe2aec1bf74b40cf30c130ad10cd968941694630386db33e1d044c22a2e", + "sha256:19d4bd0fc29aa405bb1781456c9cfff9fceabb68543741eb17234952dbc2bbb0", + "sha256:1bab889ae7640eba739f67fcbf8eff252dddc60d4495e6ddd3a87cd9a95fdb52", + "sha256:1bc6fe7279ff40c6818db002bf5284aa03ec181ea1b1ceaeee33c289d412afa7", + "sha256:208debdcf76ed39ebf24f38509f50dc1c100e31e8653817fedb8e1f867850a13", + "sha256:2399a85b54f68008e483b2871f4a458b4c980469c7fe921595ede073e4844f1e", + "sha256:246ec420e4c8744fceb4e259f906211b9c198e1f345e6158dcd7cbad3737e11e", + "sha256:24f8aeec4d6b894a6128844e50ff423dd02462ee83addf503c598ee3a80ddf3d", + "sha256:255a35bf29185f44b412e31a927d9dcedda7c2c380127ecc4fbf2f61b72fa978", + "sha256:2dbfbbded947a83a3dffc2bd1ec4750c17e40904692186e2c55a3ad314ca0222", + "sha256:2e92aa32300a0b5e4175caec7769f482b292769807024a86d674b3f19b8e3755", + "sha256:316c1b8723afa9870567cd6dff35d440b2afeda53aa13da6c5ab85f98ed6f5ca", + "sha256:333bfad77aa9cd11711febfb75eed0bb537a1d022e1c252714dad38993590240", + "sha256:39dafa2eaf577d1969f289dc9a44501859a1897eb45bd589e93ce843fc610800", + "sha256:3ce83f17f641a62a4dfb0ba1b8a3c1ced7c842f511b5450d90c030c7828e3693", + "sha256:46d5ec90276f71af3a29917b30f2aec2315a2759b5f8d45b3b63a07ca8a070a3", + "sha256:48d5bc80ab0af6b60c4163c5617f5cd23f2f880d7600940870ea5055816af024", + "sha256:4ba0def4abef058c0e5101e05e3d5266e6fffb9795bbf8be0fe912a7361a0209", + "sha256:5af390fa9faf56c93252dab09ea57cd020c9123aa921b63a0ed51832fdb492e7", + "sha256:5e574664f1468872cd40f74e4811e22b1aa4de9399d6bcfdf1ee6ea94c017fcf", + "sha256:625befa3bc9b40746a749115cc6a15bf20b9bd7597ca55d646205b479a2c99c7", + "sha256:6261bee7c5abadeac7497f8f1c43e521da78dd13b0a2439f526a7b0fc3788824", + "sha256:657ad80de8ec9ed656f28844efc801a0802961e8c6a85038d97ff6f555ef4919", + "sha256:6b89dc51206e4971c5568c797991eaaef5dc2a6118d67165858ad11752dba055", + "sha256:6e66780f14c2efaf989cd3ac613b03ee6a8e3a0ba7b96c0bb14adca71a427e55", + "sha256:6fb3f85870ae26896bb44e67db94045f2ebf00c5d41e6b66cdcbb5afd644fc18", + "sha256:701e08457183da70ed96b35a6b43e6ba1df0b47c837b063cde39a1fbe1aeda81", + "sha256:70761fd3c576b027eec882b43ee0a8e5b22ff9c20cdf4d0400e104bc29e53e34", + "sha256:73b400fdc22de84bae0dbf1a22613928a41612ec0a3d6ed47caf7ad4d3d0f2ff", + "sha256:7412a36798966624dc4c57d64aa43c2d1100b348abd98daaac8e99e57d87e1d7", + "sha256:78ecb8d42f50d393af912bfb1fb1dcc9aabe9967973efb49ee577e8f1cea494c", + "sha256:7c6a9948916a7bbcc6d3a9f6fb75db1acb5546078023bfb3db6efabcd5a67527", + "sha256:7c72d08acdf573455b2b9d2b75b8237654841d63a48bc2327dc102c6ee89b75a", + "sha256:7d98ce3c42921bb91566121b658e0d9d59a9082a9bd6f473190607ff25ab637f", + "sha256:845a8b83798b2fb11b09928413cb32692866bfbc28830a433d9fa4c8c3720dd0", + "sha256:94d38eba4d1b5eb3e6bfece0651b855a35c44f32fd91f512ab4ba41b8c0d3e66", + "sha256:9a13661681d17e43009bb3e85e837aa1ec5feeea1e3654682a01b8821940f8b3", + "sha256:a0e5dff6701fa615f165306e642709e1c1550d5b237c5a7a6ea299886828bd50", + "sha256:a2239556ff7241584ce57be1facf25081669bb457a9e5cbe68cce4aae6567aa1", + "sha256:a325600c83e61e3c9cebc0c2b1c8c4140fa887f789085075e8f44c8ff2547eb9", + "sha256:a3566acfbcde46911c52810374ecc0354fdb841284a3efef6ff7105bc007e9a8", + "sha256:a634a4730ce0b0934ed75e45beba730968e12b4dafbb22f69b3b2f616d9e644e", + "sha256:a6d055f01b83b1a4df8bb0c61983d3bdffa913764488910af3620e5c2450bf83", + "sha256:a752ecd1a26000a6d67be7c9a2e93801994a8b3f866ac95b672fbc00225ca91a", + "sha256:a9ba2a63777027b06b116e1ea8248e66fd1bedc2c644f93124b81a91ddbf6d88", + "sha256:aaa038eafb7186a4abbb311fcf20724be9363645882bbce540bef4797e812a7a", + "sha256:af586e85144023686fb0af09c8cdf672484ea182f352e7ceead3d832de381e1b", + "sha256:b0a0cf39f589e52d801fdef418305562bc030cdf8929217463c8433c65fd5c2f", + "sha256:b1c4874331ab960429caca81acb9d2932170d66d6d6f87e65dc4507a85aca152", + "sha256:b3b5b3cbc3fdf4fcfa292529df2a85b5d9c7053913a739d3069af1e12e12219f", + "sha256:b542d56ed1b8d5cf3bb36326f814bd2fbe8812dfd2582b80a15689ea433c0e35", + "sha256:b6ea08758b6673610b3c5bdf47189286cf9c58b1077558706a2f6f8744922527", + "sha256:b754240daafecd9d5fce426b0fbaaed03f4ebb130745c8a4ae9231fffb8d75e5", + "sha256:b772bab31cbd9cb911e41e1a611ebc9497f9a32a7348e2747c38210f75c00f41", + "sha256:b88d1742159bc93a078733f9789f563cef26f5e370eba810476a71aa98e5fbc2", + "sha256:b8bf42d3b32f586f4c9e37541769993783a534ad35531ce8a4379f6fa664fba9", + "sha256:bc9ac81e73573516070d24ce15da91281922811f385645df32bd3c8a45ab4684", + "sha256:c188db6cf9e14dbbb42f5254292be96f05374a35e7dfa087cc2140f0ff4f10f6", + "sha256:c55782a55f4a013a78ac5b6ee4b8731a192dea7ab09f1b6b3044c96d5128edd4", + "sha256:c5cab230e7cabdae9ff23c12271231283efefb944c1b79bed79a91beb65ba547", + "sha256:cbf8672edeb7b7128c4a939274801f0e32bbf5159987815e3d1eace625264a46", + "sha256:cc2894fe91f31a513860238ede69fe47fada21f9e7ddfe73f7f9fef93a971e41", + "sha256:cda9e628b1315beec8341e8c04aac9a0b910650b05e0751e42e399d5694aeacb", + "sha256:ceae3ab9e11a27aaab42878f1d203600dfd24f0e43678b47298219a0f10c0d30", + "sha256:ced944dcdd561476deef7cb7bfd4987c69fffbfeff6d02ca4d5d4fd592d559b7", + "sha256:d04ca462cb99077e6c059e97c072957caf2918e6e4191e3161c01c439e0193de", + "sha256:d1131562ddc2ea8a446f66c2648d7dabec2b3816fc818528eb978a75a6d23b2e", + "sha256:d1740776b70367277323fafb76bcf09753a5cc9824f5d705bac22a34ff3668ea", + "sha256:d6e11ffd43184d529d6752d6dcb62b994f903038a17ea2168ef1910c96324d26", + "sha256:d73e10772152605f6648ba4410318594f1043bbfe36d2fadee7c4b8912eff7c5", + "sha256:da8288bc4a7807c6715416deed1c57d94d5e03e93537889e002bf985be503f1a", + "sha256:db93608a246da44d728842b8fa9e45aa9782db76955f634a707739a8d53ff544", + "sha256:dcd3d0009fbb6e454d729f8b22d0063bd9171c31a55e0f0271119bd4f2700023", + "sha256:dd1f49f949a658c4e8f81ed73f9aad25fcc7d4f62f767f591e749e30038c4e1d", + "sha256:dd6ff2192f34bd622883c745a56f492b1c9ccd44e14953e8051c33024a2947d5", + "sha256:e018a4921657c2d3f89c720b7b90b9182e277178a04a7e9542cc79d7d787ca51", + "sha256:e2b7670c0c8c6b501464150dd49dd0d6be6cb7f049e064124911cec5514fa19e", + "sha256:e7a33322e08021c37e89cae8ff06327503e8a1719e97c69f32c31cbf6c30d72c", + "sha256:e8a82e35d52ad6f867e88096a1a2b9bdc7ec4d5e65c7b4976a248bf2d1a32a93", + "sha256:e9faf8d4712d5ea301d74abfcf6dafe4b7f4af7936e91f283b0ad7bf69ed3e3a", + "sha256:ec5ca7c0007ce268048bbe0ffc6846ed1616cf3d8628b136e81d5e64ff3f52a2", + "sha256:eee42a1cc06565f6b21caa1f504ec15e07de7ebfd520ab57f8cb3308bc118e22", + "sha256:f2acf9bbcd514e901f82c4ca6926bbd2ae61716728f110b4343eb0a69612d018", + "sha256:f55c1ddcc1f6050b07d468ce594f55dbf6107b459e16f735d26818d7be1e9538", + "sha256:f6977a520bd96e097c8a37a8cbb9faa1ea99d21bf84190195056e25f688af73d", + "sha256:f94c7d22fb36b184734dded7345a04ec5f95130421c775b8b0c65044ef073f34", + "sha256:fa8957e9a1b202cb45e6b839c241cd986c897be1e722b81d2f32e9c6aeee80b0", + "sha256:fd3854148005c808c485c754a184c71116372263709958b42aefbef2e5dd373a", + "sha256:fe5872ce6f9627deac8314bdffd3862624227c3de4c17ef0cc78bbf0402999eb", + "sha256:ffbae429ba9e42d0582d3ac63fdb410338892468a2107d8ff68228ec9a39a0ed" + ], + "version": "==3.12.0" + }, "pyopenssl": { "hashes": [ "sha256:4c231c759543ba02560fcd2480c48dcec4dae34c9da7d3747c508227e0624b51", @@ -1564,11 +1799,11 @@ }, "pytest": { "hashes": [ - "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b", - "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890" + "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89", + "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134" ], "index": "pypi", - "version": "==6.2.4" + "version": "==6.2.5" }, "pytest-mock": { "hashes": [ @@ -1593,16 +1828,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, - "python-editor": { - "hashes": [ - "sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d", - "sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b", - "sha256:5f98b069316ea1c2ed3f67e7f5df6c0d8f10b689964a4a811ff64f0106819ec8", - "sha256:c3da2053dbab6b29c94e43c486ff67206eafbe7eb52dbec7390b5e2fb05aac77", - "sha256:ea87e17f6ec459e780e4221f295411462e0d0810858e055fc514684350a2f522" - ], - "version": "==1.0.4" - }, "python-nvd3": { "hashes": [ "sha256:fbd75ff47e0ef255b4aa4f3a8b10dc8b4024aa5a9a7abed5b2406bd3cb817715" @@ -1673,41 +1898,49 @@ }, "regex": { "hashes": [ - "sha256:026beb631097a4a3def7299aa5825e05e057de3c6d72b139c37813bfa351274b", - "sha256:14caacd1853e40103f59571f169704367e79fb78fac3d6d09ac84d9197cadd16", - "sha256:16d9eaa8c7e91537516c20da37db975f09ac2e7772a0694b245076c6d68f85da", - "sha256:18fdc51458abc0a974822333bd3a932d4e06ba2a3243e9a1da305668bd62ec6d", - "sha256:28e8af338240b6f39713a34e337c3813047896ace09d51593d6907c66c0708ba", - "sha256:3835de96524a7b6869a6c710b26c90e94558c31006e96ca3cf6af6751b27dca1", - "sha256:3905c86cc4ab6d71635d6419a6f8d972cab7c634539bba6053c47354fd04452c", - "sha256:3c09d88a07483231119f5017904db8f60ad67906efac3f1baa31b9b7f7cca281", - "sha256:4551728b767f35f86b8e5ec19a363df87450c7376d7419c3cac5b9ceb4bce576", - "sha256:459bbe342c5b2dec5c5223e7c363f291558bc27982ef39ffd6569e8c082bdc83", - "sha256:4f421e3cdd3a273bace013751c345f4ebeef08f05e8c10757533ada360b51a39", - "sha256:577737ec3d4c195c4aef01b757905779a9e9aee608fa1cf0aec16b5576c893d3", - "sha256:57fece29f7cc55d882fe282d9de52f2f522bb85290555b49394102f3621751ee", - "sha256:7976d410e42be9ae7458c1816a416218364e06e162b82e42f7060737e711d9ce", - "sha256:85f568892422a0e96235eb8ea6c5a41c8ccbf55576a2260c0160800dbd7c4f20", - "sha256:8764a78c5464ac6bde91a8c87dd718c27c1cabb7ed2b4beaf36d3e8e390567f9", - "sha256:8935937dad2c9b369c3d932b0edbc52a62647c2afb2fafc0c280f14a8bf56a6a", - "sha256:8fe58d9f6e3d1abf690174fd75800fda9bdc23d2a287e77758dc0e8567e38ce6", - "sha256:937b20955806381e08e54bd9d71f83276d1f883264808521b70b33d98e4dec5d", - "sha256:9569da9e78f0947b249370cb8fadf1015a193c359e7e442ac9ecc585d937f08d", - "sha256:a3b73390511edd2db2d34ff09aa0b2c08be974c71b4c0505b4a048d5dc128c2b", - "sha256:a4eddbe2a715b2dd3849afbdeacf1cc283160b24e09baf64fa5675f51940419d", - "sha256:a5c6dbe09aff091adfa8c7cfc1a0e83fdb8021ddb2c183512775a14f1435fe16", - "sha256:b63e3571b24a7959017573b6455e05b675050bbbea69408f35f3cb984ec54363", - "sha256:bb350eb1060591d8e89d6bac4713d41006cd4d479f5e11db334a48ff8999512f", - "sha256:bf6d987edd4a44dd2fa2723fca2790f9442ae4de2c8438e53fcb1befdf5d823a", - "sha256:bfa6a679410b394600eafd16336b2ce8de43e9b13f7fb9247d84ef5ad2b45e91", - "sha256:c856ec9b42e5af4fe2d8e75970fcc3a2c15925cbcc6e7a9bcb44583b10b95e80", - "sha256:cea56288eeda8b7511d507bbe7790d89ae7049daa5f51ae31a35ae3c05408531", - "sha256:ea212df6e5d3f60341aef46401d32fcfded85593af1d82b8b4a7a68cd67fdd6b", - "sha256:f35567470ee6dbfb946f069ed5f5615b40edcbb5f1e6e1d3d2b114468d505fc6", - "sha256:fbc20975eee093efa2071de80df7f972b7b35e560b213aafabcec7c0bd00bd8c", - "sha256:ff4a8ad9638b7ca52313d8732f37ecd5fd3c8e3aff10a8ccb93176fd5b3812f6" - ], - "version": "==2021.8.3" + "sha256:04f6b9749e335bb0d2f68c707f23bb1773c3fb6ecd10edf0f04df12a8920d468", + "sha256:08d74bfaa4c7731b8dac0a992c63673a2782758f7cfad34cf9c1b9184f911354", + "sha256:0fc1f8f06977c2d4f5e3d3f0d4a08089be783973fc6b6e278bde01f0544ff308", + "sha256:121f4b3185feaade3f85f70294aef3f777199e9b5c0c0245c774ae884b110a2d", + "sha256:1413b5022ed6ac0d504ba425ef02549a57d0f4276de58e3ab7e82437892704fc", + "sha256:1743345e30917e8c574f273f51679c294effba6ad372db1967852f12c76759d8", + "sha256:28fc475f560d8f67cc8767b94db4c9440210f6958495aeae70fac8faec631797", + "sha256:31a99a4796bf5aefc8351e98507b09e1b09115574f7c9dbb9cf2111f7220d2e2", + "sha256:328a1fad67445550b982caa2a2a850da5989fd6595e858f02d04636e7f8b0b13", + "sha256:473858730ef6d6ff7f7d5f19452184cd0caa062a20047f6d6f3e135a4648865d", + "sha256:4cde065ab33bcaab774d84096fae266d9301d1a2f5519d7bd58fc55274afbf7a", + "sha256:5f6a808044faae658f546dd5f525e921de9fa409de7a5570865467f03a626fc0", + "sha256:610b690b406653c84b7cb6091facb3033500ee81089867ee7d59e675f9ca2b73", + "sha256:66256b6391c057305e5ae9209941ef63c33a476b73772ca967d4a2df70520ec1", + "sha256:6eebf512aa90751d5ef6a7c2ac9d60113f32e86e5687326a50d7686e309f66ed", + "sha256:79aef6b5cd41feff359acaf98e040844613ff5298d0d19c455b3d9ae0bc8c35a", + "sha256:808ee5834e06f57978da3e003ad9d6292de69d2bf6263662a1a8ae30788e080b", + "sha256:8e44769068d33e0ea6ccdf4b84d80c5afffe5207aa4d1881a629cf0ef3ec398f", + "sha256:999ad08220467b6ad4bd3dd34e65329dd5d0df9b31e47106105e407954965256", + "sha256:9b006628fe43aa69259ec04ca258d88ed19b64791693df59c422b607b6ece8bb", + "sha256:9d05ad5367c90814099000442b2125535e9d77581855b9bee8780f1b41f2b1a2", + "sha256:a577a21de2ef8059b58f79ff76a4da81c45a75fe0bfb09bc8b7bb4293fa18983", + "sha256:a617593aeacc7a691cc4af4a4410031654f2909053bd8c8e7db837f179a630eb", + "sha256:abb48494d88e8a82601af905143e0de838c776c1241d92021e9256d5515b3645", + "sha256:ac88856a8cbccfc14f1b2d0b829af354cc1743cb375e7f04251ae73b2af6adf8", + "sha256:b4c220a1fe0d2c622493b0a1fd48f8f991998fb447d3cd368033a4b86cf1127a", + "sha256:b844fb09bd9936ed158ff9df0ab601e2045b316b17aa8b931857365ea8586906", + "sha256:bdc178caebd0f338d57ae445ef8e9b737ddf8fbc3ea187603f65aec5b041248f", + "sha256:c206587c83e795d417ed3adc8453a791f6d36b67c81416676cad053b4104152c", + "sha256:c61dcc1cf9fd165127a2853e2c31eb4fb961a4f26b394ac9fe5669c7a6592892", + "sha256:c7cb4c512d2d3b0870e00fbbac2f291d4b4bf2634d59a31176a87afe2777c6f0", + "sha256:d4a332404baa6665b54e5d283b4262f41f2103c255897084ec8f5487ce7b9e8e", + "sha256:d5111d4c843d80202e62b4fdbb4920db1dcee4f9366d6b03294f45ed7b18b42e", + "sha256:e1e8406b895aba6caa63d9fd1b6b1700d7e4825f78ccb1e5260551d168db38ed", + "sha256:e8690ed94481f219a7a967c118abaf71ccc440f69acd583cab721b90eeedb77c", + "sha256:ed283ab3a01d8b53de3a05bfdf4473ae24e43caee7dcb5584e86f3f3e5ab4374", + "sha256:ed4b50355b066796dacdd1cf538f2ce57275d001838f9b132fab80b75e8c84dd", + "sha256:ee329d0387b5b41a5dddbb6243a21cb7896587a651bebb957e2d2bb8b63c0791", + "sha256:f3bf1bc02bc421047bfec3343729c4bbbea42605bcfd6d6bfe2c07ade8b12d2a", + "sha256:f585cbbeecb35f35609edccb95efd95a3e35824cd7752b586503f7e6087303f1", + "sha256:f60667673ff9c249709160529ab39667d1ae9fd38634e006bec95611f632e759" + ], + "version": "==2021.8.28" }, "requests": { "hashes": [ @@ -1737,11 +1970,11 @@ }, "rich": { "hashes": [ - "sha256:13ac80676e12cf528dc4228dc682c8402f82577c2aa67191e294350fa2c3c4e9", - "sha256:517b4e0efd064dd1fe821ca93dd3095d73380ceac1f0a07173d507d9b18f1396" + "sha256:2c84d9b3459c16bf413fe0f9644c7ae1791971e0bb944dfae56e7c7634b187ab", + "sha256:ba285f1c519519490034284e6a9d2e6e3f16dc7690f2de3d9140737d81304d22" ], - "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==10.7.0" + "markers": "python_version >= '3.6' and python_version < '4.0'", + "version": "==10.9.0" }, "rsa": { "hashes": [ @@ -1958,11 +2191,11 @@ }, "typing-extensions": { "hashes": [ - "sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497", - "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342", - "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84" + "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918", + "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c", + "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f" ], - "version": "==3.10.0.0" + "version": "==3.7.4.3" }, "typing-inspect": { "hashes": [ @@ -1991,7 +2224,7 @@ "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4", "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.6" }, "virtualenv": { diff --git a/requirements-dev.txt b/requirements-dev.txt index 00d8cd15d..b75eae42d 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,24 +6,27 @@ # -i https://pypi.org/simple -alembic==1.6.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +alembic==1.7.1; python_version >= '3.6' anyio==3.3.0; python_full_version >= '3.6.2' -apache-airflow-providers-cncf-kubernetes==2.0.1 +apache-airflow-providers-apache-beam==3.0.0 +apache-airflow-providers-cncf-kubernetes==2.0.2 apache-airflow-providers-ftp==2.0.0; python_version ~= '3.6' apache-airflow-providers-google==5.0.0 apache-airflow-providers-imap==2.0.0; python_version ~= '3.6' apache-airflow-providers-sqlite==2.0.0; python_version ~= '3.6' apache-airflow==2.1.1 -apispec[yaml]==3.3.2; python_full_version >= '3.5.0' +apache-beam==2.32.0 +apispec[yaml]==3.3.2; python_version >= '3.5' appdirs==1.4.4 argcomplete==1.12.3 attrs==20.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +avro-python3==1.9.2.1; python_version >= '3.5' babel==2.9.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' backports.entry-points-selectable==1.1.0; python_version >= '2.7' black==20.8b1 blinker==1.4 cachetools==4.2.2; python_version ~= '3.5' -cattrs==1.5.0; python_version > '3.6' +cattrs==1.5.0; python_version >= '3.7' certifi==2021.5.30 cffi==1.14.6 charset-normalizer==2.0.4; python_version >= '3' @@ -32,77 +35,82 @@ clickclick==20.10.2 colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' colorlog==4.8.0 commonmark==0.9.1 +crcmod==1.7 croniter==1.0.15; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -cryptography==3.4.7; python_version >= '3.6' +cryptography==3.4.8; python_version >= '3.6' defusedxml==0.7.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -dill==0.3.4; python_version >= '2.7' and python_version != '3.0' +dill==0.3.1.1; python_version >= '2.6' and python_version != '3.0' distlib==0.3.2 dnspython==2.1.0; python_version >= '3.6' +docopt==0.6.2 docutils==0.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' email-validator==1.1.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +fastavro==1.4.4; python_version >= '3.6' filelock==3.0.12 flake8==3.9.2 flask-appbuilder==3.3.2; python_version ~= '3.6' flask-babel==1.0.0 -flask-caching==1.10.1; python_full_version >= '3.5.0' +flask-caching==1.10.1; python_version >= '3.5' flask-jwt-extended==3.25.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0' flask-login==0.4.1 flask-openid==1.2.5 flask-sqlalchemy==2.5.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' flask-wtf==0.14.3 flask==1.1.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -google-ads==13.0.0; python_version >= '3.7' -google-api-core[grpc,grpcgcp]==1.31.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +future==0.18.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +google-ads==14.0.0; python_version >= '3.7' +google-api-core[grpc,grpcgcp]==1.31.2; python_version >= '3.6' google-api-python-client==1.12.8; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-auth-httplib2==0.1.0 -google-auth-oauthlib==0.4.5; python_version >= '3.6' -google-auth==1.34.0; python_version >= '3.6' +google-auth-oauthlib==0.4.6; python_version >= '3.6' +google-auth==1.35.0; python_version >= '3.6' google-cloud-appengine-logging==0.1.4; python_version >= '3.6' -google-cloud-audit-log==0.1.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +google-cloud-audit-log==0.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-automl==2.4.2; python_version >= '3.6' -google-cloud-bigquery-datatransfer==3.3.1; python_version >= '3.6' -google-cloud-bigquery-storage==2.6.2 -google-cloud-bigquery[bqstorage,pandas]==2.23.2; python_version < '3.10' and python_version >= '3.6' +google-cloud-bigquery-datatransfer==3.3.2; python_version >= '3.6' +google-cloud-bigquery-storage==2.6.3 +google-cloud-bigquery[bqstorage,pandas]==2.26.0; python_version < '3.10' and python_version >= '3.6' google-cloud-bigtable==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-container==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-core==1.7.2; python_version >= '3.6' -google-cloud-datacatalog==3.4.0 +google-cloud-datacatalog==3.4.1 google-cloud-dataproc==2.5.0; python_version >= '3.6' google-cloud-dlp==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -google-cloud-kms==2.4.3; python_version >= '3.6' +google-cloud-kms==2.6.0; python_version >= '3.6' google-cloud-language==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-logging==2.6.0; python_version >= '3.6' google-cloud-memcache==1.0.0; python_version >= '3.6' -google-cloud-monitoring==2.4.2; python_version >= '3.6' +google-cloud-monitoring==2.5.0; python_version >= '3.6' google-cloud-os-login==2.3.1; python_version >= '3.6' -google-cloud-pubsub==2.7.0; python_version >= '3.6' +google-cloud-pubsub==2.7.1; python_version >= '3.6' google-cloud-redis==2.2.2; python_version >= '3.6' google-cloud-secret-manager==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-spanner==1.19.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-speech==1.3.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -google-cloud-storage==1.41.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +google-cloud-storage==1.42.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' google-cloud-tasks==2.5.1; python_version >= '3.6' google-cloud-texttospeech==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-translate==1.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-videointelligence==1.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-vision==1.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' google-cloud-workflows==1.2.1; python_version >= '3.6' -google-crc32c==1.1.2; python_version >= '3.5' -google-resumable-media==1.3.3; python_version >= '3.6' +google-crc32c==1.1.3; python_version >= '3.6' +google-resumable-media==2.0.1; python_version >= '3.6' googleapis-common-protos[grpc]==1.53.0; python_version >= '3.6' graphviz==0.17; python_version >= '3.6' grpc-google-iam-v1==0.12.3 grpcio-gcp==0.2.2 grpcio==1.39.0 -gunicorn==20.1.0; python_full_version >= '3.5.0' +gunicorn==20.1.0; python_version >= '3.5' h11==0.12.0; python_version >= '3.6' +hdfs==2.6.0 httpcore==0.13.6; python_version >= '3.6' httplib2==0.19.1 -httpx==0.18.2; python_version >= '3.6' +httpx==0.19.0; python_version >= '3.6' idna==3.2 importlib-metadata==1.7.0; python_version < '3.9' -importlib-resources==1.5.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -inflection==0.5.1; python_full_version >= '3.5.0' +importlib-resources==1.5.0; python_version < '3.9' +inflection==0.5.1; python_version >= '3.5' iniconfig==1.1.1 iso8601==0.1.16 isodate==0.6.0 @@ -113,69 +121,72 @@ json-merge-patch==0.2 jsonschema==3.2.0 kubernetes==11.0.0 lazy-object-proxy==1.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' -libcst==0.3.19; python_version >= '3.6' +libcst==0.3.20; python_version >= '3.6' lockfile==0.12.2 -mako==1.1.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +mako==1.1.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' markdown==3.3.4; python_version >= '3.6' markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' marshmallow-enum==1.5.1 marshmallow-oneofschema==3.0.1; python_version >= '3.6' marshmallow-sqlalchemy==0.23.1; python_version >= '3.6' -marshmallow==3.13.0; python_full_version >= '3.5.0' +marshmallow==3.13.0; python_version >= '3.5' mccabe==0.6.1 mypy-extensions==0.4.3 nox==2020.12.31; python_version >= '3.6' -numpy==1.21.1; python_version >= '3.7' +numpy==1.20.3; python_version >= '3.7' +oauth2client==4.1.3 oauthlib==3.1.1; python_version >= '3.6' openapi-schema-validator==0.1.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' openapi-spec-validator==0.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +orjson==3.6.3; python_version >= '3.7' packaging==21.0; python_version >= '3.6' pandas-gbq==0.14.1 -pandas==1.3.1 +pandas==1.3.2 pathspec==0.9.0 pendulum==2.1.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -platformdirs==2.2.0; python_version >= '3.6' -pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -prison==0.1.3 -proto-plus==1.19.0; python_version >= '3.6' +platformdirs==2.3.0; python_version >= '3.6' +pluggy==1.0.0; python_version >= '3.6' +prison==0.2.1 +proto-plus==1.18.1; python_version >= '3.6' protobuf==3.17.3 psutil==5.8.0; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' py==1.10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -pyarrow==5.0.0 +pyarrow==4.0.1 pyasn1-modules==0.2.8 pyasn1==0.4.8 pycodestyle==2.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pycparser==2.20; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pydata-google-auth==1.2.0 +pydot==1.4.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pyflakes==2.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -pygments==2.9.0; python_full_version >= '3.5.0' +pygments==2.10.0; python_version >= '3.5' pyjwt==1.7.1 +pymongo==3.12.0 pyopenssl==20.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' +pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' pyrsistent==0.18.0; python_version >= '3.6' pytest-mock==3.6.1 -pytest==6.2.4 +pytest==6.2.5 python-daemon==2.3.0 -python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' -python-editor==1.0.4 +python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-nvd3==0.15.0 python-slugify==4.0.1 python3-openid==3.2.0 pytz==2021.1 pytzdata==2020.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pyyaml==5.4.1 -regex==2021.8.3 +regex==2021.8.28 requests-oauthlib==1.3.0 requests==2.26.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' rfc3986[idna2008]==1.5.0 -rich==10.7.0; python_version >= '3.6' and python_version < '4.0' +rich==10.9.0; python_version >= '3.6' and python_version < '4.0' rsa==4.7.2; python_version >= '3.6' -ruamel.yaml.clib==0.2.6; platform_python_implementation == 'CPython' and python_version < '3.10' -ruamel.yaml==0.17.10 +ruamel.yaml.clib==0.2.6; python_version < '3.10' and platform_python_implementation == 'CPython' +ruamel.yaml==0.17.11 setproctitle==1.2.2; python_version >= '3.6' -six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' -sniffio==1.2.0; python_full_version >= '3.5.0' -sqlalchemy-jsonfield==1.0.0; python_full_version >= '3.5.0' +six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +sniffio==1.2.0; python_version >= '3.5' +sqlalchemy-jsonfield==1.0.0; python_version >= '3.5' sqlalchemy-utils==0.37.8; python_version ~= '3.4' sqlalchemy==1.3.18 swagger-ui-bundle==0.0.8 @@ -183,15 +194,15 @@ tabulate==0.8.9 tenacity==6.2.0 termcolor==1.1.0 text-unidecode==1.3 -toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' +toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' typed-ast==1.4.3 -typing-extensions==3.10.0.0 +typing-extensions==3.7.4.3 typing-inspect==0.7.1 unicodecsv==0.14.1 uritemplate==3.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' urllib3==1.26.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0' -virtualenv==20.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -websocket-client==1.1.1; python_version >= '3.6' +virtualenv==20.7.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +websocket-client==1.2.1; python_version >= '3.6' werkzeug==1.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' wtforms==2.3.3 zipp==3.5.0; python_version >= '3.6' diff --git a/samples/pipeline.yaml b/samples/pipeline.yaml index cac42c1b6..16c524f44 100644 --- a/samples/pipeline.yaml +++ b/samples/pipeline.yaml @@ -405,6 +405,140 @@ dag: # Optional service account to impersonate using short-term credentials impersonation_chain: "{{ var.json.DATASET_FOLDER_NAME.PIPELINE_FOLDER_NAME.service_account }}" + + # Launch Cloud Dataflow jobs written in Python + # https://airflow.apache.org/docs/apache-airflow-providers-google/stable/_api/airflow/providers/google/cloud/operators/dataflow/index.html#airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator + # + # You must organize your files as follows: + # + # datasets/DATASET_NAME/PIPELINE_NAME/custom/ + # DATAFLOW_JOB_NAME/ + # main.py (contains the Beam Pipeline definition) + # setup.py (this file is required) + # my_package/ (optional folder containing custom dependencies) + # __init__.py (this file is required inside every package subfolder) + # my_file.py + # ... + # ... + - operator: "DataflowCreatePythonJobOperator" + description: "Task to run a Dataflow job" + + args: + task_id: "sample_dataflow_task" + + # Path to the Python script containing the Apache Beam Pipeline definition. + # Place the script in your pipeline's `custom` folder, preferrably namespaced + # under a folder named after the job name (templated as `DATAFLOW_JOB_NAME`). + py_file: "{{ var.json.shared.airflow_dags_folder }}/DATASET_NAME/PIPELINE_NAME/custom/DATAFLOW_JOB_NAME/main.py" + + # Python version of the Beam pipeline. If None, this defaults to python3. + py_interpreter: "python3" + + # List of Python package(s) required by your Dataflow job. + # The three packages listed below are often needed. + py_requirements: + - "apache-beam" + - "google-apitools" + - "google-cloud-storage" + + dataflow_default_options: + + # Your Google Cloud project ID + project: "{{ var.json.shared.gcp_project }}" + + # [Required] Path for temporary files. + # You can use the Composer bucket (preferred) or your own bucket. + # When using the Cloud Composer bucket, use the templated path below to prevent collisions. + temp_location: "{{ var.json.shared.composer_bucket }}/data/DATASET_NAME/PIPELINE_NAME/dataflow/tmp/DATAFLOW_JOB_NAME/" + + # The GCS bucket path to store staging binary files used by Apache Beam + # You can use the Composer bucket (preferred) or your own bucket. + # When using the Cloud Composer bucket, use the templated path below to prevent collisions. + staging_location: "{{ var.json.shared.composer_bucket }}/data/DATASET_NAME/PIPELINE_NAME/dataflow/staging/" + + # [Optional] If your pipeline uses public packages from the PyPI, make + # these packages available via a requirements.txt file. Create the file + # in the `custom` folder, in the same subfolder as your `main.py`. + requirements_file: "{{ var.json.shared.airflow_dags_folder }}/DATASET_NAME/PIPELINE_NAME/custom/DATAFLOW_JOB_NAME/requirements.txt" + + # [Required] The pipeline runner to use. Must be set to "DataflowRunner" to run on + # Google Cloud Dataflow. Keep this unchanged. + runner: "DataflowRunner" + + # [Optional] Use the parameter below if you have custom Python dependencies for the job. + # For instance, if you have a main.py file that contains your Apache Beam Pipeline definition + # along with references to other packages/files you've written (i.e. non-PyPI): + # + # import apache_beam as beam + # from my_package import my_file + # + # # Define data transforms + # my_class = my_file.MyClass(...) + # my_file.my_function(...) + # ... + # + # # Define pipeline + # with beam.Pipeline(options=options) as pipeline: + # ( + # pipeline + # | "stage_1" >> beam.Create(collection) + # | "stage_2" >> beam.Map(...) + # | "stage_3" >> beam.MapTuple(...) + # | "stage_3" >> beam.FlatMap(...) + # ... + # ) + # ... + # + # you must organize your files as follows: + # + # datasets/DATASET_NAME/PIPELINE_NAME/custom/ + # DATAFLOW_JOB_NAME/ + # main.py + # setup.py (this file is required when using custom packages) + # requirements.txt + # my_package/ (folder containing a custom package) + # __init__.py (this file is required inside every package subfolder) + # my_file.py + # ... + # ... + # + # where every subfolder is a package. You must also include a setup.py in the same directory as main.py. + # Your setup.py must contain + # + # import setuptools + # + # setuptools.setup( + # name='DATAFLOW_JOB_NAME', + # version='1.0.0', + # install_requires=[], + # packages=setuptools.find_packages(), + # ) + # + # (For more info, see https://beam.apache.org/documentation/sdks/python-pipeline-dependencies/#multiple-file-dependencies) + # + # Now, use this parameter to declare the path to the setup.py file under /home/airflow/gcs/dags + setup_file: "/home/airflow/gcs/dags/DATASET_NAME/PIPELINE_NAME/custom/DATAFLOW_JOB_NAME/setup.py" + + # [Optional] The Google Compute Engine machine type, must be n1-standard-2 or higher. + # GCE machine types: https://cloud.google.com/compute/docs/machine-types. + # The Dataflow service chooses the machine type based on your job if you do not set this option. + machine_type: "n1-standard-2" + + # [Optional] The disk size, in gigabytes, to use on each remote Compute Engine worker instance. + # Set to 0 to use the default size defined in your Cloud Platform project. + disk_size_gb: 64 + + # [Optional] The maximum number of workers to be made available to your + # pipeline during execution. If unspecified, Dataflow determines an + # appropriate number of workers, usually set to 12 or 18. + max_num_workers: 18 + + # [Optional] Number of workers to use when executing the Dataflow job. + # If not set, the Dataflow service will use a reasonable default. + # Prefer using `max_num_workers` instead of this one to control resource + # limits and let autoscaling adjust the workers up to that number. + num_workers: 1 + graph_paths: # This is where you specify the relationships (i.e. directed paths/edges) # among the tasks specified above. Use the bitshift operator to define the @@ -413,5 +547,5 @@ dag: # For more info, see # https://airflow.apache.org/docs/apache-airflow/stable/tutorial.html#setting-up-dependencies - "sample_bash_task >> [sample_gcs_to_bq_task, sample_gcs_to_gcs_task]" - - "sample_gcs_to_bq_task >> [sample_bq_sql_task, gcs_delete_task]" + - "sample_gcs_to_bq_task >> sample_dataflow_task >> [sample_bq_sql_task, gcs_delete_task]" - "gke_create_cluster_task >> gke_start_pod_task >> gke_delete_cluster_task" diff --git a/scripts/dag_imports.json b/scripts/dag_imports.json index c0f88ee0d..27965bbab 100644 --- a/scripts/dag_imports.json +++ b/scripts/dag_imports.json @@ -69,6 +69,10 @@ "GKEDeleteClusterOperator": { "import": "from airflow.providers.google.cloud.operators import kubernetes_engine", "class": "kubernetes_engine.GKEDeleteClusterOperator" + }, + "DataflowCreatePythonJobOperator": { + "import": "from airflow.providers.google.cloud.operators import dataflow", + "class": "dataflow.DataflowCreatePythonJobOperator" } } }