diff --git a/Pipfile b/Pipfile index 58731ba8e..6d0394e2e 100644 --- a/Pipfile +++ b/Pipfile @@ -16,6 +16,7 @@ rfc3987 = "*" pystrict = "*" openpyxl = "*" pyparsing = "==2.4.7" +networkx = "*" [dev-packages] mkdocs = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 5bdfda07f..1f941413f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e607e33572576010c82d744bb5e3e602031f77bd59c6cd7d4b4272b5a902c149" + "sha256": "cc82ff2fd365554f59380994a1a47b2c0468899dc8dcdd99e7a9cc6c9dd82338" }, "pipfile-spec": 6, "requires": { @@ -49,11 +49,11 @@ }, "click": { "hashes": [ - "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e", - "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72" + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" ], "index": "pypi", - "version": "==8.1.2" + "version": "==8.1.3" }, "decorator": { "hashes": [ @@ -97,11 +97,11 @@ }, "jsonschema": { "hashes": [ - "sha256:636694eb41b3535ed608fe04129f26542b59ed99808b4f688aa32dcf55317a83", - "sha256:77281a1f71684953ee8b3d488371b162419767973789272434bbc3f29d9c8823" + "sha256:71b5e39324422543546572954ce71c67728922c104902cb7ce252e522235b33f", + "sha256:7c6d882619340c3347a1bf7315e147e6d3dae439033ae6383d6acb908c101dfc" ], "index": "pypi", - "version": "==4.4.0" + "version": "==4.5.1" }, "lxml": { "hashes": [ @@ -170,6 +170,14 @@ "index": "pypi", "version": "==4.8.0" }, + "networkx": { + "hashes": [ + "sha256:1a1e8fe052cc1b4e0339b998f6795099562a264a13a5af7a32cad45ab9d4e126", + "sha256:4a52cf66aed221955420e11b3e2e05ca44196b4829aab9576d4d439212b0a14f" + ], + "index": "pypi", + "version": "==2.8" + }, "openpyxl": { "hashes": [ "sha256:40f568b9829bf9e446acfffce30250ac1fa39035124d55fc024025c41481c90f", @@ -254,18 +262,18 @@ }, "setuptools": { "hashes": [ - "sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8", - "sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592" + "sha256:5534570b9980fc650d45c62877ff603c7aaaf24893371708736cc016bd221c3c", + "sha256:ca6ba73b7fd5f734ae70ece8c4c1f7062b07f3352f6428f6277e27c8f5c64237" ], "markers": "python_version >= '3.7'", - "version": "==62.1.0" + "version": "==62.2.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "urllib3": { @@ -278,11 +286,10 @@ }, "validators": { "hashes": [ - "sha256:0143dcca8a386498edaf5780cbd5960da1a4c85e0719f3ee5c9b41249c4fefbd", - "sha256:37cd9a9213278538ad09b5b9f9134266e7c226ab1fede1d500e29e0a8fbb9ea6" + "sha256:dec45f4381f042f1e705cfa74949505b77f1e27e8b05409096fee8152c839cbe" ], "index": "pypi", - "version": "==0.18.2" + "version": "==0.19.0" } }, "develop": { @@ -340,11 +347,11 @@ }, "click": { "hashes": [ - "sha256:24e1a4a9ec5bf6299411369b208c1df2188d9eb8d916302fe6bf03faed227f1e", - "sha256:479707fe14d9ec9a0757618b7a100a0ae4c4e236fac5b7f80ca68028141a1a72" + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" ], "index": "pypi", - "version": "==8.1.2" + "version": "==8.1.3" }, "colorama": { "hashes": [ @@ -363,10 +370,10 @@ }, "ghp-import": { "hashes": [ - "sha256:5f8962b30b20652cdffa9c5a9812f7de6bcb56ec475acac579807719bf242c46", - "sha256:947b3771f11be850c852c64b561c600fdddf794bab363060854c1ee7ad05e071" + "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", + "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343" ], - "version": "==2.0.2" + "version": "==2.1.0" }, "idna": { "hashes": [ @@ -393,19 +400,19 @@ }, "jinja2": { "hashes": [ - "sha256:539835f51a74a69f41b848a9645dbdc35b4f20a3b601e2d9a7e22947b15ff119", - "sha256:640bed4bb501cbd17194b3cace1dc2126f5b619cf068a726b98192a0fde74ae9" + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], "markers": "python_version >= '3.7'", - "version": "==3.1.1" + "version": "==3.1.2" }, "markdown": { "hashes": [ - "sha256:76df8ae32294ec39dcf89340382882dfa12975f87f45c3ed1ecdb1e8cefc7006", - "sha256:9923332318f843411e9932237530df53162e29dc7a4e2b91e35764583c46c9a3" + "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874", + "sha256:f5da449a6e1c989a4cea2631aa8ee67caa5a2ef855d551c88f9e309f4634c621" ], "markers": "python_version >= '3.6'", - "version": "==3.3.6" + "version": "==3.3.7" }, "markupsafe": { "hashes": [ @@ -471,19 +478,19 @@ }, "mkdocs-include-markdown-plugin": { "hashes": [ - "sha256:1462d343f150b59e78f0aebf9ff9ef9d591fbd9630193e0c9d661abf6df92d6a", - "sha256:d820523a28c893b92f4c5c383332d36379874a3741f4fe04f11bf41afca18a2b" + "sha256:238feaa482af13ba3fc77ce215ec92bbd506fd2d1fcebe3ff35f4427865dc64a", + "sha256:b538e89ded65c3cf18ea5b50aa0e2369cc80128a7e5bff77ba8ab155054f10ea" ], "index": "pypi", - "version": "==3.3.0" + "version": "==3.4.0" }, "mkdocs-material": { "hashes": [ - "sha256:4a3631ba22cff7ceca00c39465a8db5b2116fcd74f3abd82b801f7711cceb699", - "sha256:af6bbd608a54b8493cb5fa0d2f2cf29cde3bf348837ab718afb1be0a8bea6509" + "sha256:93b57e53733051431cc83216446e774bdf08bf516a6251ff2f24974f45f98149", + "sha256:9d6c4ca1ceecc00b2e38c214665ed7605d275321dcaa22f38b9d1175edc58955" ], "index": "pypi", - "version": "==8.2.11" + "version": "==8.2.15" }, "mkdocs-material-extensions": { "hashes": [ @@ -495,32 +502,32 @@ }, "mypy": { "hashes": [ - "sha256:0e2dd88410937423fba18e57147dd07cd8381291b93d5b1984626f173a26543e", - "sha256:10daab80bc40f84e3f087d896cdb53dc811a9f04eae4b3f95779c26edee89d16", - "sha256:17e44649fec92e9f82102b48a3bf7b4a5510ad0cd22fa21a104826b5db4903e2", - "sha256:1a0459c333f00e6a11cbf6b468b870c2b99a906cb72d6eadf3d1d95d38c9352c", - "sha256:246e1aa127d5b78488a4a0594bd95f6d6fb9d63cf08a66dafbff8595d8891f67", - "sha256:2b184db8c618c43c3a31b32ff00cd28195d39e9c24e7c3b401f3db7f6e5767f5", - "sha256:2bc249409a7168d37c658e062e1ab5173300984a2dada2589638568ddc1db02b", - "sha256:3841b5433ff936bff2f4dc8d54cf2cdbfea5d8e88cedfac45c161368e5770ba6", - "sha256:4c3e497588afccfa4334a9986b56f703e75793133c4be3a02d06a3df16b67a58", - "sha256:5bf44840fb43ac4074636fd47ee476d73f0039f4f54e86d7265077dc199be24d", - "sha256:64235137edc16bee6f095aba73be5334677d6f6bdb7fa03cfab90164fa294a17", - "sha256:6776e5fa22381cc761df53e7496a805801c1a751b27b99a9ff2f0ca848c7eca0", - "sha256:6ce34a118d1a898f47def970a2042b8af6bdcc01546454726c7dd2171aa6dfca", - "sha256:6f6ad963172152e112b87cc7ec103ba0f2db2f1cd8997237827c052a3903eaa6", - "sha256:6f7106cbf9cc2f403693bf50ed7c9fa5bb3dfa9007b240db3c910929abe2a322", - "sha256:7742d2c4e46bb5017b51c810283a6a389296cda03df805a4f7869a6f41246534", - "sha256:9521c1265ccaaa1791d2c13582f06facf815f426cd8b07c3a485f486a8ffc1f3", - "sha256:a1b383fe99678d7402754fe90448d4037f9512ce70c21f8aee3b8bf48ffc51db", - "sha256:b840cfe89c4ab6386c40300689cd8645fc8d2d5f20101c7f8bd23d15fca14904", - "sha256:d8d3ba77e56b84cd47a8ee45b62c84b6d80d32383928fe2548c9a124ea0a725c", - "sha256:dcd955f36e0180258a96f880348fbca54ce092b40fbb4b37372ae3b25a0b0a46", - "sha256:e865fec858d75b78b4d63266c9aff770ecb6a39dfb6d6b56c47f7f8aba6baba8", - "sha256:edf7237137a1a9330046dbb14796963d734dd740a98d5e144a3eb1d267f5f9ee" + "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d", + "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8", + "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de", + "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038", + "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed", + "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334", + "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff", + "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2", + "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22", + "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2", + "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2", + "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605", + "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb", + "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519", + "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0", + "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc", + "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b", + "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f", + "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075", + "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef", + "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb", + "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a", + "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b" ], "index": "pypi", - "version": "==0.942" + "version": "==0.950" }, "mypy-extensions": { "hashes": [ @@ -552,7 +559,7 @@ "sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802", "sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa" ], - "markers": "platform_machine != 'aarch64' and platform_machine != 'arm64' and python_version < '3.10'", + "markers": "python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64'", "version": "==1.22.3" }, "orderedmultidict": { @@ -606,11 +613,11 @@ }, "pip": { "hashes": [ - "sha256:b3a9de2c6ef801e9247d1527a4b16f92f2cc141cd1489f3fffaf6a9e96729764", - "sha256:c6aca0f2f081363f689f041d90dab2a07a9a07fb840284db2218117a52da800b" + "sha256:2debf847016cfe643fa1512e2d781d3ca9e5c878ba0652583842d50cc2bcc605", + "sha256:802e797fb741be1c2d475533d4ea951957e4940091422bd4a24848a7ac95609d" ], "markers": "python_version >= '3.7'", - "version": "==22.0.4" + "version": "==22.1" }, "pip-shims": { "hashes": [ @@ -679,19 +686,19 @@ }, "pygments": { "hashes": [ - "sha256:44238f1b60a76d78fc8ca0528ee429702aae011c265fe6a8dd8b63049ae41c65", - "sha256:4e426f72023d88d03b2fa258de560726ce890ff3b630f88c21cbb8b2503b8c6a" + "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb", + "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519" ], - "markers": "python_version >= '3.5'", - "version": "==2.11.2" + "markers": "python_version >= '3.6'", + "version": "==2.12.0" }, "pymdown-extensions": { "hashes": [ - "sha256:a80553b243d3ed2d6c27723bcd64ca9887e560e6f4808baa96f36e93061eaf90", - "sha256:b37461a181c1c8103cfe1660081726a0361a8294cbfda88e5b02cefe976f0546" + "sha256:1baa22a60550f731630474cad28feb0405c8101f1a7ddc3ec0ed86ee510bcc43", + "sha256:5b7432456bf555ce2b0ab3c2439401084cda8110f24f6b3ecef952b8313dfa1b" ], "markers": "python_version >= '3.7'", - "version": "==9.3" + "version": "==9.4" }, "pyparsing": { "hashes": [ @@ -714,7 +721,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "pytz": { @@ -789,18 +796,18 @@ }, "setuptools": { "hashes": [ - "sha256:26ead7d1f93efc0f8c804d9fafafbe4a44b179580a7105754b245155f9af05a8", - "sha256:47c7b0c0f8fc10eec4cf1e71c6fdadf8decaa74ffa087e68cd1c20db7ad6a592" + "sha256:5534570b9980fc650d45c62877ff603c7aaaf24893371708736cc016bd221c3c", + "sha256:ca6ba73b7fd5f734ae70ece8c4c1f7062b07f3352f6428f6277e27c8f5c64237" ], "markers": "python_version >= '3.7'", - "version": "==62.1.0" + "version": "==62.2.0" }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "toml": { @@ -808,7 +815,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tomli": { @@ -824,7 +831,7 @@ "sha256:30d54c0b914e595f3d10a87888599eab5321a2a69abc773bbefff51599b72db6", "sha256:905cf92c2111ef80d355708f47ac24ad1b6fc2adc5107455940088c9bbecaedb" ], - "markers": "python_version >= '3.6' and python_version < '4'", + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==0.10.2" }, "typing-extensions": { @@ -853,33 +860,34 @@ }, "watchdog": { "hashes": [ - "sha256:03b43d583df0f18782a0431b6e9e9965c5b3f7cf8ec36a00b930def67942c385", - "sha256:0908bb50f6f7de54d5d31ec3da1654cb7287c6b87bce371954561e6de379d690", - "sha256:0b4a1fe6201c6e5a1926f5767b8664b45f0fcb429b62564a41f490ff1ce1dc7a", - "sha256:177bae28ca723bc00846466016d34f8c1d6a621383b6caca86745918d55c7383", - "sha256:19b36d436578eb437e029c6b838e732ed08054956366f6dd11875434a62d2b99", - "sha256:1d1cf7dfd747dec519486a98ef16097e6c480934ef115b16f18adb341df747a4", - "sha256:1e877c70245424b06c41ac258023ea4bd0c8e4ff15d7c1368f17cd0ae6e351dd", - "sha256:340b875aecf4b0e6672076a6f05cfce6686935559bb6d34cebedee04126a9566", - "sha256:351e09b6d9374d5bcb947e6ac47a608ec25b9d70583e9db00b2fcdb97b00b572", - "sha256:3fd47815353be9c44eebc94cc28fe26b2b0c5bd889dafc4a5a7cbdf924143480", - "sha256:49639865e3db4be032a96695c98ac09eed39bbb43fe876bb217da8f8101689a6", - "sha256:4d0e98ac2e8dd803a56f4e10438b33a2d40390a72750cff4939b4b274e7906fa", - "sha256:6e6ae29b72977f2e1ee3d0b760d7ee47896cb53e831cbeede3e64485e5633cc8", - "sha256:7f14ce6adea2af1bba495acdde0e510aecaeb13b33f7bd2f6324e551b26688ca", - "sha256:81982c7884aac75017a6ecc72f1a4fedbae04181a8665a34afce9539fc1b3fab", - "sha256:81a5861d0158a7e55fe149335fb2bbfa6f48cbcbd149b52dbe2cd9a544034bbd", - "sha256:ae934e34c11aa8296c18f70bf66ed60e9870fcdb4cc19129a04ca83ab23e7055", - "sha256:b26e13e8008dcaea6a909e91d39b629a39635d1a8a7239dd35327c74f4388601", - "sha256:b3750ee5399e6e9c69eae8b125092b871ee9e2fcbd657a92747aea28f9056a5c", - "sha256:b61acffaf5cd5d664af555c0850f9747cc5f2baf71e54bbac164c58398d6ca7b", - "sha256:b9777664848160449e5b4260e0b7bc1ae0f6f4992a8b285db4ec1ef119ffa0e2", - "sha256:bdcbf75580bf4b960fb659bbccd00123d83119619195f42d721e002c1621602f", - "sha256:d802d65262a560278cf1a65ef7cae4e2bc7ecfe19e5451349e4c67e23c9dc420", - "sha256:ed6d9aad09a2a948572224663ab00f8975fae242aa540509737bb4507133fa2d" + "sha256:036ed15f7cd656351bf4e17244447be0a09a61aaa92014332d50719fc5973bc0", + "sha256:0c520009b8cce79099237d810aaa19bc920941c268578436b62013b2f0102320", + "sha256:0fb60c7d31474b21acba54079ce9ff0136411183e9a591369417cddb1d7d00d7", + "sha256:156ec3a94695ea68cfb83454b98754af6e276031ba1ae7ae724dc6bf8973b92a", + "sha256:1ae17b6be788fb8e4d8753d8d599de948f0275a232416e16436363c682c6f850", + "sha256:1e5d0fdfaa265c29dc12621913a76ae99656cf7587d03950dfeb3595e5a26102", + "sha256:24dedcc3ce75e150f2a1d704661f6879764461a481ba15a57dc80543de46021c", + "sha256:2962628a8777650703e8f6f2593065884c602df7bae95759b2df267bd89b2ef5", + "sha256:47598fe6713fc1fee86b1ca85c9cbe77e9b72d002d6adeab9c3b608f8a5ead10", + "sha256:4978db33fc0934c92013ee163a9db158ec216099b69fce5aec790aba704da412", + "sha256:5e2e51c53666850c3ecffe9d265fc5d7351db644de17b15e9c685dd3cdcd6f97", + "sha256:676263bee67b165f16b05abc52acc7a94feac5b5ab2449b491f1a97638a79277", + "sha256:68dbe75e0fa1ba4d73ab3f8e67b21770fbed0651d32ce515cd38919a26873266", + "sha256:6d03149126864abd32715d4e9267d2754cede25a69052901399356ad3bc5ecff", + "sha256:6ddf67bc9f413791072e3afb466e46cc72c6799ba73dea18439b412e8f2e3257", + "sha256:746e4c197ec1083581bb1f64d07d1136accf03437badb5ff8fcb862565c193b2", + "sha256:7721ac736170b191c50806f43357407138c6748e4eb3e69b071397f7f7aaeedd", + "sha256:88ef3e8640ef0a64b7ad7394b0f23384f58ac19dd759da7eaa9bc04b2898943f", + "sha256:aa68d2d9a89d686fae99d28a6edf3b18595e78f5adf4f5c18fbfda549ac0f20c", + "sha256:b962de4d7d92ff78fb2dbc6a0cb292a679dea879a0eb5568911484d56545b153", + "sha256:ce7376aed3da5fd777483fe5ebc8475a440c6d18f23998024f832134b2938e7b", + "sha256:ddde157dc1447d8130cb5b8df102fad845916fe4335e3d3c3f44c16565becbb7", + "sha256:efcc8cbc1b43902571b3dce7ef53003f5b97fe4f275fe0489565fc6e2ebe3314", + "sha256:f9ee4c6bf3a1b2ed6be90a2d78f3f4bbd8105b6390c04a86eb48ed67bbfa0b0b", + "sha256:fed4de6e45a4f16e4046ea00917b4fe1700b97244e5d114f594b4a1b9de6bed8" ], "markers": "python_version >= '3.6'", - "version": "==2.1.7" + "version": "==2.1.8" }, "wheel": { "hashes": [ diff --git a/dev-requirements.txt b/dev-requirements.txt index 48dc02907..34970ec54 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -13,30 +13,30 @@ cerberus==1.3.4 certifi==2021.10.8 chardet==4.0.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' charset-normalizer==2.0.12; python_version >= '3' -click==8.1.2 +click==8.1.3 colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' distlib==0.3.4 -ghp-import==2.0.2 +ghp-import==2.1.0 idna==3.3; python_version >= '3' importlib-metadata==4.11.3; python_version >= '3.7' iniconfig==1.1.1 -jinja2==3.1.1; python_version >= '3.7' -markdown==3.3.6; python_version >= '3.6' +jinja2==3.1.2; python_version >= '3.7' +markdown==3.3.7; python_version >= '3.6' markupsafe==2.1.1; python_version >= '3.7' mergedeep==1.3.4; python_version >= '3.6' -mkdocs-include-markdown-plugin==3.3.0 +mkdocs-include-markdown-plugin==3.4.0 mkdocs-material-extensions==1.0.3; python_version >= '3.6' -mkdocs-material==8.2.11 +mkdocs-material==8.2.15 mkdocs==1.3.0 mypy-extensions==0.4.3 -mypy==0.942 -numpy==1.22.3; platform_machine != 'aarch64' and platform_machine != 'arm64' and python_version < '3.10' +mypy==0.950 +numpy==1.22.3; python_version < '3.10' and platform_machine != 'aarch64' and platform_machine != 'arm64' orderedmultidict==1.0.1 packaging==20.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pandas==1.4.2 pep517==0.12.0 pip-shims==0.7.0; python_version >= '3.6' -pip==22.0.4; python_version >= '3.7' +pip==22.1; python_version >= '3.7' pipenv-setup==3.2.0 pipfile==0.0.2 platformdirs==2.5.2; python_version >= '3.7' @@ -44,24 +44,24 @@ plette[validation]==0.2.3; python_version >= '2.6' and python_version not in '3. pluggy==1.0.0; python_version >= '3.6' py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycodestyle==2.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pygments==2.11.2; python_version >= '3.5' -pymdown-extensions==9.3; python_version >= '3.7' +pygments==2.12.0; python_version >= '3.6' +pymdown-extensions==9.4; python_version >= '3.7' pyparsing==2.4.7 pytest==7.1.2 -python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' +python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pytz==2022.1 pyyaml-env-tag==0.1; python_version >= '3.6' pyyaml==6.0; python_version >= '3.6' requests==2.27.1 requirementslib==1.6.4; python_version >= '3.7' -setuptools==62.1.0; python_version >= '3.7' -six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' -toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' +setuptools==62.2.0; python_version >= '3.7' +six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' tomli==2.0.1; python_version >= '3.7' -tomlkit==0.10.2; python_version >= '3.6' and python_version < '4' +tomlkit==0.10.2; python_version >= '3.6' and python_version < '4.0' typing-extensions==4.2.0; python_version >= '3.7' urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' vistir==0.5.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -watchdog==2.1.7; python_version >= '3.6' +watchdog==2.1.8; python_version >= '3.6' wheel==0.37.1 zipp==3.8.0; python_version >= '3.7' diff --git a/knora/dsplib/utils/onto_validate.py b/knora/dsplib/utils/onto_validate.py index 5b97046ce..5a397875e 100644 --- a/knora/dsplib/utils/onto_validate.py +++ b/knora/dsplib/utils/onto_validate.py @@ -1,9 +1,10 @@ -import json import os import re -from typing import Any, Union, List, Set +from typing import Any, Union import jsonschema +import json import jsonpath_ng, jsonpath_ng.ext +import networkx as nx from ..utils.expand_all_lists import expand_lists_from_excel @@ -56,33 +57,65 @@ def validate_ontology(input_file_or_json: Union[str, dict[Any, Any], 'os.PathLik def check_cardinalities_of_circular_references(data_model: dict[Any, Any]) -> bool: """ - Check if there are properties derived from hasLinkTo that form a circular reference. If so, these + Check a data model if it contains properties derived from hasLinkTo that form a circular reference. If so, these properties must have the cardinality 0-1 or 0-n, because during the xmlupload process, these values are temporarily removed. + + Args: + data_model: dictionary with a DSP project (as defined in a JSON ontology file) + + Returns: + True if no circle was detected, or if all elements of all circles are of cardinality "0-1" or "0-n". + False if there is a circle with at least one element that has a cardinality of "1" or "1-n". """ - # search the ontology for all properties that are derived from hasLinkTo, store them in a dict, and map - # them to their objects (i.e. the resource classes they point to) - # example: if the property 'rosetta:hasTextMedium' points to 'rosetta:Image2D': - # link_properties = {'rosetta:hasTextMedium': ['rosetta:Image2D'], ...} + link_properties = collect_link_properties(data_model) + errors = identify_problematic_cardinalities(data_model, link_properties) + + if len(errors) == 0: + return True + else: + print('ERROR: Your ontology contains properties derived from "hasLinkTo" that allow circular references ' + 'between resources. This is not a problem in itself, but if you try to upload data that actually ' + 'contains circular references, these "hasLinkTo" properties will be temporarily removed from the ' + 'affected resources. Therefore, it is necessary that all involved "hasLinkTo" properties have a ' + 'cardinality of 0-1 or 0-n. \n' + 'Please make sure that the following properties have a cardinality of 0-1 or 0-n:') + for error in errors: + print(f'\t- Resource {error[0]}, property {error[1]}') + return False + + +def collect_link_properties(data_model: dict[Any, Any]) -> dict[str, list[str]]: + """ + map the properties derived from hasLinkTo to the resource classes they point to, for example: + link_properties = {'rosetta:hasImage2D': ['rosetta:Image2D'], ...} + """ ontos = data_model['project']['ontologies'] - link_properties: dict[str, List[str]] = dict() + hasLinkTo_props = {'hasLinkTo', 'isPartOf', 'isRegionOf', 'isAnnotationOf'} + link_properties: dict[str, list[str]] = dict() for index, onto in enumerate(ontos): - hasLinkTo_matches = jsonpath_ng.ext.parse( - f'$.project.ontologies[{index}].properties[?@.super[*] == hasLinkTo]' - ).find(data_model) - prop_obj_pair: dict[str, List[str]] = dict() + hasLinkTo_matches = list() + # look for child-properties down to 5 inheritance levels that are derived from hasLinkTo-properties + for i in range(5): + for hasLinkTo_prop in hasLinkTo_props: + hasLinkTo_matches.extend(jsonpath_ng.ext.parse( + f'$.project.ontologies[{index}].properties[?super[*] == {hasLinkTo_prop}]' + ).find(data_model)) + # make the children from this iteration to the parents of the next iteration + hasLinkTo_props = {x.value['name'] for x in hasLinkTo_matches} + prop_obj_pair: dict[str, list[str]] = dict() for match in hasLinkTo_matches: prop = onto['name'] + ':' + match.value['name'] target = match.value['object'] if target != 'Resource': # make the target a fully qualified name (with the ontology's name prefixed) - target = re.sub(r'^(:?)([^:]+)$', f'{onto["name"]}:\\2', target) + target = re.sub(r'^:([^:]+)$', f'{onto["name"]}:\\1', target) prop_obj_pair[prop] = [target] link_properties.update(prop_obj_pair) # in case the object of a property is "Resource", the link can point to any resource class - all_res_names: List[str] = list() + all_res_names: list[str] = list() for index, onto in enumerate(ontos): matches = jsonpath_ng.ext.parse(f'$.resources[*].name').find(onto) tmp = [f'{onto["name"]}:{match.value}' for match in matches] @@ -91,11 +124,19 @@ def check_cardinalities_of_circular_references(data_model: dict[Any, Any]) -> bo if 'Resource' in targ: link_properties[prop] = all_res_names - # make a dict that maps resource classes to their hasLinkTo-properties, and to the classes they point to - # example: if 'rosetta:Text' has the property 'rosetta:hasTextMedium' that points to 'rosetta:Image2D': - # dependencies = {'rosetta:Text': {'rosetta:hasTextMedium': ['rosetta:Image2D'], ...}} - dependencies: dict[str, dict[str, List[str]]] = dict() - for onto in ontos: + return link_properties + + +def identify_problematic_cardinalities(data_model: dict[Any, Any], link_properties: dict[str, list[str]]) -> list[tuple[str, str]]: + """ + make an error list with all cardinalities that are part of a circle but have a cardinality of "1" or "1-n" + """ + # make 2 dicts of the following form: + # dependencies = {'rosetta:Text': {'rosetta:hasImage2D': ['rosetta:Image2D'], ...}} + # cardinalities = {'rosetta:Text': {'rosetta:hasImage2D': '0-1', ...}} + dependencies: dict[str, dict[str, list[str]]] = dict() + cardinalities: dict[str, dict[str, str]] = dict() + for onto in data_model['project']['ontologies']: for resource in onto['resources']: resname: str = onto['name'] + ':' + resource['name'] for card in resource['cardinalities']: @@ -111,64 +152,31 @@ def check_cardinalities_of_circular_references(data_model: dict[Any, Any]) -> bo if resname not in dependencies: dependencies[resname] = dict() dependencies[resname][cardname] = targets + cardinalities[resname] = dict() + cardinalities[resname][cardname] = card['cardinality'] elif cardname not in dependencies[resname]: dependencies[resname][cardname] = targets + cardinalities[resname][cardname] = card['cardinality'] else: dependencies[resname][cardname].extend(targets) - # iteratively purge dependencies from non-circular references - for _ in range(30): - # remove targets that point to a resource that is not in dependencies, - # remove cardinalities that have no targets - for res, cards in dependencies.copy().items(): - for card, targets in cards.copy().items(): - dependencies[res][card] = [target for target in targets if target in dependencies] - if len(dependencies[res][card]) == 0: - del dependencies[res][card] - # remove resources that have no cardinalities - dependencies = {res: cards for res, cards in dependencies.items() if len(cards) > 0} - # remove resources that are not pointed to by any target - all_targets: Set[str] = set() - for cards in dependencies.values(): - for trgt in cards.values(): - all_targets = all_targets | set(trgt) - dependencies = {res: targets for res, targets in dependencies.items() if res in all_targets} - - # check the remaining dependencies (which are only the circular ones) if they have all 0-1 or 0-n - ok_cardinalities = ['0-1', '0-n'] - notok_dependencies: dict[str, List[str]] = dict() - for res, cards in dependencies.items(): - ontoname, resname = res.split(':') - for card in cards: - # the name of the cardinality could be with prepended onto, only with colon, or without anything - card_without_colon = card.split(':')[1] - card_with_colon = ':' + card_without_colon - card_variations = [card, card_with_colon, card_without_colon] - for card_variation in card_variations: - match = jsonpath_ng.ext.parse( - f'$[?@.name == {ontoname}].resources[?@.name == {resname}].cardinalities[?@.propname == "{card_variation}"]' - ).find(ontos) - if len(match) > 0: - break - card_numbers = match[0].value['cardinality'] - if card_numbers not in ok_cardinalities: - if res not in notok_dependencies: - notok_dependencies[res] = [card] - else: - notok_dependencies[res].append(card) - - if len(notok_dependencies) == 0: - return True - else: - print('ERROR: Your ontology contains properties derived from "hasLinkTo" that allow circular references ' - 'between resources. This is not a problem in itself, but if you try to upload data that actually ' - 'contains circular references, these "hasLinkTo" cardinalities will be temporarily removed from the ' - 'affected resources. Therefore, it is necessary that the involved "hasLinkTo" cardinalities have a ' - 'cardinality of 0-1 or 0-n. \n' - 'Please make sure that the following cardinalities have a cardinality of 0-1 or 0-n:') - for _res, _cards in notok_dependencies.items(): - print(_res) - for card in _cards: - print(f'\t{card}') - return False - + # transform the dependencies into a graph structure + graph = nx.MultiDiGraph() + for start, cards in dependencies.items(): + for edge, targets in cards.items(): + for target in targets: + graph.add_edge(start, target, edge) + + # find elements of circles that have a cardinality of "1" or "1-n" + errors: set[tuple[str, str]] = set() + circles = list(nx.simple_cycles(graph)) + for circle in circles: + for index, resource in enumerate(circle): + target = circle[(index+1) % len(circle)] + for property, targets in dependencies[resource].items(): + if target in targets: + prop = property + if cardinalities[resource][prop] not in ['0-1', '0-n']: + errors.add((resource, prop)) + + return sorted(errors, key=lambda x: x[0]) diff --git a/requirements.txt b/requirements.txt index 29d6593fd..50b55eff7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,14 +10,15 @@ argparse==1.4.0 attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' certifi==2021.10.8 charset-normalizer==2.0.12; python_version >= '3' -click==8.1.2 +click==8.1.3 decorator==5.1.1; python_version >= '3.5' et-xmlfile==1.1.0; python_version >= '3.6' idna==3.3; python_version >= '3' isodate==0.6.1 jsonpath-ng==1.5.3 -jsonschema==4.4.0 +jsonschema==4.5.1 lxml==4.8.0 +networkx==2.8 openpyxl==3.0.9 ply==3.11 pyparsing==2.4.7 @@ -26,7 +27,7 @@ pystrict==1.2 rdflib==6.1.1 requests==2.27.1 rfc3987==1.3.8 -setuptools==62.1.0; python_version >= '3.7' -six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2' +setuptools==62.2.0; python_version >= '3.7' +six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' -validators==0.18.2 +validators==0.19.0 diff --git a/setup.py b/setup.py index 97508764b..1095c5ffb 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ "Operating System :: OS Independent", ], python_requires='>=3.9.0', - install_requires=['argparse==1.4.0', "attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 'certifi==2021.10.8', "charset-normalizer==2.0.12; python_version >= '3'", 'click==8.1.2', "decorator==5.1.1; python_version >= '3.5'", "et-xmlfile==1.1.0; python_version >= '3.6'", "idna==3.3; python_version >= '3'", 'isodate==0.6.1', 'jsonpath-ng==1.5.3', 'jsonschema==4.4.0', 'lxml==4.8.0', 'openpyxl==3.0.9', 'ply==3.11', 'pyparsing==2.4.7', "pyrsistent==0.18.1; python_version >= '3.7'", 'pystrict==1.2', 'rdflib==6.1.1', 'requests==2.27.1', 'rfc3987==1.3.8', "setuptools==62.1.0; python_version >= '3.7'", "six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", 'validators==0.18.2' + install_requires=['argparse==1.4.0', "attrs==21.4.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", 'certifi==2021.10.8', "charset-normalizer==2.0.12; python_version >= '3'", 'click==8.1.3', "decorator==5.1.1; python_version >= '3.5'", "et-xmlfile==1.1.0; python_version >= '3.6'", "idna==3.3; python_version >= '3'", 'isodate==0.6.1', 'jsonpath-ng==1.5.3', 'jsonschema==4.5.1', 'lxml==4.8.0', 'networkx==2.8', 'openpyxl==3.0.9', 'ply==3.11', 'pyparsing==2.4.7', "pyrsistent==0.18.1; python_version >= '3.7'", 'pystrict==1.2', 'rdflib==6.1.1', 'requests==2.27.1', 'rfc3987==1.3.8', "setuptools==62.2.0; python_version >= '3.7'", "six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "urllib3==1.26.9; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", 'validators==0.19.0' ], entry_points={ 'console_scripts': [ diff --git a/test/unittests/test_create_ontology.py b/test/unittests/test_create_ontology.py index c8d2fc2be..5aeca6973 100644 --- a/test/unittests/test_create_ontology.py +++ b/test/unittests/test_create_ontology.py @@ -2,14 +2,18 @@ import unittest import json from typing import Any +import jsonpath_ng.ext -from knora.dsplib.utils.onto_create_ontology import * +from knora.dsplib.utils.onto_create_ontology import sort_resources, sort_prop_classes +from knora.dsplib.utils.onto_validate import collect_link_properties, identify_problematic_cardinalities class TestOntoCreation(unittest.TestCase): with open('testdata/test-onto.json', 'r') as json_file: - json_onto: dict[str, Any] = json.load(json_file) - ontology: dict[str, Any] = json_onto['project']['ontologies'][0] + project: dict[str, Any] = json.load(json_file) + ontology: dict[str, Any] = project['project']['ontologies'][0] + with open('testdata/circular-onto.json', 'r') as json_file: + circular_onto: dict[str, Any] = json.load(json_file) def test_sort_resources(self) -> None: """ @@ -43,5 +47,15 @@ def test_sort_prop_classes(self) -> None: self.assertListEqual(unsorted_props, sorted_props) + def test_circular_references_in_onto(self) -> None: + link_properties = collect_link_properties(self.circular_onto) + errors = identify_problematic_cardinalities(self.circular_onto, link_properties) + expected_errors = [ + ('testonto:AnyResource', 'testonto:linkToTestThing1'), + ('testonto:TestThing3', 'testonto:linkToResource') + ] + self.assertListEqual(sorted(errors), sorted(expected_errors)) + + if __name__ == '__main__': unittest.main() diff --git a/testdata/circular-onto.json b/testdata/circular-onto.json new file mode 100644 index 000000000..6a26d5940 --- /dev/null +++ b/testdata/circular-onto.json @@ -0,0 +1,118 @@ +{ + "project": { + "shortcode": "1233", + "shortname": "test", + "longname": "test", + "descriptions": { + "en": "test" + }, + "ontologies": [ + { + "name": "testonto", + "label": "Test ontology", + "properties": [ + { + "name": "linkToResource", + "super": [ + "hasLinkTo" + ], + "object": "Resource", + "labels": { + "en": "has region" + }, + "gui_element": "Searchbox" + }, + { + "name": "linkToTestThing1", + "super": [ + "foaf:fantasy", + "isPartOf" + ], + "object": ":TestThing1", + "labels": { + "en": "has region" + }, + "gui_element": "Searchbox" + }, + { + "name": "linkToTestThing2", + "super": [ + "isAnnotationOf", + "foaf:fantasy" + ], + "object": ":TestThing2", + "labels": { + "en": "has region" + }, + "gui_element": "Searchbox" + }, + { + "name": "linkToTestThing3", + "super": [ + "isRegionOf" + ], + "object": ":TestThing3", + "labels": { + "en": "has region" + }, + "gui_element": "Searchbox" + } + ], + "resources": [ + { + "name": "TestThing1", + "super": "Resource", + "labels": { + "en": "TestThing1" + }, + "cardinalities": [ + { + "propname": ":linkToTestThing2", + "cardinality": "0-1" + } + ] + }, + { + "name": "TestThing2", + "super": "Resource", + "labels": { + "en": "TestThing2" + }, + "cardinalities": [ + { + "propname": ":linkToTestThing3", + "cardinality": "0-n" + } + ] + }, + { + "name": "TestThing3", + "super": "Resource", + "labels": { + "en": "TestThing3" + }, + "cardinalities": [ + { + "propname": ":linkToResource", + "cardinality": "1" + } + ] + }, + { + "name": "AnyResource", + "super": "Resource", + "labels": { + "en": "AnyResource" + }, + "cardinalities": [ + { + "propname": ":linkToTestThing1", + "cardinality": "1-n" + } + ] + } + ] + } + ] + } +}