Upgrade jsonschema (#1477)

* Suppress SQLAlchemy warnings.

* Fixed spelling mistake in whats_new and fixed drop schema where no schema does not exist.

* Switch to jsonschema 4.18+ "references" API.

* Rebase onto develop.

* Add repin (as in pin again) to the dictionary.

* Ugh - could have sworn this was working on Friday.

conda-environment.yml

@@ -22,7 +22,7 @@ dependencies:
   - dask
   - pyproj >=2.5
   - shapely >=2.0
-  - jsonschema <4.18
+  - jsonschema >=4.18
   - lark
   - netcdf4
   - numpy

datacube/utils/documents.py

@@ -202,6 +202,7 @@ def read_strings_from_netcdf(path, variable):
 
 def validate_document(document, schema, schema_folder=None):
     import jsonschema
+    import referencing
 
     try:
         # Allow schemas to reference other schemas in the given folder.
@@ -210,14 +211,14 @@ def doc_reference(path):
             if not path.exists():
                 raise ValueError("Reference not found: %s" % path)
             referenced_schema = next(iter(read_documents(path)))[1]
-            return referenced_schema
+            return referencing.Resource(referenced_schema, referencing.jsonschema.DRAFT4)
 
+        if schema_folder:
+            registry = referencing.Registry(retrieve=doc_reference)
+        else:
+            registry = referencing.Registry()
         jsonschema.Draft4Validator.check_schema(schema)
-        ref_resolver = jsonschema.RefResolver.from_schema(
-            schema,
-            handlers={'': doc_reference} if schema_folder else ()
-        )
-        validator = jsonschema.Draft4Validator(schema, resolver=ref_resolver)
+        validator = jsonschema.Draft4Validator(schema, registry=registry)
         validator.validate(document)
     except jsonschema.ValidationError as e:
         raise InvalidDocException(e)

docker/constraints.in

@@ -14,8 +14,8 @@ dask>=2021.10.1
 distributed>=2021.10.0
 fiona
 geoalchemy2
-jsonschema<4.18
-# Was lark-parser>=0.6.7
+# New reference resolution API
+jsonschema>=4.18
 lark
 matplotlib
 moto

docker/constraints.txt

@@ -24,6 +24,7 @@ attrs==22.2.0
     #   jsonschema
     #   pytest
     #   rasterio
+    #   referencing
 babel==2.11.0
     # via sphinx
 bleach==6.0.0
@@ -163,8 +164,10 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsonschema==4.17.3
+jsonschema==4.18.4
     # via -r constraints.in
+jsonschema-specifications==2023.7.1
+    # via jsonschema
 keyring==23.13.1
     # via twine
 kiwisolver==1.4.4
@@ -266,8 +269,6 @@ pyproj==3.4.1
     # via
     #   -r constraints.in
     #   compliance-checker
-pyrsistent==0.19.3
-    # via jsonschema
 pytest==7.2.1
     # via
     #   -r constraints.in
@@ -309,6 +310,10 @@ recommonmark==0.7.1
     # via -r constraints.in
 redis==4.5.1
     # via -r constraints.in
+referencing==0.30.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 regex==2022.10.31
     # via compliance-checker
 requests==2.28.2
@@ -328,6 +333,10 @@ rfc3986==2.0.0
     # via twine
 rich==13.3.1
     # via twine
+rpds-py==0.9.2
+    # via
+    #   jsonschema
+    #   referencing
 ruamel-yaml==0.17.21
     # via -r constraints.in
 ruamel-yaml-clib==0.2.7

docs/about/whats_new.rst

@@ -8,6 +8,7 @@ What's New
 v1.8.next
 =========
 - Fix schema creation with postgres driver when initialising system with ``--no-init-users`` (:pull:`1504`)
+- Switch to new jsonschema 'referencing' API and repin jsonschema to >=4.18 (:pull:`1477`)
 
 v1.8.16 (17th October 2023)
 ===========================

setup.py

@@ -97,7 +97,7 @@
         'cloudpickle>=0.4',
         'dask[array]',
         'distributed',
-        'jsonschema<4.18',
+        'jsonschema>=4.18',  # New reference resolution API
         'netcdf4',
         'numpy',
         'psycopg2',

wordlist.txt

@@ -391,6 +391,7 @@ reampling
 redis
 Reflectance
 reflectance
+repin
 Reproject
 reproject
 reprojected