From 43903ea36a2ccdc30684617f6280abe77440374f Mon Sep 17 00:00:00 2001
From: Paul Haesler <linkjuggler@gmail.com>
Date: Mon, 6 Nov 2023 13:24:21 +1100
Subject: [PATCH] Upgrade jsonschema (#1477)

* Suppress SQLAlchemy warnings.

* Fixed spelling mistake in whats_new and fixed drop schema where no schema does not exist.

* Switch to jsonschema 4.18+ "references" API.

* Rebase onto develop.

* Add repin (as in pin again) to the dictionary.

* Ugh - could have sworn this was working on Friday.
---
 conda-environment.yml       |  2 +-
 datacube/utils/documents.py | 13 +++++++------
 docker/constraints.in       |  4 ++--
 docker/constraints.txt      | 15 ++++++++++++---
 docs/about/whats_new.rst    |  1 +
 setup.py                    |  2 +-
 wordlist.txt                |  1 +
 7 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/conda-environment.yml b/conda-environment.yml
index 623afe441..fb867b0fb 100644
--- a/conda-environment.yml
+++ b/conda-environment.yml
@@ -22,7 +22,7 @@ dependencies:
   - dask
   - pyproj >=2.5
   - shapely >=2.0
-  - jsonschema <4.18
+  - jsonschema >=4.18
   - lark
   - netcdf4
   - numpy
diff --git a/datacube/utils/documents.py b/datacube/utils/documents.py
index 874a1f213..3d7d6fb38 100644
--- a/datacube/utils/documents.py
+++ b/datacube/utils/documents.py
@@ -202,6 +202,7 @@ def read_strings_from_netcdf(path, variable):
 
 def validate_document(document, schema, schema_folder=None):
     import jsonschema
+    import referencing
 
     try:
         # Allow schemas to reference other schemas in the given folder.
@@ -210,14 +211,14 @@ def doc_reference(path):
             if not path.exists():
                 raise ValueError("Reference not found: %s" % path)
             referenced_schema = next(iter(read_documents(path)))[1]
-            return referenced_schema
+            return referencing.Resource(referenced_schema, referencing.jsonschema.DRAFT4)
 
+        if schema_folder:
+            registry = referencing.Registry(retrieve=doc_reference)
+        else:
+            registry = referencing.Registry()
         jsonschema.Draft4Validator.check_schema(schema)
-        ref_resolver = jsonschema.RefResolver.from_schema(
-            schema,
-            handlers={'': doc_reference} if schema_folder else ()
-        )
-        validator = jsonschema.Draft4Validator(schema, resolver=ref_resolver)
+        validator = jsonschema.Draft4Validator(schema, registry=registry)
         validator.validate(document)
     except jsonschema.ValidationError as e:
         raise InvalidDocException(e)
diff --git a/docker/constraints.in b/docker/constraints.in
index 67b98b60e..f5c24dc46 100644
--- a/docker/constraints.in
+++ b/docker/constraints.in
@@ -14,8 +14,8 @@ dask>=2021.10.1
 distributed>=2021.10.0
 fiona
 geoalchemy2
-jsonschema<4.18
-# Was lark-parser>=0.6.7
+# New reference resolution API
+jsonschema>=4.18
 lark
 matplotlib
 moto
diff --git a/docker/constraints.txt b/docker/constraints.txt
index 98de0175e..d6ccf8e2a 100644
--- a/docker/constraints.txt
+++ b/docker/constraints.txt
@@ -24,6 +24,7 @@ attrs==22.2.0
     #   jsonschema
     #   pytest
     #   rasterio
+    #   referencing
 babel==2.11.0
     # via sphinx
 bleach==6.0.0
@@ -163,8 +164,10 @@ jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsonschema==4.17.3
+jsonschema==4.18.4
     # via -r constraints.in
+jsonschema-specifications==2023.7.1
+    # via jsonschema
 keyring==23.13.1
     # via twine
 kiwisolver==1.4.4
@@ -266,8 +269,6 @@ pyproj==3.4.1
     # via
     #   -r constraints.in
     #   compliance-checker
-pyrsistent==0.19.3
-    # via jsonschema
 pytest==7.2.1
     # via
     #   -r constraints.in
@@ -309,6 +310,10 @@ recommonmark==0.7.1
     # via -r constraints.in
 redis==4.5.1
     # via -r constraints.in
+referencing==0.30.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 regex==2022.10.31
     # via compliance-checker
 requests==2.28.2
@@ -328,6 +333,10 @@ rfc3986==2.0.0
     # via twine
 rich==13.3.1
     # via twine
+rpds-py==0.9.2
+    # via
+    #   jsonschema
+    #   referencing
 ruamel-yaml==0.17.21
     # via -r constraints.in
 ruamel-yaml-clib==0.2.7
diff --git a/docs/about/whats_new.rst b/docs/about/whats_new.rst
index 77d49116c..18b930986 100644
--- a/docs/about/whats_new.rst
+++ b/docs/about/whats_new.rst
@@ -8,6 +8,7 @@ What's New
 v1.8.next
 =========
 - Fix schema creation with postgres driver when initialising system with ``--no-init-users`` (:pull:`1504`)
+- Switch to new jsonschema 'referencing' API and repin jsonschema to >=4.18 (:pull:`1477`)
 
 v1.8.16 (17th October 2023)
 ===========================
diff --git a/setup.py b/setup.py
index 0d069d829..8464f799a 100755
--- a/setup.py
+++ b/setup.py
@@ -97,7 +97,7 @@
         'cloudpickle>=0.4',
         'dask[array]',
         'distributed',
-        'jsonschema<4.18',
+        'jsonschema>=4.18',  # New reference resolution API
         'netcdf4',
         'numpy',
         'psycopg2',
diff --git a/wordlist.txt b/wordlist.txt
index 2f935c5da..a4ebca390 100644
--- a/wordlist.txt
+++ b/wordlist.txt
@@ -391,6 +391,7 @@ reampling
 redis
 Reflectance
 reflectance
+repin
 Reproject
 reproject
 reprojected