Merge pull request #313 from Exabyte-io/update/SOF-7321-clean

add python model classes generation: update/SOF 7321 clean

.husky/pre-commit

@@ -8,10 +8,24 @@ then
   echo "Re-building JS and PY assets using JS script."
   npm run transpile-and-build-assets
 fi
+pwd
+
+if test -f .venv/bin/activate
+then
+  echo "Python virtual environment found. Running build scripts"
+  echo "Re-building JS and PY assets using JS script."
+  source .venv/bin/activate
+  # Note: resolved (no include statements) schemas have to exist by this time
+  pre-commit run --all-files
+  #git add dist/py
+else
+  echo "ERROR: Python virtual environment found. Create one in '.venv' and install '.[all]'"
+  exit 1
+fi
 
 npx lint-staged --allow-empty
 
 npm run lint:fix
 npm run transpile
-git add dist
+#git add dist/js
 

.pre-commit-config.yaml

@@ -3,18 +3,31 @@ repos:
     rev: 2023.6.13
     hooks:
       - id: ruff
-        exclude: ^src/py/mat3ra/esse/data
+        exclude: ^src/py/mat3ra/esse/data|^dist*
       - id: black
-        exclude: ^src/py/mat3ra/esse/data
+        exclude: ^src/py/mat3ra/esse/data|^dist*
   - repo: local
     hooks:
+      # Note: only python 3.8.12 is tested for the model generation
       - id: generate-python-modules
         name: Regenerate data modules classes from static assets
         # yamllint disable rule:line-length
         entry: |
-          bash -c 'if ! [ "$(git diff --cached --name-only | grep -e "example|schema")" ]; then echo "Skipping model generation because static files were not changed."; exit 0; fi'
-          bash -c 'python build_schemas.py'
-        # yamllint enable rule:line-length
+          bash -c 'if ! [ "$(git diff --cached --name-only | grep "schema/")" ]; then echo "Skipping model generation because schemas were not changed."; exit 0; fi && \
+            datamodel-codegen \
+                --input ./dist/js/schema/ \
+                --input-file-type jsonschema \
+                --output ./dist/py \
+                --output-model-type pydantic_v2.BaseModel \
+                --use-field-description \
+                --use-double-quotes \
+                --enable-version-header \
+                --use-title-as-name \
+                --class-name ESSE \
+                --disable-timestamp && \
+            # Move 3rd party modules to a separate directory with a PEP8 compliant name
+            rm -rf ./dist/py/third_party && \
+            mv ./dist/py/3pse ./dist/py/third_party'
         language: system
         pass_filenames: false
         verbose: true

build_schemas.py

@@ -58,4 +58,3 @@
     Path(full_path).parent.mkdir(parents=True, exist_ok=True)
     with open(full_path, "w") as f:
         f.write(json.dumps(example, sort_keys=True, indent=4, separators=(",", ": ")))
-

dist/js/schema/3pse/db/nist_jarvis/2024.3.13/atoms.json

@@ -1,7 +1,8 @@
 {
     "$id": "3pse/db/nist-jarvis/2024.3.13/atoms",
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
+    "title": "NIST JARVIS Atoms schema",
+    "description": "NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
     "type": "object",
     "additionalProperties": false,
     "properties": {

dist/js/schema/3pse/db/nist_jarvis/2024.3.13/db_entry.json

@@ -1,13 +1,15 @@
 {
     "$id": "3pse/db/nist-jarvis/2024.3.13/db-entry",
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "NIST J.A.R.V.I.S. db entry schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
+    "title": "NIST JARVIS Atoms schema",
+    "description": "NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
     "type": "object",
     "additionalProperties": true,
     "properties": {
         "atoms": {
             "$schema": "http://json-schema.org/draft-07/schema#",
-            "title": "NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
+            "title": "NIST JARVIS Atoms schema",
+            "description": "NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571",
             "type": "object",
             "additionalProperties": false,
             "properties": {

dist/js/types.ts

@@ -1,6 +1,9 @@
 /** Schema dist/js/schema/3pse/db/nist_jarvis/2024.3.13/atoms.json */
 
-export interface NISTJARVISDbEntryAtomsKeySchemaBasedOnHttpsFigshareComArticlesDatasetMonolayerDataForHeterostructure22344571 {
+/**
+ * NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571
+ */
+export interface NISTJARVISAtomsSchema {
   /**
    * Crystal lattice vectors as a 3x3 matrix, in Angstroms
    *
@@ -42,7 +45,13 @@ export interface NISTJARVISDbEntryAtomsKeySchemaBasedOnHttpsFigshareComArticlesD
 
 /** Schema dist/js/schema/3pse/db/nist_jarvis/2024.3.13/db_entry.json */
 
-export interface NISTJARVISDbEntrySchemaBasedOnHttpsFigshareComArticlesDatasetMonolayerDataForHeterostructure22344571 {
+/**
+ * NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571
+ */
+export interface NISTJARVISAtomsSchema {
+  /**
+   * NIST J.A.R.V.I.S. db entry `atoms` key schema. Based on https://figshare.com/articles/dataset/Monolayer_data_for_heterostructure/22344571
+   */
   atoms?: {
     /**
      * Crystal lattice vectors as a 3x3 matrix, in Angstroms

dist/py/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   version:   0.25.5

dist/py/core/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   version:   0.25.5

dist/py/core/abstract/2d_data.py

@@ -0,0 +1,17 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/2d_data.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List, Union
+
+from pydantic import BaseModel, Field
+
+
+class Field2DimensionDataSchema(BaseModel):
+    xDataArray: List
+    """
+    array containing values of x Axis
+    """
+    yDataSeries: List[List[Union[float, str]]] = Field(..., title="1 dimension data series schema")

dist/py/core/abstract/2d_plot.py

@@ -0,0 +1,34 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/2d_plot.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Field
+
+
+class AxisSchema(BaseModel):
+    label: str
+    """
+    label of an axis object
+    """
+    units: Optional[str] = None
+    """
+    units for an axis
+    """
+
+
+class Field2DimensionPlotSchema(BaseModel):
+    xAxis: AxisSchema = Field(..., title="axis schema")
+    yAxis: AxisSchema = Field(..., title="axis schema")
+    legend: Optional[List] = Field(None, min_length=1)
+    """
+    Legend of y Axis data series
+    """
+    xDataArray: List
+    """
+    array containing values of x Axis
+    """
+    yDataSeries: List[List[Union[float, str]]] = Field(..., title="1 dimension data series schema")

dist/py/core/abstract/3d_grid.py

@@ -0,0 +1,14 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/3d_grid.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import BaseModel, Field
+
+
+class Field3DimensionalGridSchema(BaseModel):
+    dimensions: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")
+    shifts: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")

dist/py/core/abstract/3d_tensor.py

@@ -0,0 +1,19 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/3d_tensor.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import Any, List
+
+from pydantic import Field, RootModel
+
+
+class ArrayOf3NumberElementsSchema(RootModel[List[Any]]):
+    root: List[Any] = Field(..., title="array of 3 number elements schema")
+
+
+class Field3DimensionalTensorSchema(RootModel[List[ArrayOf3NumberElementsSchema]]):
+    root: List[ArrayOf3NumberElementsSchema] = Field(
+        ..., max_length=3, min_length=3, title="3 dimensional tensor schema"
+    )

dist/py/core/abstract/3d_vector_basis.py

@@ -0,0 +1,15 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/3d_vector_basis.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import BaseModel, Field
+
+
+class Field3DimensionalVectorBasis(BaseModel):
+    a: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")
+    b: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")
+    c: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")

dist/py/core/abstract/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   version:   0.25.5

dist/py/core/abstract/point.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/point.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import Field, RootModel
+
+
+class PointSchema(RootModel[List[float]]):
+    root: List[float] = Field(..., max_length=3, min_length=3, title="point schema")

dist/py/core/abstract/vector.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  core/abstract/vector.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List, Union
+
+from pydantic import Field, RootModel
+
+
+class ESSE(RootModel[Union[List[float], List[bool]]]):
+    root: Union[List[float], List[bool]] = Field(..., title="vector schema")

dist/py/core/primitive/1d_data_series.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/1d_data_series.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List, Union
+
+from pydantic import Field, RootModel
+
+
+class Field1DimensionDataSeriesSchema(RootModel[List[List[Union[float, str]]]]):
+    root: List[List[Union[float, str]]] = Field(..., title="1 dimension data series schema")

dist/py/core/primitive/3d_lattice.py

@@ -0,0 +1,34 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/3d_lattice.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class Field3DimensionalLatticeSchema(BaseModel):
+    a: float
+    """
+    length of the first lattice vector
+    """
+    b: float
+    """
+    length of the second lattice vector
+    """
+    c: float
+    """
+    length of the third lattice vector
+    """
+    alpha: float
+    """
+    angle between first and second lattice vector
+    """
+    beta: float
+    """
+    angle between second and third lattice vector
+    """
+    gamma: float
+    """
+    angle between first and third lattice vector
+    """

dist/py/core/primitive/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  schema
+#   version:   0.25.5

dist/py/core/primitive/array_of_3_booleans.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/array_of_3_booleans.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import Field, RootModel
+
+
+class ArrayOf3BooleanElementsSchema(RootModel[List[bool]]):
+    root: List[bool] = Field(..., max_length=3, min_length=3, title="array of 3 boolean elements schema")

dist/py/core/primitive/array_of_3_numbers.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/array_of_3_numbers.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import Field, RootModel
+
+
+class ArrayOf3NumberElementsSchema(RootModel[List[float]]):
+    root: List[float] = Field(..., max_length=3, min_length=3, title="array of 3 number elements schema")

dist/py/core/primitive/array_of_ids.py

@@ -0,0 +1,23 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/array_of_ids.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Field, RootModel
+
+
+class AtomicId(BaseModel):
+    id: Optional[int] = None
+    """
+    integer id of this entry
+    """
+
+
+class AtomicIds(RootModel[List[AtomicId]]):
+    root: List[AtomicId] = Field(..., title="atomic ids")
+    """
+    array of objects containing integer id each
+    """

dist/py/core/primitive/array_of_strings.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  core/primitive/array_of_strings.json
+#   version:   0.25.5
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import Field, RootModel
+
+
+class ArrayOfStrings(RootModel[List[str]]):
+    root: List[str] = Field(..., title="array of strings")
+    """
+    array of strings, e.g. metadata tags
+    """