Merge pull request #5625 from OpenMined/dev

0.5.0rc2 one more time

.github/workflows/syft-benchmark.yml

@@ -49,6 +49,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -72,4 +73,3 @@ jobs:
           alert-threshold: "200%"
           comment-on-alert: true
           fail-on-alert: true
-

.github/workflows/syft-merge_tests.yml

@@ -69,6 +69,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[dev] -f https://download.pytorch.org/whl/torch_stable.html
 
       - name: Check import works without dev packages

.github/workflows/syft-pr_tests.yml

@@ -148,6 +148,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -210,6 +211,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -274,6 +276,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -329,11 +332,12 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
       - name: Test PyGrid
         continue-on-error: true
         run: |
           python ./scripts/mnist.py
-          pytest -m grid --durations=50
+          pytest -m grid --durations=50

.github/workflows/syft-security.yml

@@ -13,7 +13,6 @@ defaults:
     working-directory: ./packages/syft
 
 jobs:
-
   package-security:
     strategy:
       max-parallel: 1
@@ -53,6 +52,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -72,4 +72,4 @@ jobs:
       - name: Snyk security check
         uses: snyk/actions/python-3.8@master
         env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

.github/workflows/syft-version_tests.yml

@@ -74,6 +74,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -146,6 +147,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[ci-all] -f https://download.pytorch.org/whl/torch_stable.html
           python scripts/parse_setup.py setup.cfg | xargs pip install --no-deps || true
 
@@ -207,6 +209,7 @@ jobs:
           pip install packaging --default-timeout=60
           python scripts/adjust_torch_versions.py ./setup.cfg ${{ matrix.torch-version }}
           grep -A 3 "torch_ecosystem" setup.cfg
+          pip install -r requirements.unstable.txt
           pip install -e .[dev] -f https://download.pytorch.org/whl/torch_stable.html
 
       - name: Check import works without dev packages

packages/syft/proto/lib/torch/size.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package syft.lib.torch;
+import "proto/core/common/common_object.proto";
+
+message Size {
+  repeated uint64 data = 1;
+  syft.core.common.UID id = 2;
+}

packages/syft/requirements.unstable.txt

@@ -0,0 +1,3 @@
+flake8-kwarger @ git+https://github.com/madhavajay/flake8-kwarger
+pytorch-lightning @ git+https://github.com/tudorcebere/pytorch-lightning@tudor-om-v0
+sympc @ git+https://github.com/OpenMined/SyMPC@main

packages/syft/setup.cfg

@@ -45,6 +45,7 @@ syft =
     pyarrow
     PyJWT==1.7.1
     PyNaCl
+    PyYAML>=5.4.1
     requests
     scikit-learn
     requests_toolbelt
@@ -91,9 +92,7 @@ libs =
     petlib
     pillow>=8.1.2 #security-issues
     python-dp
-    pytorch-lightning @ git+https://github.com/tudorcebere/pytorch-lightning@tudor-om-v0
     statsmodels
-    sympc @ git+https://github.com/OpenMined/SyMPC@main
     tenseal
     xgboost>=1.4
     zksk
@@ -113,7 +112,6 @@ dev =
     darglint
     doc8
     flake8
-    flake8-kwarger @ git+https://github.com/madhavajay/flake8-kwarger
     isort
     matplotlib
     nbconvert
@@ -150,9 +148,7 @@ ci-libs =
 ;    pillow>=8.1.2 #install-custom-dependency
 ;    pyarrow #install-custom-dependency
 ;    python-dp #install-custom-dependency
-;    pytorch-lightning@git+https://github.com/tudorcebere/pytorch-lightning@tudor-om-v0  #install-custom-dependency
 ;    statsmodels #install-custom-dependency
-;    sympc@git+https://github.com/OpenMined/SyMPC@main #install-custom-dependency
 ;    tenseal #install-custom-dependency
 ;    xgboost>=1.4 #install-custom-dependency
 ;    zksk #install-custom-dependency

packages/syft/src/syft/lib/torch/__init__.py

@@ -11,6 +11,7 @@
 from . import device  # noqa: 401
 from . import parameter  # noqa: 401
 from . import return_types  # noqa: 401
+from . import size  # noqa: 401
 from . import uppercase_tensor  # noqa: 401
 from ...ast.globals import Globals
 from ...logger import info

packages/syft/src/syft/lib/torch/size.py

@@ -0,0 +1,25 @@
+# third party
+import torch
+
+# syft relative
+from ...generate_wrapper import GenerateWrapper
+from ...proto.lib.torch.size_pb2 import Size as TorchSize_PB
+
+
+def protobuf_torch_size_serializer(torch_size: torch.Size) -> TorchSize_PB:
+    serialized_size = TorchSize_PB(data=torch_size)
+    return serialized_size
+
+
+def protobuf_torch_size_deserializer(proto_size: TorchSize_PB) -> torch.Size:
+    torch_size = torch.Size(proto_size.data)
+    return torch_size
+
+
+GenerateWrapper(
+    wrapped_type=torch.Size,
+    import_path="torch.Size",
+    protobuf_scheme=TorchSize_PB,
+    type_object2proto=protobuf_torch_size_serializer,
+    type_proto2object=protobuf_torch_size_deserializer,
+)

packages/syft/tests/syft/lib/torch/size_test.py

@@ -0,0 +1,22 @@
+# third party
+import torch
+
+# syft absolute
+import syft as sy
+
+
+def test_protobuf_torch_size_serializer_deserializer() -> None:
+    torch_size = torch.Size([4, 5, 2, 1])
+    torch_size_pb = sy.lib.torch.size.protobuf_torch_size_serializer(torch_size)
+    torch_size_deserialized = sy.lib.torch.size.protobuf_torch_size_deserializer(
+        torch_size_pb
+    )
+
+    assert isinstance(torch_size_deserialized, torch.Size)
+    assert torch_size == torch_size_deserialized
+
+
+def test_torch_size_serde(client: sy.VirtualMachineClient) -> None:
+    torch_size = torch.Size([2, 3, 4, 100])
+    torch_size_ptr = torch_size.send(client)
+    assert torch_size_ptr.get() == torch_size