Corrections to file storage logic to allow removing activities

synapseclient/api/__init__.py

@@ -13,6 +13,7 @@
     post_entity,
     put_entity,
     create_access_requirements_if_none,
+    delete_entity_generated_by,
 )
 from .file_services import (
     get_file_handle,
@@ -51,4 +52,5 @@
     "get_upload_destination",
     "get_upload_destination_location",
     "create_access_requirements_if_none",
+    "delete_entity_generated_by",
 ]

synapseclient/api/entity_services.py

@@ -175,3 +175,23 @@ async def create_access_requirements_if_none(
             "the Synapse access control team to start the process of adding "
             "terms-of-use or review board approval for this entity."
         )
+
+
+async def delete_entity_generated_by(
+    entity_id: str,
+    synapse_client: Optional["Synapse"] = None,
+) -> None:
+    """
+    Arguments:
+        entity_id: The ID of the entity.
+        synapse_client: If not passed in or None this will use the last client from
+            the `.login()` method.
+
+    Returns: None
+    """
+    from synapseclient import Synapse
+
+    client = Synapse.get_client(synapse_client=synapse_client)
+    return await client.rest_delete_async(
+        uri=f"/entity/{entity_id}/generatedBy",
+    )

synapseclient/core/utils.py

@@ -1440,3 +1440,32 @@ def merge_dataclass_entities(
         setattr(destination, key, value)
 
     return destination
+
+
+def merge_metadata_fields(
+    source: typing.Union["Project", "Folder", "File"],
+    destination: typing.Union["Project", "Folder", "File"],
+) -> typing.Union["Project", "Folder"]:
+    """
+    Utility function to merge two dataclass entities together. This will only merge the
+    minimum amount of data required for an update with Synapse.
+
+    Arguments:
+        source: The source entity to merge from.
+        destination: The destination entity to merge into.
+
+    Returns:
+        The destination entity with the merged values.
+    """
+    # pylint: disable=protected-access
+    destination._last_persistent_instance = (
+        destination._last_persistent_instance or source._last_persistent_instance
+    )
+    destination.id = destination.id or source.id
+    destination.etag = destination.etag or source.etag
+    destination.version_number = destination.version_number or source.version_number
+    destination.version_label = destination.version_label or source.version_label
+    destination.version_comment = destination.version_comment or source.version_comment
+    destination.modified_on = destination.modified_on or source.modified_on
+
+    return destination

synapseclient/models/activity.py

@@ -5,6 +5,7 @@
 from opentelemetry import context, trace
 
 from synapseclient import Synapse
+from synapseclient.api import delete_entity_generated_by
 from synapseclient.activity import Activity as Synapse_Activity
 from synapseclient.core.async_utils import async_to_sync, otel_trace_method
 from synapseclient.core.constants.concrete_types import USED_ENTITY, USED_URL
@@ -389,3 +390,30 @@ async def delete_async(
                     current_context,
                 ),
             )
+
+    @classmethod
+    async def disassociate_from_entity_async(
+        cls,
+        parent: Union["Table", "File"],
+        synapse_client: Optional[Synapse] = None,
+    ) -> None:
+        """
+        Disassociate the Activity from the parent entity. This is the first step in
+        deleting the Activity. If you have other entities that are associated with this
+        Activity you must disassociate them by calling this method on them as well.
+
+        Arguments:
+            parent: The parent entity this activity is associated with.
+            synapse_client: If not passed in or None this will use the last client
+                from the `.login()` method.
+
+        Raises:
+            ValueError: If the parent does not have an ID.
+        """
+        # TODO: Input validation: SYNPY-1400
+        with tracer.start_as_current_span(
+            name=f"Activity_disassociate: Parent_ID: {parent.id}"
+        ):
+            await delete_entity_generated_by(
+                entity_id=parent.id, synapse_client=synapse_client
+            )

synapseclient/models/file.py

@@ -24,6 +24,7 @@
     delete_none_keys,
     guess_file_name,
     merge_dataclass_entities,
+    merge_metadata_fields,
     run_and_attach_otel_context,
 )
 from synapseclient.entity import File as Synapse_File
@@ -249,6 +250,12 @@ class File(FileSynchronousProtocol, AccessControllable):
             being restricted and the requirements of access.
 
             This may be used only by an administrator of the specified file.
+        merge_with_found_resource: (Store only)
+            Works in conjunction with `create_or_update` in that this is only evaluated
+            if `create_or_update` is True. If this is True the metadata will be merged
+            with the existing metadata in Synapse. If False the existing metadata will
+            be replaced with the new metadata. When this is False any updates will act
+            as a destructive update.
 
         synapse_store: (Store only)
             Whether the File should be uploaded or if false: only the path should
@@ -418,6 +425,16 @@ class File(FileSynchronousProtocol, AccessControllable):
     This may be used only by an administrator of the specified file.
     """
 
+    merge_with_found_resource: bool = field(default=True, repr=False, compare=False)
+    """
+    (Store only)
+
+    Works in conjunction with `create_or_update` in that this is only evaluated if
+    `create_or_update` is True. If this is True the metadata will be merged with the
+    existing metadata in Synapse. If False the existing metadata will be replaced with
+    the new metadata. When this is False any updates will act as a destructive update.
+    """
+
     synapse_store: bool = field(default=True, repr=False)
     """
     (Store only)
@@ -615,7 +632,9 @@ async def get_file(existing_id: str) -> "File":
                     synapse_container_limit=self.synapse_container_limit,
                     parent_id=self.parent_id,
                 )
-                return await file_copy.get_async(synapse_client=synapse_client)
+                return await file_copy.get_async(
+                    synapse_client=synapse_client, include_activity=True
+                )
             except SynapseFileNotFoundError:
                 return None
 
@@ -722,7 +741,10 @@ async def store_async(
         client = Synapse.get_client(synapse_client=synapse_client)
 
         if existing_file := await self._find_existing_file(synapse_client=client):
-            merge_dataclass_entities(source=existing_file, destination=self)
+            if self.merge_with_found_resource:
+                merge_dataclass_entities(source=existing_file, destination=self)
+            else:
+                merge_metadata_fields(source=existing_file, destination=self)
 
         if self.path:
             self.path = os.path.expanduser(self.path)

synapseclient/models/protocols/activity_protocol.py

@@ -88,3 +88,24 @@ def delete(
             ValueError: If the parent does not have an ID.
         """
         return None
+
+    @classmethod
+    async def disassociate_from_entity(
+        cls,
+        parent: Union["Table", "File"],
+        synapse_client: Optional[Synapse] = None,
+    ) -> None:
+        """
+        Disassociate the Activity from the parent entity. This is the first step in
+        deleting the Activity. If you have other entities that are associated with this
+        Activity you must disassociate them by calling this method on them as well.
+
+        Arguments:
+            parent: The parent entity this activity is associated with.
+            synapse_client: If not passed in or None this will use the last client
+                from the `.login()` method.
+
+        Raises:
+            ValueError: If the parent does not have an ID.
+        """
+        return None

synapseclient/models/services/storable_entity_components.py

@@ -203,16 +203,23 @@ async def _store_activity_and_annotations(
 
     if (
         hasattr(root_resource, "activity")
-        and root_resource.activity is not None
+        and (root_resource.activity or last_persistent_instance)
         and (
             last_persistent_instance is None
             or last_persistent_instance.activity != root_resource.activity
         )
     ):
-        result = await root_resource.activity.store_async(
-            parent=root_resource, synapse_client=synapse_client
-        )
+        if root_resource.activity:
+            result = await root_resource.activity.store_async(
+                parent=root_resource, synapse_client=synapse_client
+            )
+            root_resource.activity = result
+        else:
+            from synapseclient.models import Activity
+
+            await Activity.disassociate_from_entity_async(
+                parent=root_resource, synapse_client=synapse_client
+            )
 
-        root_resource.activity = result
         return True
     return False

synapseutils/sync.py

@@ -1345,6 +1345,7 @@ async def _manifest_upload(syn: Synapse, df) -> bool:
             synapse_store=row["synapseStore"] if "synapseStore" in row else True,
             content_type=row["contentType"] if "contentType" in row else None,
             force_version=row["forceVersion"] if "forceVersion" in row else True,
+            merge_with_found_resource=False,
         )
 
         manifest_style_annotations = dict(