Multi recording archive

sigmf/archive.py

@@ -10,6 +10,8 @@
 import shutil
 import tarfile
 import tempfile
+from typing import BinaryIO, Iterable
+
 
 from .error import SigMFFileError
 
@@ -21,80 +23,83 @@
 
 
 class SigMFArchive():
-    """Archive a SigMFFile.
+    """Archive one or more `SigMFFile`s.
 
     A `.sigmf` file must include both valid metadata and data.
     If `self.data_file` is not set or the requested output file
     is not writable, raise `SigMFFileError`.
 
     Parameters:
 
-      sigmffile -- A SigMFFile object with valid metadata and data_file
+      sigmffile -- An iterable of SigMFFile objects with valid metadata and data_files
 
       name      -- path to archive file to create. If file exists, overwrite.
-                   If `name` doesn't end in .sigmf, it will be appended.
-                   For example: if `name` == "/tmp/archive1", then the
-                   following archive will be created:
-                       /tmp/archive1.sigmf
-                       - archive1/
-                         - archive1.sigmf-meta
-                         - archive1.sigmf-data
+                    If `name` doesn't end in .sigmf, it will be appended. The
+                    `self.path` instance variable will be updated upon
+                    successful writing of the archive to point to the final
+                    archive path.
+
 
       fileobj   -- If `fileobj` is specified, it is used as an alternative to
-                   a file object opened in binary mode for `name`. It is
-                   supposed to be at position 0. `name` is not required, but
-                   if specified will be used to determine the directory and
-                   file names within the archive. `fileobj` won't be closed.
-                   For example: if `name` == "archive1" and fileobj is given,
-                   a tar archive will be written to fileobj with the
-                   following structure:
-                       - archive1/
-                         - archive1.sigmf-meta
-                         - archive1.sigmf-data
+                    a file object opened in binary mode for `name`. If
+                    `fileobj` is an open tarfile, it will be appended to. It is
+                    supposed to be at position 0. `fileobj` won't be closed. If
+                    `fileobj` is given, `name` has no effect.
     """
-    def __init__(self, sigmffile, name=None, fileobj=None):
-        self.sigmffile = sigmffile
+    def __init__(self, sigmffiles : Iterable["SigMFFile"], name : str = None, fileobj : BinaryIO =None):
+        self.sigmffiles = sigmffiles
+
         self.name = name
         self.fileobj = fileobj
 
         self._check_input()
 
-        archive_name = self._get_archive_name()
+        mode = "a" if fileobj is not None else "w"
         sigmf_fileobj = self._get_output_fileobj()
-        sigmf_archive = tarfile.TarFile(mode="w",
-                                        fileobj=sigmf_fileobj,
-                                        format=tarfile.PAX_FORMAT)
-        tmpdir = tempfile.mkdtemp()
-        sigmf_md_filename = archive_name + SIGMF_METADATA_EXT
-        sigmf_md_path = os.path.join(tmpdir, sigmf_md_filename)
-        sigmf_data_filename = archive_name + SIGMF_DATASET_EXT
-        sigmf_data_path = os.path.join(tmpdir, sigmf_data_filename)
-
-        with open(sigmf_md_path, "w") as mdfile:
-            self.sigmffile.dump(mdfile, pretty=True)
-
-        shutil.copy(self.sigmffile.data_file, sigmf_data_path)
-
+        try:
+            sigmf_archive = tarfile.TarFile(mode=mode,
+                                            fileobj=sigmf_fileobj,
+                                            format=tarfile.PAX_FORMAT)
+        except tarfile.ReadError:
+             # fileobj doesn't contain any archives yet, so reopen in 'w' mode
+            sigmf_archive = tarfile.TarFile(mode='w',
+                                            fileobj=sigmf_fileobj,
+                                            format=tarfile.PAX_FORMAT)
+
         def chmod(tarinfo):
             if tarinfo.isdir():
                 tarinfo.mode = 0o755  # dwrxw-rw-r
             else:
                 tarinfo.mode = 0o644  # -wr-r--r--
             return tarinfo
 
-        sigmf_archive.add(tmpdir, arcname=archive_name, filter=chmod)
+        for sigmffile in self.sigmffiles:
+            with tempfile.TemporaryDirectory() as tmpdir:
+                sigmf_md_filename = sigmffile.name + SIGMF_METADATA_EXT
+                sigmf_md_path = os.path.join(tmpdir, sigmf_md_filename)
+                sigmf_data_filename = sigmffile.name + SIGMF_DATASET_EXT
+                sigmf_data_path = os.path.join(tmpdir, sigmf_data_filename)
+
+                with open(sigmf_md_path, "w") as mdfile:
+                    sigmffile.dump(mdfile, pretty=True)
+
+                shutil.copy(sigmffile.data_file, sigmf_data_path)
+                sigmf_archive.add(tmpdir, arcname=sigmffile.name, filter=chmod)
+
         sigmf_archive.close()
         if not fileobj:
             sigmf_fileobj.close()
-
-        shutil.rmtree(tmpdir)
+        else:
+            sigmf_fileobj.seek(0)  # ensure next open can read this as a tar
 
         self.path = sigmf_archive.name
 
     def _check_input(self):
         self._ensure_name_has_correct_extension()
-        self._ensure_data_file_set()
-        self._validate_sigmffile_metadata()
+        for sigmffile in self.sigmffiles:
+            self._ensure_sigmffile_name_set(sigmffile)
+            self._ensure_data_file_set(sigmffile)
+            self._validate_sigmffile_metadata(sigmffile)
 
     def _ensure_name_has_correct_extension(self):
         name = self.name
@@ -110,13 +115,21 @@ def _ensure_name_has_correct_extension(self):
 
         self.name = name if has_correct_extension else name + SIGMF_ARCHIVE_EXT
 
-    def _ensure_data_file_set(self):
-        if not self.sigmffile.data_file:
+    @staticmethod
+    def _ensure_sigmffile_name_set(sigmffile):
+        if not sigmffile.name:
+            err = "the `name` attribute must be set to pass to `SigMFArchive`"
+            raise SigMFFileError(err)
+
+    @staticmethod
+    def _ensure_data_file_set(sigmffile):
+        if not sigmffile.data_file:
             err = "no data file - use `set_data_file`"
             raise SigMFFileError(err)
 
-    def _validate_sigmffile_metadata(self):
-        self.sigmffile.validate()
+    @staticmethod
+    def _validate_sigmffile_metadata(sigmffile):
+        sigmffile.validate()
 
     def _get_archive_name(self):
         if self.fileobj and not self.name:

sigmf/archivereader.py

@@ -28,65 +28,75 @@ class SigMFArchiveReader():
     """
     def __init__(self, name=None, skip_checksum=False, map_readonly=True, archive_buffer=None):
         self.name = name
-        if self.name is not None:
-            if not name.endswith(SIGMF_ARCHIVE_EXT):
-                err = "archive extension != {}".format(SIGMF_ARCHIVE_EXT)
-                raise SigMFFileError(err)
+        tar_obj = None
+        try:
+            if self.name is not None:
+                if not name.endswith(SIGMF_ARCHIVE_EXT):
+                    err = "archive extension != {}".format(SIGMF_ARCHIVE_EXT)
+                    raise SigMFFileError(err)
 
-            tar_obj = tarfile.open(self.name)
+                tar_obj = tarfile.open(self.name)
 
-        elif archive_buffer is not None:
-            tar_obj = tarfile.open(fileobj=archive_buffer, mode='r:')
+            elif archive_buffer is not None:
+                tar_obj = tarfile.open(fileobj=archive_buffer, mode='r:')
 
-        else:
-            raise ValueError('In sigmf.archivereader.__init__(), either `name` or `archive_buffer` must be not None')
-
-        json_contents = None
-        data_offset_size = None
-
-        for memb in tar_obj.getmembers():
-            if memb.isdir():  # memb.type == tarfile.DIRTYPE:
-                # the directory structure will be reflected in the member name
-                continue
-
-            elif memb.isfile():  # memb.type == tarfile.REGTYPE:
-                if memb.name.endswith(SIGMF_METADATA_EXT):
-                    json_contents = memb.name
-                    if data_offset_size is None:
-                        # consider a warnings.warn() here; the datafile should be earlier in the
-                        # archive than the metadata, so that updating it (like, adding an annotation)
-                        # is fast.
-                        pass
-                    with tar_obj.extractfile(memb) as memb_fid:
-                        json_contents = memb_fid.read()
-
-                elif memb.name.endswith(SIGMF_DATASET_EXT):
-                    data_offset_size = memb.offset_data, memb.size
-
-                else:
-                    print('A regular file', memb.name, 'was found but ignored in the archive')
             else:
-                print('A member of type', memb.type, 'and name', memb.name, 'was found but not handled, just FYI.')
-
-        if data_offset_size is None:
-            raise SigMFFileError('No .sigmf-data file found in archive!')
+                raise ValueError('In sigmf.archivereader.__init__(), either `name` or `archive_buffer` must be not None')
+
+            json_contents = None
+            data_offset_size = None
+            self.sigmffiles = []
+            data_found = False
+
+            for memb in tar_obj.getmembers():
+                if memb.isdir():  # memb.type == tarfile.DIRTYPE:
+                    # the directory structure will be reflected in the member name
+                    continue
+
+                elif memb.isfile():  # memb.type == tarfile.REGTYPE:
+                    if memb.name.endswith(SIGMF_METADATA_EXT):
+                        json_contents = memb.name
+                        if data_offset_size is None:
+                            # consider a warnings.warn() here; the datafile should be earlier in the
+                            # archive than the metadata, so that updating it (like, adding an annotation)
+                            # is fast.
+                            pass
+                        with tar_obj.extractfile(memb) as memb_fid:
+                            json_contents = memb_fid.read()
+
+                    elif memb.name.endswith(SIGMF_DATASET_EXT):
+                        data_offset_size = memb.offset_data, memb.size
+                        data_found = True
+
+                    else:
+                        print('A regular file', memb.name, 'was found but ignored in the archive')
+                else:
+                    print('A member of type', memb.type, 'and name', memb.name, 'was found but not handled, just FYI.')
 
-        self.sigmffile = SigMFFile(metadata=json_contents)
-        valid_md = self.sigmffile.validate()
+                if data_offset_size is not None and json_contents is not None:
+                    sigmffile = SigMFFile(metadata=json_contents)
+                    valid_md = sigmffile.validate()
 
-        self.sigmffile.set_data_file(self.name, data_buffer=archive_buffer, skip_checksum=skip_checksum, offset=data_offset_size[0],
-                                     size_bytes=data_offset_size[1], map_readonly=map_readonly)
+                    sigmffile.set_data_file(self.name, data_buffer=archive_buffer, skip_checksum=skip_checksum, offset=data_offset_size[0],
+                                                size_bytes=data_offset_size[1], map_readonly=map_readonly)
 
-        self.ndim = self.sigmffile.ndim
-        self.shape = self.sigmffile.shape
+                    self.ndim = sigmffile.ndim
+                    self.shape = sigmffile.shape
+                    self.sigmffiles.append(sigmffile)
+                    data_offset_size = None
+                    json_contents = None
+
 
-        tar_obj.close()
+            if not data_found:
+                raise SigMFFileError('No .sigmf-data file found in archive!')
+        finally:
+            if tar_obj: tar_obj.close()
 
     def __len__(self):
-        return self.sigmffile.__len__()
+        return len(self.sigmffiles)
 
     def __iter__(self):
-        return self.sigmffile.__iter__()
+        return self.sigmffiles.__iter__()
 
     def __getitem__(self, sli):
-        return self.sigmffile.__getitem__(sli)
+        return self.sigmffiles.__getitem__(sli)

sigmf/sigmffile.py

@@ -148,7 +148,7 @@ class SigMFFile(SigMFMetafile):
     ]
     VALID_KEYS = {GLOBAL_KEY: VALID_GLOBAL_KEYS, CAPTURE_KEY: VALID_CAPTURE_KEYS, ANNOTATION_KEY: VALID_ANNOTATION_KEYS}
 
-    def __init__(self, metadata=None, data_file=None, global_info=None, skip_checksum=False, map_readonly=True):
+    def __init__(self, metadata=None, data_file=None, global_info=None, skip_checksum=False, map_readonly=True, name=None):
         '''
         API for SigMF I/O
 
@@ -164,6 +164,13 @@ def __init__(self, metadata=None, data_file=None, global_info=None, skip_checksu
             When True will skip calculating hash on data_file (if present) to check against metadata.
         map_readonly: bool, default True
             Indicates whether assignments on the numpy.memmap are allowed.
+        name: Name used for directory and filenames if archived.
+                     For example, given `name=archive1`, then passing this
+                     sigmffile to SigMFArchive will add the following files
+                     to the archive:
+                        - archive1/
+                          - archive1.sigmf-meta
+                          - archive1.sigmf-data
         '''
         super(SigMFFile, self).__init__()
         self.data_file = None
@@ -183,6 +190,7 @@ def __init__(self, metadata=None, data_file=None, global_info=None, skip_checksu
             self.set_global_info(global_info)
         if data_file is not None:
             self.set_data_file(data_file, skip_checksum, map_readonly=map_readonly)
+        self.name = name
 
     def __len__(self):
         return self._memmap.shape[0]
@@ -212,6 +220,20 @@ def __getitem__(self, sli):
             else:
                 raise ValueError("unhandled ndim in SigMFFile.__getitem__(); this shouldn't happen")
         return a
+
+    def __eq__(self, other):
+        """Define equality between two `SigMFFile`s.
+
+        Rely on the `core:sha512` value in the metadata to decide whether
+        `data_file` is the same since the same sigmf archive could be extracted
+        twice to two different temp directories and the SigMFFiles should still
+        be equivalent.
+
+        """
+        if isinstance(other, SigMFFile):
+            return self._metadata == other._metadata
+
+        return False
 
     def _get_start_offset(self):
         """
@@ -511,13 +533,23 @@ def validate(self):
         version = self.get_global_field(self.VERSION_KEY)
         validate.validate(self._metadata, self.get_schema())
 
-    def archive(self, name=None, fileobj=None):
+    def archive(self, sigmffile_name=None, archive_name=None, fileobj=None):
         """Dump contents to SigMF archive format.
 
-        `name` and `fileobj` are passed to SigMFArchive and are defined there.
+        `sigmffile_name` determines the directory and filenames inside the archive. If
+        not specified, you must have set the instance variable `self.name`
+
+        `arhive_name` is passed to SigMFArchive `name` and `fileobj` is passed to
+        SigMFArchive `fileobj`.
 
         """
-        archive = SigMFArchive(self, name, fileobj)
+        if sigmffile_name is not None:
+            self.name = sigmffile_name
+
+        if archive_name is None:
+            archive_name = self.name
+
+        archive = SigMFArchive([self], archive_name, fileobj)
         return archive.path
 
     def tofile(self, file_path, pretty=True, toarchive=False, skip_validate=False):
@@ -891,13 +923,13 @@ def get_dataset_filename_from_metadata(meta_fn, metadata=None):
 
 
 def fromarchive(archive_path, dir=None):
-    """Extract an archive and return a SigMFFile.
+    """Extract an archive and return containing SigMFFiles.
 
     The `dir` parameter is no longer used as this function has been changed to
     access SigMF archives without extracting them.
     """
     from .archivereader import SigMFArchiveReader
-    return SigMFArchiveReader(archive_path).sigmffile
+    return SigMFArchiveReader(archive_path).sigmffiles
 
 
 def fromfile(filename, skip_checksum=False):