Draft: support clearing reports and CLIs in SBOM and in new command "bom downloadattachments"

ChangeLog.md

@@ -12,8 +12,11 @@
   But you still need this output when you want to create a Readme.
 * `project createbom` add purls, source and repository url from SW360 if available.
   If multiple purls are found, a warning is printed asking user to manually edit SBOM.
-* `project createbom` add SW360 source and binary attachments as external reference to SBOM.
+* `project createbom` add SW360 attachment info as external references to SBOM
+  (currently supported: source, binary, CLI, report).
+* `project createbom` will not add rejected attachments to SBOM
 * `project createbom` adds SW360 project name, version and description to SBOM.
+* new command `bom downloadattachments` to download CLI and report attachments
 
 ## 2.0.0 (2023-06-02)
 

capycli/bom/download_attachments.py

@@ -0,0 +1,176 @@
+# -------------------------------------------------------------------------------
+# Copyright (c) 2020-2023 Siemens
+# All Rights Reserved.
+# Author: gernot.hillier@siemens.com, thomas.graf@siemens.com
+#
+# SPDX-License-Identifier: MIT
+# -------------------------------------------------------------------------------
+
+import logging
+import os
+import sys
+from typing import Tuple
+
+import sw360.sw360_api
+from cyclonedx.model.bom import Bom
+
+import capycli.common.json_support
+import capycli.common.script_base
+from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomWriter
+from capycli.common.print import print_red, print_text, print_yellow
+from capycli.common.script_support import ScriptSupport
+from capycli.common.json_support import load_json_file
+from capycli.main.result_codes import ResultCode
+
+LOG = capycli.get_logger(__name__)
+
+
+class BomDownloadAttachments(capycli.common.script_base.ScriptBase):
+    """
+    Download SW360 attachments as specified in the SBOM.
+    """
+
+    def download_attachments(self, sbom: Bom, control_components: list, source_folder: str, bompath: str = None,
+                             attachment_types: Tuple[str] = ("COMPONENT_LICENSE_INFO_XML", "CLEARING_REPORT")) -> Bom:
+
+        for component in sbom.components:
+            item_name = ScriptSupport.get_full_name_from_component(component)
+            print_text("  " + item_name)
+
+            for ext_ref in component.external_references:
+                if not ext_ref.comment:
+                    continue
+                found = False
+                for at_type in attachment_types:
+                    if ext_ref.comment.startswith(CaPyCliBom.FILE_COMMENTS[at_type]):
+                        found = True
+                if not found:
+                    continue
+
+                release_id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
+                if not release_id:
+                    print_red("    No sw360Id for release!")
+                    continue
+                filename = os.path.join(source_folder, ext_ref.url)
+
+                details = [e for e in control_components
+                           if e["Sw360Id"] == release_id and (
+                               e.get("CliFile", "") == ext_ref.url
+                               or e.get("ReportFile", "") == ext_ref.url)]
+                if len(details) != 1:
+                    print_red("    ERROR: Found", len(details), "entries for attachment",
+                              ext_ref.url, "of", item_name, "in control file!")
+                    continue
+                attachment_id = details[0]["Sw360AttachmentId"]
+
+                print_text("    Downloading file " + filename)
+                try:
+                    self.client.download_release_attachment(filename, release_id, attachment_id)
+                    ext_ref.url = filename
+                    try:
+                        if bompath:
+                            CycloneDxSupport.have_relative_ext_ref_path(ext_ref, bompath)
+                    except ValueError:
+                        print_yellow("    SBOM file is not relative to source file " + ext_ref.url)
+
+                except sw360.sw360_api.SW360Error as swex:
+                    print_red("    Error getting", swex.url, swex.response)
+        return sbom
+
+    def run(self, args):
+        """Main method
+
+        @params:
+            args - command line arguments
+        """
+        if args.debug:
+            global LOG
+            LOG = capycli.get_logger(__name__)
+        else:
+            # suppress (debug) log output from requests and urllib
+            logging.getLogger("requests").setLevel(logging.WARNING)
+            logging.getLogger("urllib3").setLevel(logging.WARNING)
+            logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING)
+
+        print_text(
+            "\n" + capycli.APP_NAME + ", " + capycli.get_app_version() +
+            " - Download SW360 attachments as specified in the SBOM\n")
+
+        if args.help:
+            print("usage: capycli bom downloadattachments -i bom.json [-source <folder>]")
+            print("")
+            print("optional arguments:")
+            print("    -h, --help            show this help message and exit")
+            print("    -i INPUTFILE,         input SBOM to read from, e.g. created by \"project CreateBom\"")
+            print("    -ct CONTROLFILE,      control file to read from as created by \"project CreateBom\"")
+            print("    -source SOURCE        source folder or additional source file")
+            print("    -o OUTPUTFILE         output file to write to")
+            print("    -v                    be verbose")
+            return
+
+        if not args.inputfile:
+            print_red("No input file specified!")
+            sys.exit(ResultCode.RESULT_COMMAND_ERROR)
+
+        if not args.controlfile:
+            print_red("No control file specified!")
+            sys.exit(ResultCode.RESULT_COMMAND_ERROR)
+
+        if not os.path.isfile(args.inputfile):
+            print_red("Input file not found!")
+            sys.exit(ResultCode.RESULT_FILE_NOT_FOUND)
+
+        print_text("Loading SBOM file " + args.inputfile)
+        try:
+            bom = CaPyCliBom.read_sbom(args.inputfile)
+        except Exception as ex:
+            print_red("Error reading input SBOM file: " + repr(ex))
+            sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
+
+        if args.verbose:
+            print_text(" " + str(len(bom.components)) + "components read from SBOM file")
+
+        print_text("Loading control file " + args.controlfile)
+        try:
+            control = load_json_file(args.controlfile)
+        except Exception as ex:
+            print_red("JSON error reading control file: " + repr(ex))
+            sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
+        if "Components" not in control:
+            print_red("missing Components in control file")
+            sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
+
+        source_folder = "./"
+        if args.source:
+            source_folder = args.source
+        if (not source_folder) or (not os.path.isdir(source_folder)):
+            print_red("Target source code folder does not exist!")
+            sys.exit(ResultCode.RESULT_COMMAND_ERROR)
+
+        if args.sw360_token and args.oauth2:
+            self.analyze_token(args.sw360_token)
+
+        print_text("  Checking access to SW360...")
+        if not self.login(token=args.sw360_token, url=args.sw360_url, oauth2=args.oauth2):
+            print_red("ERROR: login failed!")
+            sys.exit(ResultCode.RESULT_AUTH_ERROR)
+
+        print_text("Downloading source files to folder " + source_folder + " ...")
+
+        self.download_attachments(bom, control["Components"], source_folder, os.path.dirname(args.outputfile))
+
+        if args.outputfile:
+            print_text("Updating path information")
+            self.update_local_path(bom, args.outputfile)
+
+            print_text("Writing updated SBOM to " + args.outputfile)
+            try:
+                SbomWriter.write_to_json(bom, args.outputfile, True)
+            except Exception as ex:
+                print_red("Error writing updated SBOM file: " + repr(ex))
+                sys.exit(ResultCode.RESULT_ERROR_WRITING_BOM)
+
+            if args.verbose:
+                print_text(" " + str(len(bom.components)) + " components written to SBOM file")
+
+        print("\n")

capycli/bom/handle_bom.py

@@ -15,6 +15,7 @@
 import capycli.bom.create_components
 import capycli.bom.diff_bom
 import capycli.bom.download_sources
+import capycli.bom.download_attachments
 import capycli.bom.filter_bom
 import capycli.bom.findsources
 import capycli.bom.map_bom
@@ -35,19 +36,20 @@ def run_bom_command(args) -> None:
 
         # display `bom` related help
         print("bom               bill of material (BOM) specific sub-commands")
-        print("    Show              display contents of a BOM")
-        print("    Convert           Convert SBOM formats")
-        print("    Filter            apply filter file to a BOM")
-        print("    Check             check that all releases in the BOM exist on target SW360 instance")
-        print("    CheckItemStatus   show additional information about BOM items on SW360")
-        print("    Map               map a given BOM to data on SW360")
-        print("    CreateReleases    create new releases for existing components on SW360")
-        print("    CreateComponents  create new components and releases on SW360 (use with care!)")
-        print("    DownloadSources   download source files from the URL specified in the SBOM")
-        print("    Granularity       check a bill of material for potential component granularity issues")
-        print("    Diff              compare two bills of material.")
-        print("    Merge             merge two bills of material.")
-        print("    Findsources       determine the source code for SBOM items.")
+        print("    Show                display contents of a BOM")
+        print("    Convert             Convert SBOM formats")
+        print("    Filter              apply filter file to a BOM")
+        print("    Check               check that all releases in the BOM exist on target SW360 instance")
+        print("    CheckItemStatus     show additional information about BOM items on SW360")
+        print("    Map                 map a given BOM to data on SW360")
+        print("    CreateReleases      create new releases for existing components on SW360")
+        print("    CreateComponents    create new components and releases on SW360 (use with care!)")
+        print("    DownloadAttachments download SW360 attachments as specified in the SBOM")
+        print("    DownloadSources     download source files from the URL specified in the SBOM")
+        print("    Granularity         check a bill of material for potential component granularity issues")
+        print("    Diff                compare two bills of material.")
+        print("    Merge               merge two bills of material.")
+        print("    Findsources         determine the source code for SBOM items.")
         return
 
     subcommand = args.command[1].lower()
@@ -100,6 +102,12 @@ def run_bom_command(args) -> None:
         app.run(args)
         return
 
+    if subcommand == "downloadattachments":
+        """Download attachments from SW360 as specified in the SBOM."""
+        app = capycli.bom.download_attachments.BomDownloadAttachments()
+        app.run(args)
+        return
+
     if subcommand == "granularity":
         """Check the granularity of the releases in the SBOM."""
         app = capycli.bom.check_granularity.CheckGranularity()

capycli/common/capycli_bom_support.py

@@ -632,6 +632,20 @@ class CaPyCliBom():
     SOURCE_FILE_COMMENT = "source archive (local copy)"
     BINARY_URL_COMMENT = "binary (download location)"
     BINARY_FILE_COMMENT = "relativePath"
+    # machine-readable XML description of licensing situation of a component
+    # see https://github.com/sw360/clipython for more information
+    CLI_FILE_COMMENT = "component license information (local copy)"
+    # human-readable description of licensing situation and obligations
+    CRT_FILE_COMMENT = "clearing report (local copy)"
+
+    FILE_COMMENTS = {
+        "SOURCE": SOURCE_FILE_COMMENT,
+        "SOURCE_SELF": SOURCE_FILE_COMMENT,
+        "BINARY": BINARY_FILE_COMMENT,
+        "BINARY_SELF": BINARY_FILE_COMMENT,
+        "COMPONENT_LICENSE_INFO_XML": CLI_FILE_COMMENT,
+        "CLEARING_REPORT": CRT_FILE_COMMENT
+    }
 
     @classmethod
     def read_sbom(cls, inputfile: str) -> Bom:

capycli/common/script_base.py

@@ -138,6 +138,11 @@ def release_web_url(self, release_id) -> str:
         return (self.sw360_url + "group/guest/components/-/component/release/detailRelease/"
                 + release_id)
 
+    def attachment_api_url(self, release_id, attachment_id) -> str:
+        """Returns the REST API URL for an attachment."""
+        return (self.sw360_url + "resource/api/releases/" + release_id
+                + "/attachments/" + attachment_id)
+
     def find_project(self, name: str, version: str, show_results: bool = False) -> str:
         """Find the project with the matching name and version on SW360"""
         print_text("  Searching for project...")

capycli/main/options.py

@@ -28,47 +28,49 @@ def __init__(self):
         custom_usage = "CaPyCli command subcommand [options]"
         command_help = """Commands and Sub-Commands
     getdependencies     dependency detection specific commands
-        Nuget             determine dependencies for a .Net/Nuget project
-        Python            determine dependencies for a Python project
-        Javascript        determine dependencies for a JavaScript project
-        MavenPom          determine dependencies for a Java/Maven project using the pom.xml file
-        MavenList         determine dependencies for a Java/Maven project using a Maven command
+        Nuget               determine dependencies for a .Net/Nuget project
+        Python              determine dependencies for a Python project
+        Javascript          determine dependencies for a JavaScript project
+        MavenPom            determine dependencies for a Java/Maven project using the pom.xml file
+        MavenList           determine dependencies for a Java/Maven project using a Maven command
 
     bom                 bill of material (BOM) specific commands
-        Show              display contents of a SBOM
-        Convert           convert SBOM formats
-        Filter            apply filter file to a SBOM
-        Check             check that all releases in the SBOM exist on target SW360 instance
-        CheckItemStatus   show additional information about SBOM items on SW360
-        Map               map a given SBOM to data on SW360
-        CreateReleases    create new releases for existing components on SW360
-        CreateComponents  create new components and releases on SW360 (use with care!)
-        DownloadSources   download source files from the URL specified in the SBOM
-        Granularity       check a bill of material for potential component granularity issues
-        Diff              compare two bills of material.
-        Merge             merge two bills of material.
-        Findsources       determine the source code for SBOM items.
+        Show                display contents of a SBOM
+        Convert             convert SBOM formats
+        Filter              apply filter file to a SBOM
+        Check               check that all releases in the SBOM exist on target SW360 instance
+        CheckItemStatus     show additional information about SBOM items on SW360
+        Map                 map a given SBOM to data on SW360
+        CreateReleases      create new releases for existing components on SW360
+        CreateComponents    create new components and releases on SW360 (use with care!)
+        DownloadSources     download source files from the URL specified in the SBOM
+        DownloadAttachments download SW360 attachments as specified in the SBOM
+
+        Granularity         check a bill of material for potential component granularity issues
+        Diff                compare two bills of material.
+        Merge               merge two bills of material.
+        Findsources         determine the source code for SBOM items.
 
     mapping
-        ToHtml            create a HTML page showing the mapping result
-        ToXlsx            create an Excel sheet showing the mapping result
+        ToHtml              create a HTML page showing the mapping result
+        ToXlsx              create an Excel sheet showing the mapping result
 
     moverview
-        ToHtml            create a HTML page showing the mapping result overview
-        ToXlsx            create an Excel sheet showing the mapping result overview
+        ToHtml              create a HTML page showing the mapping result overview
+        ToXlsx              create an Excel sheet showing the mapping result overview
 
     project
-        Find              find a project by name
-        Prerequisites     checks whether all prerequisites for a successfull
-                          software clearing are fulfilled
-        Show              show project details
-        Licenses          show licenses of all cleared compponents
-        Create            create or update a project on SW360
-        Update            update an exiting project, preserving linked releases
-        GetLicenseInfo    get license info of all project components
-        CreateReadme      create a Readme_OSS
-        Vulnerabilities   show security vulnerabilities of a project
-        ECC               show export control status of a project
+        Find                find a project by name
+        Prerequisites       checks whether all prerequisites for a successfull
+                            software clearing are fulfilled
+        Show                show project details
+        Licenses            show licenses of all cleared compponents
+        Create              create or update a project on SW360
+        Update              update an exiting project, preserving linked releases
+        GetLicenseInfo      get license info of all project components
+        CreateReadme        create a Readme_OSS
+        Vulnerabilities     show security vulnerabilities of a project
+        ECC                 show export control status of a project
 
     Note that each command has also its own help display, i.e. if you enter
     `capycli project vulnerabilities -h` you will get a help that only shows the options
@@ -208,6 +210,13 @@ def register_options(self):
             help="create an mapping overview JSON file",
         )
 
+        self.parser.add_argument(
+            "-ct",
+            "--controlfile",
+            dest="controlfile",
+            help="control file for \"bom DownloadAttachments\" and \"project CreateReadme\"",
+        )
+
         self.parser.add_argument(
             "-mr",
             "--mapresult",