Skip to content

Commit

Permalink
feat(bom downloadAttachments): read attachment id from control file
Browse files Browse the repository at this point in the history
  • Loading branch information
@gernot-h
gernot-h committed Aug 22, 2023
1 parent ddfa863 commit 49f551a
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 138 deletions.
43 changes: 28 additions & 15 deletions capycli/bom/download_attachments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport, SbomWriter
from capycli.common.print import print_red, print_text, print_yellow
from capycli.common.script_support import ScriptSupport
from capycli.common.json_support import load_json_file
from capycli.main.result_codes import ResultCode

LOG = capycli.get_logger(__name__)
Expand All @@ -29,7 +30,7 @@ class BomDownloadAttachments(capycli.common.script_base.ScriptBase):
Download SW360 attachments as specified in the SBOM.
"""

def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = None,
def download_attachments(self, sbom: Bom, control_components: list, source_folder: str, bompath: str = None,
attachment_types: Tuple[str] = ("COMPONENT_LICENSE_INFO_XML", "CLEARING_REPORT")) -> Bom:

for component in sbom.components:
Expand All @@ -46,27 +47,24 @@ def download_attachments(self, sbom: Bom, source_folder: str, bompath: str = Non
if not found:
continue

attachment_id = ext_ref.comment.split(", sw360Id: ")
if len(attachment_id) != 2:
print_red(" No sw360Id for attachment!")
continue
attachment_id = attachment_id[1]

release_id = CycloneDxSupport.get_property_value(component, CycloneDxSupport.CDX_PROP_SW360ID)
if not release_id:
print_red(" No sw360Id for release!")
continue
print(" ", ext_ref.url, release_id, attachment_id)
filename = os.path.join(source_folder, ext_ref.url)

details = [e for e in control_components
if e["Sw360Id"] == release_id and (
e.get("CliFile", "") == ext_ref.url
or e.get("ReportFile", "") == ext_ref.url)]
if len(details) != 1:
print_red(" ERROR: Found", len(details), "entries for attachment",
ext_ref.url, "of", item_name, "in control file!")
continue
attachment_id = details[0]["Sw360AttachmentId"]

print_text(" Downloading file " + filename)
try:
at_info = self.client.get_attachment(attachment_id)
at_info = {k: v for k, v in at_info.items()
if k.startswith("check")
or k.startswith("created")}
print(at_info)

self.client.download_release_attachment(filename, release_id, attachment_id)
ext_ref.url = filename
try:
Expand Down Expand Up @@ -104,6 +102,7 @@ def run(self, args):
print("optional arguments:")
print(" -h, --help show this help message and exit")
print(" -i INPUTFILE, input SBOM to read from, e.g. created by \"project CreateBom\"")
print(" -ct CONTROLFILE, control file to read from as created by \"project CreateBom\"")
print(" -source SOURCE source folder or additional source file")
print(" -o OUTPUTFILE output file to write to")
print(" -v be verbose")
Expand All @@ -113,6 +112,10 @@ def run(self, args):
print_red("No input file specified!")
sys.exit(ResultCode.RESULT_COMMAND_ERROR)

if not args.controlfile:
print_red("No control file specified!")
sys.exit(ResultCode.RESULT_COMMAND_ERROR)

if not os.path.isfile(args.inputfile):
print_red("Input file not found!")
sys.exit(ResultCode.RESULT_FILE_NOT_FOUND)
Expand All @@ -127,6 +130,16 @@ def run(self, args):
if args.verbose:
print_text(" " + str(len(bom.components)) + "components read from SBOM file")

print_text("Loading control file " + args.controlfile)
try:
control = load_json_file(args.controlfile)
except Exception as ex:
print_red("JSON error reading control file: " + repr(ex))
sys.exit(ResultCode.RESULT_ERROR_READING_BOM)
if "Components" not in control:
print_red("missing Components in control file")
sys.exit(ResultCode.RESULT_ERROR_READING_BOM)

source_folder = "./"
if args.source:
source_folder = args.source
Expand All @@ -144,7 +157,7 @@ def run(self, args):

print_text("Downloading source files to folder " + source_folder + " ...")

self.download_attachments(bom, source_folder, os.path.dirname(args.outputfile))
self.download_attachments(bom, control["Components"], source_folder, os.path.dirname(args.outputfile))

if args.outputfile:
print_text("Updating path information")
Expand Down
24 changes: 24 additions & 0 deletions tests/fixtures/sbom_for_download-control.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"ProjectName": "CaPyCLI, 2.0.0-dev1",
"Components": [
{
"ComponentName": "certifi 2022.12.7",
"Sw360Id": "ae8c7ed",
"Sw360AttachmentId": "794446",
"CreatedBy": "user1@siemens.com",
"CreatedTeam": "AA",
"CreatedOn": "2020-10-23",
"CheckStatus": "ACCEPTED",
"CheckedBy": "user2@siemens.com",
"CheckedTeam": "BB",
"CheckedOn": "2020-10-30",
"CliFile": "CLIXML_certifi-2022.12.7.xml"
},
{
"ComponentName": "certifi 2022.12.7",
"Sw360Id": "ae8c7ed",
"Sw360AttachmentId": "63b368",
"ReportFile": "certifi-2022.12.7_clearing_report.docx"
}
]
}
152 changes: 29 additions & 123 deletions tests/test_bom_downloadattachments.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,16 @@

import responses

from capycli.common.capycli_bom_support import CaPyCliBom, CycloneDxSupport
from capycli.common.capycli_bom_support import CaPyCliBom
from capycli.common.json_support import load_json_file
from capycli.bom.download_attachments import BomDownloadAttachments
from capycli.main.result_codes import ResultCode
from cyclonedx.model import ExternalReferenceType, HashAlgorithm
from tests.test_base import AppArguments, TestBase


class TestBomDownloadAttachments(TestBase):
INPUTFILE = "sbom_for_download.json"
CONTROLFILE = "sbom_for_download-control.json"
INPUTERROR = "plaintext.txt"
OUTPUTFILE = "output.json"

Expand Down Expand Up @@ -69,6 +70,8 @@ def test_file_not_found(self) -> None:
args.command.append("bom")
args.command.append("downloadattachments")
args.inputfile = "DOESNOTEXIST"
args.controlfile = os.path.join(os.path.dirname(__file__),
"fixtures", TestBomDownloadAttachments.CONTROLFILE)

sut.run(args)
self.assertTrue(False, "Failed to report missing file")
Expand All @@ -85,6 +88,8 @@ def test_error_loading_file(self) -> None:
args.command.append("bom")
args.command.append("downloadattachments")
args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTERROR)
args.controlfile = os.path.join(os.path.dirname(__file__),
"fixtures", TestBomDownloadAttachments.CONTROLFILE)

sut.run(args)
self.assertTrue(False, "Failed to report invalid file")
Expand All @@ -103,6 +108,8 @@ def test_source_folder_does_not_exist(self) -> None:
args.command.append("downloadattachments")

args.inputfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
args.controlfile = os.path.join(os.path.dirname(__file__),
"fixtures", TestBomDownloadAttachments.CONTROLFILE)
args.source = "XXX"

sut.run(args)
Expand All @@ -113,32 +120,10 @@ def test_source_folder_does_not_exist(self) -> None:
@responses.activate
def test_simple_bom(self) -> None:
bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
bom = CaPyCliBom.read_sbom(bom)
controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)

# attachment info - CLI
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/attachments/794446",
body="""
{
"filename": "CLIXML_certifi-2022.12.7.xml",
"sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
"attachmentType": "COMPONENT_LICENSE_INFO_XML",
"createdBy": "thomas.graf@siemens.com",
"createdTeam": "AA",
"createdComment": "comment1",
"createdOn": "2020-10-08",
"checkStatus": "NOTCHECKED",
"_links": {
"self": {
"href": "https://my.server.com/resource/api/attachments/794446"
}
}
}""",
status=200,
content_type="application/json",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)
bom = CaPyCliBom.read_sbom(bom)
controlfile = load_json_file(controlfile)

# get attachment - CLI
cli_file = self.get_cli_file_mit()
Expand All @@ -150,35 +135,6 @@ def test_simple_bom(self) -> None:
content_type="application/text",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)

# attachment info - report
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/attachments/63b368",
body="""
{
"filename": "certifi-2022.12.7_clearing_report.docx",
"sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
"attachmentType": "CLEARING_REPORT",
"createdBy": "gernot.hillier@siemens.com",
"createdTeam": "BB",
"createdComment": "comment3",
"createdOn": "2020-10-08",
"checkedBy": "thomas.graf@siemens.com",
"checkedOn" : "2021-01-18",
"checkedComment": "comment4",
"checkStatus": "ACCEPTED",
"_links": {
"self": {
"href": "https://my.server.com/resource/api/attachments/63b368"
}
}
}""",
status=200,
content_type="application/json",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)

# get attachment - report
responses.add(
method=responses.GET,
Expand All @@ -191,7 +147,7 @@ def test_simple_bom(self) -> None:

with tempfile.TemporaryDirectory() as tmpdirname:
try:
bom = self.app.download_attachments(bom, tmpdirname)
bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname)
resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
self.assertEqual(bom.components[0].external_references[5].url, resultfile)
self.assertTrue(os.path.isfile(resultfile), "CLI file missing")
Expand All @@ -211,25 +167,8 @@ def test_simple_bom_relpath(self) -> None:
bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
bom = CaPyCliBom.read_sbom(bom)

# attachment info - CLI
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/attachments/794446",
body="""
{
"filename": "CLIXML_certifi-2022.12.7.xml",
"sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
"attachmentType": "COMPONENT_LICENSE_INFO_XML",
"_links": {
"self": {
"href": "https://my.server.com/resource/api/attachments/794446"
}
}
}""",
status=200,
content_type="application/json",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)
controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)
controlfile = load_json_file(controlfile)

# get attachment - CLI
cli_file = self.get_cli_file_mit()
Expand All @@ -244,7 +183,8 @@ def test_simple_bom_relpath(self) -> None:

with tempfile.TemporaryDirectory() as tmpdirname:
try:
bom = self.app.download_attachments(bom, tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",))
bom = self.app.download_attachments(bom, controlfile["Components"],
tmpdirname, tmpdirname, ("COMPONENT_LICENSE_INFO_XML",))
resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
self.assertEqual(bom.components[0].external_references[5].url,
"file://CLIXML_certifi-2022.12.7.xml")
Expand All @@ -263,59 +203,29 @@ def test_simple_bom_download_errors(self) -> None:
bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
bom = CaPyCliBom.read_sbom(bom)

# attachment info - CLI, ok
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/attachments/794446",
body="""
{
"filename": "CLIXML_certifi-2022.12.7.xml",
"sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
"attachmentType": "COMPONENT_LICENSE_INFO_XML",
"_links": {
"self": {
"href": "https://my.server.com/resource/api/attachments/794446"
}
}
}""",
status=200,
content_type="application/json",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)
controlfile = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.CONTROLFILE)
controlfile = load_json_file(controlfile)

# get attachment - CLI, error
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/794446",
body="cli_file",
status=500,
content_type="application/text",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)

# attachment info - report, error
# get attachment - CLI, error
responses.add(
method=responses.GET,
url=self.MYURL + "resource/api/attachments/63b368",
body="""
{
"filename": "certifi-2022.12.7_clearing_report.docx",
"sha1": "3cd24769fa3da4af74d0118433619a130da091b0",
"attachmentType": "CLEARING_REPORT",
"_links": {
"self": {
"href": "https://my.server.com/resource/api/attachments/63b368"
}
}
}""",
status=404,
content_type="application/json",
url=self.MYURL + "resource/api/releases/ae8c7ed/attachments/63b368",
status=403,
content_type="application/text",
adding_headers={"Authorization": "Token " + self.MYTOKEN},
)

with tempfile.TemporaryDirectory() as tmpdirname:
try:
bom = self.app.download_attachments(bom, tmpdirname)
bom = self.app.download_attachments(bom, controlfile["Components"], tmpdirname)
resultfile = os.path.join(tmpdirname, "CLIXML_certifi-2022.12.7.xml")
self.assertFalse(os.path.isfile(resultfile), "CLI created despite HTTP 500")

Expand All @@ -335,8 +245,8 @@ def test_simple_bom_no_release_id(self) -> None:
bom.components[0].properties = []
with tempfile.TemporaryDirectory() as tmpdirname:
try:
err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname)
assert "No sw360Id for release" in err
err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname)
self.assertIn("No sw360Id for release", err)

return
except Exception as e: # noqa
Expand All @@ -346,18 +256,14 @@ def test_simple_bom_no_release_id(self) -> None:
self.assertTrue(False, "Error: we must never arrive here")

@responses.activate
def test_simple_bom_no_attachment_id(self) -> None:
def test_simple_bom_no_ctrl_file_entry(self) -> None:
bom = os.path.join(os.path.dirname(__file__), "fixtures", TestBomDownloadAttachments.INPUTFILE)
bom = CaPyCliBom.read_sbom(bom)
bom.components[0].external_references = []
CycloneDxSupport.set_ext_ref(bom.components[0], ExternalReferenceType.OTHER,
CaPyCliBom.CLI_FILE_COMMENT, "CLIXML_foo.xml",
HashAlgorithm.SHA_1, "123")

with tempfile.TemporaryDirectory() as tmpdirname:
try:
err = self.capture_stdout(self.app.download_attachments, bom, tmpdirname)
assert "No sw360Id for attachment" in err
err = self.capture_stdout(self.app.download_attachments, bom, [], tmpdirname)
assert "Found 0 entries for attachment CLIXML_certifi-2022.12.7.xml" in err

return
except Exception as e: # noqa
Expand Down

0 comments on commit 49f551a

Please sign in to comment.