First shot at adding allow list

Dockerfile

@@ -2,7 +2,7 @@ FROM singularityware/singularity:v3.2.1-slim as base
 
 ################################################################################
 #
-# Copyright (C) 2019 Vanessa Sochat.
+# Copyright (C) 2019-2022 Vanessa Sochat.
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License as published by

README.md

@@ -45,8 +45,8 @@ This experiment is based on early discussion in [this thread](https://github.com
 You'll need to first clone the repository:
 
 ```bash
-git clone https://github.com/singularityhub/stools
-cd stools
+$ git clone https://github.com/singularityhub/stools
+$ cd stools
 ```
 
 ### Build Containers
@@ -100,6 +100,15 @@ http://people.ubuntu.com/~ubuntu-security/cve/CVE-2016-9843
 The crc32_big function in crc32.c in zlib 1.2.8 might allow context-dependent attackers to have unspecified impact via vectors involving big-endian CRC calculation.
 ```
 
+To include an allowlist, e.g., [allowlist.yaml](allowlist.yaml) you can do:
+
+```bash
+$ docker exec -it clair-scanner sclair --allowlist allowlist.yaml singularity-images_latest.sif
+```
+
+You'll notice the previous last entry is different, because it was removed. Currently, we just match CVE names (and don't do
+further parsing) but this can be tweaked if desired.
+
 ### Save a Report
 
 However, if you want to save a report to file (json), you can add the `--report` argument

allowlist.yaml

@@ -0,0 +1,10 @@
+generalallowlist: # Approve CVE for any image
+  CVE-2017-6055: XML
+  CVE-2017-5586: OpenText
+  CVE-2019-13627: ""
+images:
+  ubuntu: # Approve CVE only for ubuntu image, regardles of the version. If it is a private registry with a custom port registry:777/ubuntu:tag this won't work due to a bug.
+    CVE-2017-5230: Java
+    CVE-2017-5230: XSX
+  alpine:
+    CVE-2017-3261: SE

requirements.txt

@@ -1,2 +1,4 @@
 aiohttp==3.7.4
 requests>=2.20.0
+pyaml
+IPython

stools/clair/__init__.py

@@ -2,7 +2,7 @@
 
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by
@@ -64,6 +64,12 @@ def get_parser():
         help="save Clair reports to chosen directory",
     )
 
+    parser.add_argument(
+        "--allowlist",
+        default=None,
+        help="include a yaml allow list (example in stools repository)",
+    )
+
     parser.add_argument(
         "--no-print",
         dest="no_print",
@@ -147,6 +153,10 @@ def help(retval=0):
     # Local Server
     webroot = "/var/www/images"
 
+    # If we have an allowlist, make sure it exists
+    if args.allowlist and not os.path.exists(args.allowlist):
+        sys.exit("%s does not exist." % args.allowlist)
+
     # Start the server and serve static files from root
 
     if args.server:
@@ -183,7 +193,7 @@ def help(retval=0):
 
         # 4. Generate report
         print("3. Generating report!")
-        report = clair.report(os.path.basename(image))
+        report = clair.report(os.path.basename(image), args.allowlist)
         if args.report_location:
             fpath = os.path.join(
                 args.report_location,

stools/clair/api.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by
@@ -19,11 +19,12 @@
 
 
 import requests
+import yaml
 import os
 import sys
 
 
-class Clair(object):
+class Clair:
     """the ClairOS security scanner to scan Docker layers"""
 
     def __init__(self, host, port, api_version="v1"):
@@ -50,22 +51,72 @@ def scan(self, targz_url, name):
             print("Error creating %s at %s" % (data["Path"], url))
             sys.exit(1)
 
-    def report(self, name):
+    def report(self, name, allowlist=None):
         """generate a report for an image of interest. The name should
         correspond to the same name used when adding the layer...
-
-        Parameters
-        ==========
         """
-
         url = os.path.join(self.url, "layers", name)
         response = requests.get(url, params={"features": True, "vulnerabilities": True})
         if response.status_code == 200:
-            return response.json()
+            hits = response.json()
+            if allowlist:
+                hits = self.apply_allowlist(allowlist, hits)
+            return hits
         else:
             print("Error with %s" % url)
             sys.exit(1)
 
+    def apply_allowlist(self, filename, hits):
+        """
+        Apply an allowlist, meaning a yaml of vulnerabilities to ignore / remove.
+        """
+        with open(filename, "r") as fd:
+            allow = yaml.load(fd.read(), Loader=yaml.SafeLoader)
+
+        # No results?
+        if "Layer" not in hits:
+            return hits
+
+        # General allowlist
+        general = set(allow.get("generalallowlist", {}))
+
+        for image, cves in allow["images"].items():
+
+            # Just match based on list of names (we might want to extend this)
+            cves = set(cves)
+            if not hits["Layer"]["NamespaceName"].startswith(image):
+                continue
+
+            # Don't continue if no features
+            if not hits["Layer"].get("Features", []):
+                continue
+
+            # Keep list of updated features
+            updated = []
+            for feature in hits["Layer"].get("Features", []):
+                if "Vulnerabilities" not in feature:
+                    updated.append(feature)
+                    continue
+
+                # Keep record of vulns and allowed
+                vulns = []
+                allowed = feature.get("Allowed", [])
+
+                # For a vulnerability, if it's not in allow list, add
+                for vuln in feature["Vulnerabilities"]:
+                    if vuln["Name"] in cves or vuln["Name"] in general:
+                        print("Allowlist: skipping %s" % vuln["Name"])
+                        allowed.append(vuln)
+                        continue
+                    vulns.append(vuln)
+
+                feature["Vulnerabilities"] = vulns
+                feature["Allowed"] = allowed
+                updated.append(feature)
+
+            hits["Layer"]["Features"] = updated
+        return hits
+
     def ping(self):
         """ping serves as a health check. If healthy, will return True.
         We do this because the user is starting Clair as
@@ -93,17 +144,27 @@ def ping(self):
     def print(self, report):
         """print the report items"""
 
-        if "Features" in report["Layer"]:
-            items = report["Layer"]["Features"]
-
-            for item in items:
-                if "Vulnerabilities" in item:
-                    print("%s - %s" % (item["Name"], item["Version"]))
-                    print("-" * len(item["Name"] + " - " + item["Version"]))
-                    for v in item["Vulnerabilities"]:
-                        print(v["Name"] + " (" + v["Severity"] + ")")
-                        print(v["Link"])
-                        print(v["Description"])
-                        print("\n")
-        else:
+        features = report["Layer"].get("Features", [])
+        if not features:
             print("%s does not have any vulnerabilities!" % report["Layer"]["Name"])
+            return
+
+        for item in features:
+
+            # Print a header given any items
+            if "Allowed" in item or "Vulnerabilities" in item:
+                print("%s - %s" % (item["Name"], item["Version"]))
+                print("-" * len(item["Name"] + " - " + item["Version"]))
+
+            if "Allowed" in item:
+                for v in item["Allowed"]:
+                    print(v["Name"] + " (" + v["Severity"] + ")")
+                    print(v["Link"])
+                    print(v["Description"])
+                    print("\n")
+            if "Vulnerabilities" in item:
+                for v in item["Vulnerabilities"]:
+                    print(v["Name"] + " (" + v["Severity"] + ") unapproved ")
+                    print(v["Link"])
+                    print(v["Description"])
+                    print("\n")

stools/clair/image.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by

stools/clair/server/main.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by

stools/clair/server/routes.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by

stools/clair/server/views.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by

stools/utils.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by

stools/version.py

@@ -1,6 +1,6 @@
 """
 
-Copyright (C) 2018-2021 Vanessa Sochat.
+Copyright (C) 2018-2022 Vanessa Sochat.
 
 This program is free software: you can redistribute it and/or modify it
 under the terms of the GNU Affero General Public License as published by