Merge pull request #60 from PidgeyL/master

Development + Bugfixes

bin/db_mgmt.py

@@ -71,10 +71,11 @@ def __init__(self):
 
     def startElement(self, name, attrs):
         if name == 'entry':
-            self.cves.append({'id': attrs.get('id'), 'references': [], 'vulnerable_configuration': []})
+            self.cves.append({'id': attrs.get('id'), 'references': [], 'vulnerable_configuration': [], 'vulnerable_configuration_old':[]})
             self.ref = attrs.get('id')
         elif name == 'cpe-lang:fact-ref':
             self.cves[-1]['vulnerable_configuration'].append(toStringFormattedCPE(attrs.get('name')))
+            self.cves[-1]['vulnerable_configuration_old'].append(attrs.get('name'))
         elif name == 'cvss:score':
             self.inCVSSElem = 1
             self.CVSS = ""
@@ -217,7 +218,9 @@ def endElement(self, name):
                     item['cvss'] = defaultvalue['cvss']
                 if 'cwe' not in item:
                     item['cwe'] = defaultvalue['cwe']
-                collection.update({'id': item['id']}, {"$set": {'cvss': item['cvss'], 'summary': item['summary'], 'references': item['references'], 'cwe': item['cwe'], 'vulnerable_configuration': item['vulnerable_configuration'], 'last-modified': item['Modified']}})
+                collection.update({'id': item['id']}, {'$set': {'cvss': item['cvss'], 'summary': item['summary'], 'references': item['references'],
+                                                       'cwe': item['cwe'], 'vulnerable_configuration': item['vulnerable_configuration'],
+                                                       'vulnerable_configuration_cpe_2_2':item['vulnerable_configuration_old'], 'last-modified': item['Modified']}})
             else:
                 collection.insert(item)
         # get the 'recent' file
@@ -240,7 +243,9 @@ def endElement(self, name):
                     item['cvss'] = float(item['cvss'])
                 if 'cwe' not in item:
                     item['cwe'] = defaultvalue['cwe']
-                collection.update({'id': item['id']}, {"$set": {'cvss': item['cvss'], 'summary': item['summary'], 'references': item['references'], 'cwe': item['cwe'], 'vulnerable_configuration': item['vulnerable_configuration'], 'last-modified': item['Modified']}})
+                collection.update({'id': item['id']}, {'$set': {'cvss': item['cvss'], 'summary': item['summary'], 'references': item['references'],
+                                                       'cwe': item['cwe'], 'vulnerable_configuration': item['vulnerable_configuration'],
+                                                       'vulnerable_configuration_cpe_2_2':item['vulnerable_configuration_old'], 'last-modified': item['Modified']}})
             # if not, create it.
             else:
                 collection.insert(item)

bin/db_mgmt_cpe_dictionary.py

@@ -91,11 +91,12 @@ def endElement(self, name):
 bulk = cpe.initialize_ordered_bulk_op()
 for x in progressbar(ch.cpe):
      name = toStringFormattedCPE(x['name'])
+     oldCPE = x['name']
      title = x['title'][0]
      if x['references']:
-         bulk.find({'id': name}).upsert().update({"$set":{'title': title, 'references': x['references']}})
+         bulk.find({'id': name}).upsert().update({"$set":{'title': title, 'cpe_2_2':oldCPE, 'references': x['references']}})
      else:
-         bulk.find({'id': name}).upsert().update({"$set":{'title': title}})
+         bulk.find({'id': name}).upsert().update({"$set":{'title': title, 'cpe_2_2':oldCPE}})
 bulk.execute()
 
 #update database info after successful program-run

lib/Toolkit.py

@@ -11,7 +11,17 @@
 # Note of warning: CPEs like cpe:/o:microsoft:windows_8:-:-:x64 are given to us by Mitre
 #  x64 will be parsed as Edition in this case, not Architecture
 def toStringFormattedCPE(cpe,autofill=False):
-
+    if not cpe.startswith('cpe:2.3:'):
+      cpe=cpe.replace('cpe:/','cpe:2.3:')
+      cpe=cpe.replace('::',':-:')
+      cpe=cpe.replace('~-','~')
+      cpe=cpe.replace('~',':-:')
+      cpe=cpe.replace('::',':')
+      cpe=cpe.strip(':-')
+    if autofill:
+      e=cpe.split(':')
+      for x in range(0,13-len(e)):
+        cpe+=':-'
     return cpe
 
 def impactScore(cve):

test/test.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3.3
+# -*- coding: utf-8 -*-
+#
+# Tests for functions
+#
+# Software is free software released under the "Modified BSD license"
+#
+# Copyright (c) 2015 	Pieter-Jan Moreels - pieterjan.moreels@gmail.com
+
+# Imports
+import os
+import sys
+runPath = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(os.path.join(runPath, ".."))
+
+from lib.Toolkit import toStringFormattedCPE
+
+def resultOf(original, result, expected):
+  test={'in':original,'out':result,'expect':expected}
+  test['passed'] = True if result==expected else False
+  return test
+
+def printResults(test, results):
+  l = [x['passed'] for x in results]
+  if False in l:
+    print('[x] %s failed!'%test)
+    for x in [x for x in results if x['passed']==False]:
+      print('    in:       %s'%x['in'])
+      print('    out:      %s'%x['out'])
+      print('    expected: %s'%x['expect'])
+  else:
+    print('[ ] %s passed'%test)
+
+extend=[{'in':'cpe:/o:microsoft:windows_server_2008::sp2:itanium',                         'expect':'cpe:2.3:o:microsoft:windows_server_2008:-:sp2:itanium:-:-:-:-:-'},
+        {'in':'cpe:/a:activehelper:activehelper_livehelp_live_chat:2.7.4::~~~wordpress~~', 'expect':'cpe:2.3:a:activehelper:activehelper_livehelp_live_chat:2.7.4:-:-:-:-:wordpress:-:-'},
+        {'in':'cpe:/o:microsoft:windows:vista:sp1:x64-enterprise',                         'expect':'cpe:2.3:o:microsoft:windows:vista:sp1:x64-enterprise:-:-:-:-:-'},
+        {'in':'cpe:/o:microsoft:windows-nt:vista::enterprise',                             'expect':'cpe:2.3:o:microsoft:windows-nt:vista:-:enterprise:-:-:-:-:-'},
+        {'in':'cpe:/a:novell:iprint:5.90:-:~~~windows_vista~~',                            'expect':'cpe:2.3:a:novell:iprint:5.90:-:-:-:-:windows_vista:-:-'},
+        {'in':'cpe:/o:linux:linux_kernel:-',                                               'expect':'cpe:2.3:o:linux:linux_kernel:-:-:-:-:-:-:-:-'},
+        {'in':'cpe:/a:aokitaka:zip_with_pass_pro:6.3.4:-:~-~-~android~~',                  'expect':'cpe:2.3:a:aokitaka:zip_with_pass_pro:6.3.4:-:-:-:-:android:-:-'},
+        {'in':'cpe:/a:7-zip:7-zip:4.65::~~~~x64~',                                         'expect':'cpe:2.3:a:7-zip:7-zip:4.65:-:-:-:-:-:x64:-'},
+        {'in':'cpe:/a:acl:acl:9.1.0.213',                                                  'expect':'cpe:2.3:a:acl:acl:9.1.0.213:-:-:-:-:-:-:-'}]
+
+trans= [{'in':'cpe:/o:microsoft:windows_server_2008::sp2:itanium',                         'expect':'cpe:2.3:o:microsoft:windows_server_2008:-:sp2:itanium'},
+        {'in':'cpe:/a:activehelper:activehelper_livehelp_live_chat:2.7.4::~~~wordpress~~', 'expect':'cpe:2.3:a:activehelper:activehelper_livehelp_live_chat:2.7.4:-:-:-:-:wordpress'},
+        {'in':'cpe:/o:microsoft:windows:vista:sp1:x64-enterprise',                         'expect':'cpe:2.3:o:microsoft:windows:vista:sp1:x64-enterprise'},
+        {'in':'cpe:/o:microsoft:windows-nt:vista::enterprise',                             'expect':'cpe:2.3:o:microsoft:windows-nt:vista:-:enterprise'},
+        {'in':'cpe:/a:novell:iprint:5.90:-:~~~windows_vista~~',                            'expect':'cpe:2.3:a:novell:iprint:5.90:-:-:-:-:windows_vista'},
+        {'in':'cpe:/o:linux:linux_kernel:-',                                               'expect':'cpe:2.3:o:linux:linux_kernel'},
+        {'in':'cpe:/a:aokitaka:zip_with_pass_pro:6.3.4:-:~-~-~android~~',                  'expect':'cpe:2.3:a:aokitaka:zip_with_pass_pro:6.3.4:-:-:-:-:android'},
+        {'in':'cpe:/a:7-zip:7-zip:4.65::~~~~x64~',                                         'expect':'cpe:2.3:a:7-zip:7-zip:4.65:-:-:-:-:-:x64'},
+        {'in':'cpe:/a:acl:acl:9.1.0.213',                                                  'expect':'cpe:2.3:a:acl:acl:9.1.0.213'}]
+
+result=[]
+for x in extend:
+  result.append(resultOf(x['in'],toStringFormattedCPE(x['in'],autofill=True),x['expect']))
+printResults('Translate - success/autofill',result)
+
+result=[]
+for x in trans:
+  result.append(resultOf(x['in'],toStringFormattedCPE(x['in']),x['expect']))
+printResults('Translate - success/no autofill',result)

web/index.py

@@ -66,8 +66,6 @@
 redisdb = Configuration.getRedisVendorConnection()
 
 # functions
-
-
 def getBrowseList(vendor):
     result = {}
     if (vendor is None) or type(vendor) == list:
@@ -196,13 +194,20 @@ def convertDateToDBFormat(string):
 
 
 def adminStats():
-    stats = {'cveA': db.cves.count(), 'cveU': db.info.find_one({'db': 'cve'})['last-modified'],
-             'cpeA': db.cpe.count(), 'cpeU': db.info.find_one({'db': 'cpe'})['last-modified'],
-             'cpeOtherA': db.cpeother.count(), 'cpeOtherU': db.info.find_one({'db': 'cpeother'})['last-modified'],
-             'capecA': db.capec.count(), 'capecU': db.info.find_one({'db': 'capec'})['last-modified'],
-             'd2secA': db.d2sec.count(), 'd2secU': db.info.find_one({'db': 'd2sec'})['last-modified'],
-             'vendorA': db.vendor.count(), 'vendorU': db.info.find_one({'db': 'vendor'})['last-modified'],
-             'vfeedA': db.vfeed.count(), 'vfeedU': db.info.find_one({'db': 'vfeed'})['last-modified'],
+    cveU = db.info.find_one({'db': 'cve'})
+    cpeU = db.info.find_one({'db': 'cpe'})
+    cpeOtherU = db.info.find_one({'db': 'cpeother'})
+    capecU = db.info.find_one({'db': 'capec'})
+    d2secU = db.info.find_one({'db': 'd2sec'})
+    vendorU = db.info.find_one({'db': 'vendor'})
+    vfeedU = db.info.find_one({'db': 'vfeed'})
+    stats = {'cveA': db.cves.count(), 'cveU': cveU['last-modified'] if cveU is not None else None,
+             'cpeA': db.cpe.count(), 'cpeU': cpeU['last-modified'] if cpeU is not None else None,
+             'cpeOtherA': db.cpeother.count(), 'cpeOtherU': cpeOtherU['last-modified'] if cpeOtherU is not None else None,
+             'capecA': db.capec.count(), 'capecU': capecU['last-modified'] if capecU is not None else None,
+             'd2secA': db.d2sec.count(), 'd2secU': d2secU['last-modified'] if d2secU is not None else None,
+             'vendorA': db.vendor.count(), 'vendorU': vendorU['last-modified'] if vendorU is not None else None,
+             'vfeedA': db.vfeed.count(), 'vfeedU': vfeedU['last-modified'] if vfeedU is not None else None,
              'blA': db.mgmt_blacklist.count(), 'wlA': db.mgmt_whitelist.count(),
              'dbName': Configuration.getMongoDB(), 'dbSize': db.command("dbstats")['dataSize'],
              'dbOnDisk': db.command("dbstats")['storageSize']}
@@ -296,8 +301,6 @@ def load_user(id):
     return User.get(id)
 
 # routes
-
-
 @app.route('/')
 def index():
     # get default page on HTTP get (navigating to page)
@@ -745,15 +748,12 @@ def logout():
     return redirect("/")
 
 # error handeling
-
-
 @app.errorhandler(404)
 def page_not_found(e):
     return render_template('404.html'), 404
 
 
 # filters
-
 @app.template_filter('currentTime')
 def currentTime(utc):
     timezone = tz.tzlocal()
@@ -799,8 +799,6 @@ def vFeedName(string):
     return string.title()
 
 # signal handlers
-
-
 def sig_handler(sig, frame):
     print('Caught signal: %s' % sig)
     IOLoop.instance().add_callback(shutdown)

web/templates/admin.html

@@ -81,13 +81,20 @@
                 <thead>
                   <tr class="warning"><td>Collection</td><td>#records</td><td>Last update</td></tr>
                 </thead>
-                <tr><td>CVES</td>             <td>{{stats['cveA']}}</td>     <td>{{stats['cveU']|currentTime}}</td><tr>
-                <tr><td>CPE</td>              <td>{{stats['cpeA']}}</td>     <td>{{stats['cpeU']|currentTime}}</td><tr>
-                <tr><td>CPE-other</td>        <td>{{stats['cpeOtherA']}}</td><td>{{stats['cpeOtherU']|currentTime}}</td><tr>
-                <tr><td>Capec</td>            <td>{{stats['capecA']}}</td>   <td>{{stats['capecU']|currentTime}}</td><tr>
-                <tr><td>d2sec</td>            <td>{{stats['d2secA']}}</td>   <td>{{stats['d2secU']|currentTime}}</td><tr>
-                <tr><td>Vendor statements</td><td>{{stats['vendorA']}}</td>  <td>{{stats['vendorU']|currentTime}}</td><tr>
-                <tr><td>vFeed info</td>       <td>{{stats['vfeedA']}}</td>   <td>{{stats['vfeedU']|currentTime}}</td><tr>
+                <tr><td>CVES</td>             <td>{{stats['cveA']}}</td>
+                    <td>{% if stats['cveU'] is not none %}{{stats['cveU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>CPE</td>              <td>{{stats['cpeA']}}</td>
+                    <td>{% if stats['cpeU'] is not none %}{{stats['cpeU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>CPE-other</td>        <td>{{stats['cpeOtherA']}}</td>
+                    <td>{% if stats['cpeOtherU'] is not none %}{{stats['cpeOtherU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>Capec</td>            <td>{{stats['capecA']}}</td>
+                    <td>{% if stats['capecU'] is not none %}{{stats['capecU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>d2sec</td>            <td>{{stats['d2secA']}}</td>
+                    <td>{% if stats['d2secU'] is not none %}{{stats['d2secU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>Vendor statements</td><td>{{stats['vendorA']}}</td>
+                    <td>{% if stats['vendorU'] is not none %}{{stats['vendorU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
+                <tr><td>vFeed info</td>       <td>{{stats['vfeedA']}}</td>
+                    <td>{% if stats['vfeedU'] is not none %}{{stats['vfeedU']|currentTime}} {% else %}Not updated{% endif %}</td><tr>
               </table>
               <span>Whitelist: {{stats['wlA']}} rules</span><br />
               <span>Blacklist: {{stats['blA']}} rules</span><br /><br />