Merge pull request #127 from PidgeyL/master

Plug-in manager fixes, updates etc

README.md

@@ -1,6 +1,8 @@
 cve-search
 ==========
 
+[![Join the chat at https://gitter.im/cve-search/cve-search](https://badges.gitter.im/cve-search/cve-search.svg)](https://gitter.im/cve-search/cve-search?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
 ![cve-search logo](https://avatars3.githubusercontent.com/u/15033728?v=3&s=200)
 
 [![Build Status](https://travis-ci.org/cve-search/cve-search.svg?branch=master)](https://travis-ci.org/cve-search/cve-search)
@@ -152,11 +154,11 @@ or department within your organization or any meaningful name for you.
 As an example, you can add a partial CPE name like "sap:netweaver" which is very
 critical for your accounting department.
 
-    ./python3.3 sbin/db_ranking.py  -c "sap:netweaver" -g "accounting" -r 3
+    ./sbin/db_ranking.py  -c "sap:netweaver" -g "accounting" -r 3
 
 and then you can lookup the ranking (-r option) for a specific CVE-ID:
 
-    ./python3.3 bin/search.py -c CVE-2012-4341  -r  -n
+    ./bin/search.py -c CVE-2012-4341  -r  -n
 
 Advanced usage
 --------------
@@ -192,15 +194,15 @@ Fulltext indexing
 
 If you want to index all the CVEs from your current MongoDB collection:
 
-    ./python3.3 sbin/db_fulltext.py
+    ./sbin/db_fulltext.py
 
 and you query the fulltext index (to get a list of matching CVE-ID):
 
-    ./python3.3 bin/search_fulltext.py -q NFS -q Linux
+    ./bin/search_fulltext.py -q NFS -q Linux
 
 or to query the fulltext index and output the JSON object for each CVE-ID:
 
-    ./python3.3 bin/search_fulltext.py -q NFS -q Linux -j
+    ./bin/search_fulltext.py -q NFS -q Linux -j
 
 Fulltext visualization
 ----------------------
@@ -211,7 +213,7 @@ required to generate the keywords with the most common English
 stopwords and lemmatize the output. [NTLK for Python 3](http://nltk.org/nltk3-alpha/)
 exists but you need to use the alpha version of NLTK.
 
-    ./python3.3 bin/search_fulltext.py  -g -s >cve.json
+    ./bin/search_fulltext.py  -g -s >cve.json
 
 ![cve-search visualization](https://farm9.staticflickr.com/8109/8603509755_c7690c2de4_n.jpg "CVE Keywords Visualization Using Data From cve-search")
 
@@ -225,7 +227,7 @@ query a specific CVE. You'll need flask in order to run the website and [Flask-P
 the web interface:
 
     cd ./web
-    ./python3.3 index.py
+    ./index.py
 
 Then you can connect on http://127.0.0.1:5000/ to browser the last CVE.
 

bin/cve_doc.py

@@ -17,7 +17,7 @@
 
 from optparse import OptionParser
 
-from lib.Query import lastentries, apigetcve, apibrowse, apisearch
+from lib.Query import apigetcve
 
 optp = OptionParser()
 optp.add_option('-c', '--cve', dest='cve', default='CVE-2015-0001', help='CVE id to convert')

bin/db_dump.py

@@ -37,4 +37,8 @@
     if 'cvss' in item:
         if type(item['cvss']) == str:
             item['cvss'] = float(item['cvss'])
-    print (json.dumps(item, sort_keys=True, default=json_util.default))
+    date_fields = ['cvss-time', 'Modified', 'Published']
+    for field in date_fields:
+        if field in item:
+            item[field] = str(item[field])
+    print(json.dumps(item, sort_keys=True, default=json_util.default))

bin/dump_last.py

@@ -69,7 +69,7 @@
         print ("<tr class=\"alt\">")
         print ("<td>" + str(x['id']) + " - " + x['summary'][:90] + "...</td>")
         print ("</tr>")
-        print ("<tr><td>CVSS: " + str(x['cvss']) + " Published: " + x['Published'] + "</td></tr>")
+        print ("<tr><td>CVSS: " + str(x['cvss']) + " Published: " + str(x['Published']) + "</td></tr>")
         print ("<tr>")
         print ("<td> Summary: " + x['summary'] + "</td>")
         print ("</tr>")

bin/search.py

@@ -94,9 +94,13 @@
     sorttype = -1
 
 
-def printCVE(item):
+def printCVE(item, indent=None):
+    date_fields = ['cvss-time', 'Modified', 'Published']
+    for field in date_fields:
+        if field in item:
+            item[field] = str(item[field])
     if not namelookup and not rankinglookup and not capeclookup:
-        print(json.dumps(item, sort_keys=True, default=json_util.default))
+        print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent))
     else:
         if "vulnerable_configuration" in item:
             vulconf = []
@@ -115,7 +119,7 @@ def printCVE(item):
             if "cwe" in item and capeclookup:
                 if item['cwe'].lower() != 'unknown':
                     item['capec'] = cves.getcapec(cweid=(item['cwe'].split('-')[1]))
-            print(json.dumps(item, sort_keys=True, default=json_util.default))
+            print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent))
 
 if cveSearch:
     for cveid in db.getCVEs(cves=cveSearch):
@@ -126,7 +130,7 @@ def printCVE(item):
 if vFreeSearch:
     try:
         for item in db.getFreeText(vFreeSearch):
-            print(item)
+            printCVE(item, indent=2)
     except:
         sys.exit("Free text search not enabled on the database!")
     sys.exit(0)
@@ -150,11 +154,11 @@ def printCVE(item):
                 nl = " ".join(item['vulnerable_configuration'])
             csvoutput = csv.writer(sys.stdout, delimiter='|', quotechar='|', quoting=csv.QUOTE_MINIMAL)
             if not namelookup:
-                csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs])
+                csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs])
             else:
-                csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs, nl])
+                csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs, nl])
         elif htmlOutput:
-            print("<h2>" + item['id'] + "<br></h2>CVSS score: " + str(item['cvss']) + "<br>" + "<b>" + item['Published'] + "<b><br>" + item['summary'] + "<br>")
+            print("<h2>" + item['id'] + "<br></h2>CVSS score: " + str(item['cvss']) + "<br>" + "<b>" + str(item['Published']) + "<b><br>" + item['summary'] + "<br>")
             print("References:<br>")
             for entry in item['references']:
                 print(entry + "<br>")
@@ -167,7 +171,7 @@ def printCVE(item):
             c = SubElement(r, 'id')
             c.text = item['id']
             c = SubElement(r, 'Published')
-            c.text = item['Published']
+            c.text = str(item['Published'])
             c = SubElement(r, 'cvss')
             c.text = str(item['cvss'])
             c = SubElement(r, 'summary')
@@ -182,7 +186,7 @@ def printCVE(item):
             print(item['id'])
         else:
             print("CVE\t: " + item['id'])
-            print("DATE\t: " + item['Published'])
+            print("DATE\t: " + str(item['Published']))
             print("CVSS\t: " + str(item['cvss']))
             print(item['summary'])
             print("\nReferences:")

lib/DatabaseLayer.py

@@ -89,7 +89,7 @@ def bulkvFeedUpdate(dbpath, vfeedmap):
         else:
           icveid = names.index("cveid")
       except Exception as ex:
-        sys.exit('Exeption in %s: %s' % (vmap, ex))
+        print('Exeption in %s: %s' % (vmap, ex))
         continue
       mapArray={}
       for i in range(0,len(r)):

lib/PluginManager.py

@@ -18,13 +18,11 @@
 import lib.DatabaseLayer as db
 from lib.Config import Configuration as conf
 from lib.Config import ConfigReader
-from lib.Plugins import Plugin, WebPlugin
-from flask.ext.login import current_user
 
 class PluginManager():
   def __init__(self):
     self.plugins = {}
-  
+
   def loadPlugins(self):
     settingsReader = ConfigReader(conf.getPluginsettings())
     if not os.path.exists(conf.getPluginLoadSettings()):
@@ -129,23 +127,27 @@ def cvePluginInfo(self, cve, **args):
         print("[!]  -> %s"%e)
     return cveInfo
 
-  def getSearchResults(self, text):
+  def getSearchResults(self, text, **args):
     result = {'data':[]}
     results = []
     # Get all data
     for plugin in self.plugins.values():
-      data = plugin.search(text)
+      data = plugin.search(text, **args)
       # Validate format
+      if type(data) == dict: data = [data]
       if type(data) == list and all([(type(x) == dict and 'n' in x and 'd' in x) for x in data]):
         results.extend(data)
     # Sort through data
     for collection in results:
       for item in collection['d']:
         # Check if already in result data
-        if not any(item==entry['id'] for entry in result['data']):
-          entry=db.getCVE(item)
-          entry['reason']=collection['n']
-          result['data'].append(entry)
+        try:
+          if not any(item==entry['id'] for entry in result['data']):
+            entry=db.getCVE(item)
+            entry['reason']=collection['n']
+            result['data'].append(entry)
+        except:
+          pass
     return result
 
   # Actions
@@ -176,7 +178,7 @@ def openPage(self, name, **args):
         return ("error.html", {'status': {'except': 'plugin-not-webplugin'}})
     return ("error.html", {'status': {'except': 'plugin-not-loaded'}})
 
-  def openSubpage(self, subpage, **args):
+  def openSubpage(self, name, subpage, **args):
     if name.strip() in self.plugins.keys(): # Check if plugin exists
       if self.plugins[name].isWebPlugin():  # Check if plugin is web plugin
         pageInfo = self.plugins[name].getSubpage(subpage, **args)

lib/Plugins.py

@@ -28,7 +28,7 @@ def isWebPlugin(self):  return False
   def loadSettings(self, reader): pass
   def onDatabaseUpdate(self):     pass
   # To override with returns
-  def search(self, text):         pass
+  def search(self, text, **args): pass
 
 
 class WebPlugin(Plugin):

lib/Toolkit.py

@@ -10,7 +10,6 @@
 # Imports
 from dateutil import tz
 import dateutil.parser
-import time
 import re
 
 # Note of warning: CPEs like cpe:/o:microsoft:windows_8:-:-:x64 are given to us by Mitre
@@ -61,8 +60,8 @@ def impactScore(cve):
       I=((cve['impact'])['integrity']).upper()
       A=((cve['impact'])['availability']).upper()
       res = 10.41*(1-(1-score[C])*(1-score[I])*(1-score[A]))
-      return 10.0 if res > 10.0 else res 
-    except Exception as ex:
+      return 10.0 if res > 10.0 else res
+    except:
       return '-'
 
 def exploitabilityScore(cve):
@@ -100,28 +99,6 @@ def vFeedName(string):
     string=string.replace('cve_','')
     return string.title()
 
-def convertDateToDBFormat(string):
-    result = None
-    try:
-        result = time.strptime(string, "%d-%m-%Y")
-    except:
-        pass
-    try:
-        result = time.strptime(string, "%d-%m-%y")
-    except:
-        pass
-    try:
-        result = time.strptime(string, "%d/%m/%Y")
-    except:
-        pass
-    try:
-        result = time.strptime(string, "%d/%m/%y")
-    except:
-        pass
-    if result is not None:
-        result = time.strftime('%Y-%m-%d', result)
-    return result
-
 def mergeSearchResults(database, plugins):
     if 'errors' in database:
         results = {'data':[], 'errors':database['errors']}

lib/User.py

@@ -15,7 +15,7 @@
 import os
 runPath = os.path.dirname(os.path.realpath(__file__))
 
-from flask.ext.login import UserMixin
+from flask_login import UserMixin
 
 from lib.Config import Configuration
 import lib.DatabaseLayer as db
@@ -35,7 +35,7 @@ def __init__(self, id):
             USERS = {}
         for user in db.getUsers():
             USERS[user['username']] = user['password']
-        
+
         if not id in USERS:
             raise UserNotFoundError()
         self.id = id

lib/cpelist.py

@@ -16,7 +16,6 @@
 sys.path.append(os.path.join(runPath, ".."))
 
 import json
-import re
 
 from lib.Toolkit import toStringFormattedCPE
 import lib.DatabaseLayer as db

sbin/db_blacklist.py

@@ -16,7 +16,6 @@
 
 import argparse
 
-from lib.Config import Configuration
 from lib.cpelist import CPEList
 
 # parse command line arguments

sbin/db_mgmt.py

@@ -19,6 +19,8 @@
 from xml.sax import make_parser
 from xml.sax.handler import ContentHandler
 
+from dateutil.parser import parse as parse_datetime
+
 from lib.ProgressBar import progressbar
 from lib.Toolkit import toStringFormattedCPE
 from lib.Config import Configuration
@@ -173,16 +175,16 @@ def endElement(self, name):
             self.cves[-1]['impact']['availability'] = self.impacta
         if name == 'cvss:generated-on-datetime':
             self.inCVSSgenElem = 0
-            self.cves[-1]['cvss-time'] = self.cvssgen
+            self.cves[-1]['cvss-time'] = parse_datetime(self.cvssgen, ignoretz=True)
         if name == 'vuln:summary':
             self.inSUMMElem = 0
             self.cves[-1]['summary'] = self.SUMM
         if name == 'vuln:published-datetime':
             self.inDTElem = 0
-            self.cves[-1]['Published'] = self.DT
+            self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True)
         if name == 'vuln:last-modified-datetime':
             self.inPUBElem = 0
-            self.cves[-1]['Modified'] = self.PUB
+            self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True)
 
 if __name__ == '__main__':
     parser = make_parser()
@@ -197,11 +199,12 @@ def endElement(self, name):
         except:
             sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
         i = db.getInfo("cve")
+        last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
         if i is not None:
-            if r.headers['last-modified'] == i['last-modified']:
+            if last_modified == i['last-modified']:
                 print("Not modified")
                 sys.exit(0)
-        db.setColUpdate("cve", r.headers['last-modified'])
+        db.setColUpdate("cve", last_modified)
 
         # get your parser on !!
         parser = make_parser()
@@ -275,8 +278,7 @@ def endElement(self, name):
                         item['cvss'] = float(item['cvss'])
                 # check if year is not cve-free
                 if len(ch.cves) != 0:
+                    print("Importing CVEs for year " + str(x))
                     ret = db.insertCVE(ch.cves)
-                    if ret:
-                        print ("Year " + str(x) + " imported.")
                 else:
                     print ("Year " + str(x) + " has no CVE's.")

sbin/db_mgmt_admin.py

@@ -23,7 +23,6 @@
 import getpass
 from passlib.hash import pbkdf2_sha256
 
-from lib.Config import Configuration
 import lib.DatabaseLayer as dbLayer
 
 # args