chg: [api] get object + get investigation

bin/lib/Investigations.py

@@ -152,25 +152,30 @@ def get_misp_events(self):
         return r_tracking.smembers(f'investigations:misp:{self.uuid}')
 
     # # TODO: DATE FORMAT
-    def get_metadata(self, r_str=False):
+    def get_metadata(self, options=set(), r_str=False):
         if r_str:
             analysis = self.get_analysis_str()
             threat_level = self.get_threat_level_str()
         else:
             analysis = self.get_analysis()
             threat_level = self.get_threat_level()
-        return {'uuid': self.uuid,
-                'name': self.get_name(),
+
+        # 'name': self.get_name(),
+        meta = {'uuid': self.uuid,
                 'threat_level': threat_level,
                 'analysis': analysis,
-                'tags': self.get_tags(),
+                'tags': list(self.get_tags()),
                 'user_creator': self.get_creator_user(),
                 'date': self.get_date(),
                 'timestamp': self.get_timestamp(r_str=r_str),
                 'last_change': self.get_last_change(r_str=r_str),
                 'info': self.get_info(),
                 'nb_objects': self.get_nb_objects(),
-                'misp_events': self.get_misp_events()}
+                'misp_events': list(self.get_misp_events())
+                }
+        if 'objects' in options:
+            meta['objects'] = self.get_objects()
+        return meta
 
     def set_name(self, name):
         r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name)
@@ -368,6 +373,21 @@ def get_investigations_selector():
 
 ####  API  ####
 
+def api_get_investigation(investigation_uuid): # TODO check if is UUIDv4
+    investigation = Investigation(investigation_uuid)
+    if not investigation.exists():
+        return {'status': 'error', 'reason': 'Investigation Not Found'}, 404
+
+    meta = investigation.get_metadata(options={'objects'}, r_str=False)
+    # objs = []
+    # for obj in investigation.get_objects():
+    #     obj_meta = ail_objects.get_object_meta(obj["type"], obj["subtype"], obj["id"], flask_context=True)
+    #     comment = investigation.get_objects_comment(f'{obj["type"]}:{obj["subtype"]}:{obj["id"]}')
+    #     if comment:
+    #         obj_meta['comment'] = comment
+    #     objs.append(obj_meta)
+    return meta, 200
+
 # # TODO: CHECK Mandatory Fields
 # # TODO: SANITYZE Fields
 # # TODO: Name ?????

bin/lib/ail_api.py

@@ -80,4 +80,4 @@ def authenticate_user(token, ip_address):
             return {'status': 'error', 'reason': 'Authentication failed'}, 401
     except Exception as e:
         print(e)  # TODO Logs
-        return {'status': 'error', 'reason': 'Malformed Authentication String'}, 400
+        return {'status': 'error', 'reason': 'Malformed Authentication String'}, 400

bin/lib/ail_core.py

@@ -20,6 +20,8 @@
                       'domain', 'etag', 'favicon', 'file-name', 'hhhash',
                       'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
+AIL_OBJECTS_WITH_SUBTYPES = {'chat', 'chat-subchannel', 'cryptocurrency', 'pgp', 'username', 'user-account'}
+
 def get_ail_uuid():
     ail_uuid = r_serv_db.get('ail:uuid')
     if not ail_uuid:
@@ -48,7 +50,7 @@ def get_all_objects():
     return AIL_OBJECTS
 
 def get_objects_with_subtypes():
-    return ['chat', 'cryptocurrency', 'pgp', 'username', 'user-account']
+    return AIL_OBJECTS_WITH_SUBTYPES
 
 def get_object_all_subtypes(obj_type):  # TODO Dynamic subtype
     if obj_type == 'chat':

bin/lib/objects/Favicons.py

@@ -62,11 +62,18 @@ def get_filepath(self):
         filename = os.path.join(FAVICON_FOLDER, self.get_rel_path())
         return os.path.realpath(filename)
 
-    def get_file_content(self):
+    def get_file_content(self, r_type='str'):
         filepath = self.get_filepath()
-        with open(filepath, 'rb') as f:
-            file_content = BytesIO(f.read())
-        return file_content
+        if r_type == 'str':
+            with open(filepath, 'rb') as f:
+                file_content = f.read()
+            b64 = base64.b64encode(file_content)
+            # b64 = base64.encodebytes(file_content)
+            return b64.decode()
+        elif r_type == 'io':
+            with open(filepath, 'rb') as f:
+                file_content = BytesIO(f.read())
+                return file_content
 
     def get_content(self, r_type='str'):
         return self.get_file_content()

bin/lib/objects/ail_objects.py

@@ -12,7 +12,7 @@
 
 
 from lib.ConfigLoader import ConfigLoader
-from lib.ail_core import get_all_objects, get_object_all_subtypes
+from lib.ail_core import get_all_objects, get_object_all_subtypes, get_objects_with_subtypes
 from lib import correlations_engine
 from lib import relationships_engine
 from lib import btc_ail
@@ -47,6 +47,9 @@
 def is_valid_object_type(obj_type):
     return obj_type in get_all_objects()
 
+def is_object_subtype(obj_type):
+    return obj_type in get_objects_with_subtypes()
+
 def is_valid_object_subtype(obj_type, subtype):
     return subtype in get_object_all_subtypes(obj_type)
 
@@ -117,6 +120,39 @@ def exists_obj(obj_type, subtype, obj_id):
     else:
         return False
 
+#### API ####
+
+def api_get_object(obj_type, obj_subtype, obj_id):
+    if not obj_id:
+        return {'status': 'error', 'reason': 'Invalid object id'}, 400
+    if not is_valid_object_type(obj_type):
+        return {'status': 'error', 'reason': 'Invalid object type'}, 400
+    if obj_subtype:
+        if not is_valid_object_subtype(obj_type, subtype):
+            return {'status': 'error', 'reason': 'Invalid object subtype'}, 400
+    obj = get_object(obj_type, obj_subtype, obj_id)
+    if not obj.exists():
+        return {'status': 'error', 'reason': 'Object Not Found'}, 404
+    options = {'chat', 'content', 'files-names', 'images', 'parent', 'parent_meta', 'reactions', 'thread', 'user-account'}
+    return obj.get_meta(options=options), 200
+
+
+def api_get_object_type_id(obj_type, obj_id):
+    if not is_valid_object_type(obj_type):
+        return {'status': 'error', 'reason': 'Invalid object type'}, 400
+    if is_object_subtype(obj_type):
+        subtype, obj_id = obj_type.split('/', 1)
+    else:
+        subtype = None
+    return api_get_object(obj_type, subtype, obj_id)
+
+
+def api_get_object_global_id(global_id):
+    obj_type, subtype, obj_id = global_id.split(':', 2)
+    return api_get_object(obj_type, subtype, obj_id)
+
+#### --API-- ####
+
 #########################################################################################
 #########################################################################################
 #########################################################################################

var/www/Flask_server.py

@@ -234,18 +234,25 @@ def _handle_client_error(e):
         anchor_id = anchor_id.replace('/', '_')
         api_doc_url = 'https://github.com/ail-project/ail-framework/tree/master/doc#{}'.format(anchor_id)
         res_dict['documentation'] = api_doc_url
-        return Response(json.dumps(res_dict, indent=2, sort_keys=True), mimetype='application/json'), 405
+        return Response(json.dumps(res_dict) + '\n', mimetype='application/json'), 405
     else:
         return e
 
 @app.errorhandler(404)
 def error_page_not_found(e):
     if request.path.startswith('/api/'): ## # TODO: add baseUrl
-        return Response(json.dumps({"status": "error", "reason": "404 Not Found"}, indent=2, sort_keys=True), mimetype='application/json'), 404
+        return Response(json.dumps({"status": "error", "reason": "404 Not Found"}) + '\n', mimetype='application/json'), 404
     else:
         # avoid endpoint enumeration
         return page_not_found(e)
 
+@app.errorhandler(500)
+def _handle_client_error(e):
+    if request.path.startswith('/api/'):
+        return Response(json.dumps({"status": "error", "reason": "Server Error"}) + '\n', mimetype='application/json'), 500
+    else:
+        return e
+
 @login_required
 def page_not_found(e):
     # avoid endpoint enumeration

var/www/blueprints/api_rest.py

@@ -21,14 +21,14 @@
 from lib import ail_updates
 from lib import crawlers
 
+from lib import Investigations
 from lib import Tag
 
 from lib.objects import ail_objects
-from importer.FeederImporter import api_add_json_feeder_to_queue
-
 from lib.objects import Domains
 from lib.objects import Titles
 
+from importer.FeederImporter import api_add_json_feeder_to_queue
 
 
 # ============ BLUEPRINT ============
@@ -75,8 +75,8 @@ def api_token(*args, **kwargs):
 
 # ============ FUNCTIONS ============
 
-def create_json_response(data, status_code):  # TODO REMOVE INDENT ????????????????????
-    return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
+def create_json_response(data, status_code):
+    return Response(json.dumps(data) + "\n", mimetype='application/json'), status_code
 
 # ============= ROUTES ==============
 
@@ -150,12 +150,38 @@ def import_json_item():
     return Response(json.dumps(res[0]), mimetype='application/json'), res[1]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-# # # # # # # # # # # # # # #      OBJECTS      # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # #      OBJECTS      # # # # # # # # # # # # # # # # # # # TODO LIST OBJ TYPES + SUBTYPES
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@api_rest.route("api/v1/object", methods=['GET'])  # TODO options
+@token_required('read_only')
+def v1_object():
+    obj_gid = request.args.get('gid')
+    if obj_gid:
+        r = ail_objects.api_get_object_global_id(obj_gid)
+    else:
+        obj_type = request.args.get('type')
+        obj_subtype = request.args.get('subtype')
+        obj_id = request.args.get('id')
+        r = ail_objects.api_get_object(obj_type, obj_subtype, obj_id)
+    print(r[0])
+    return create_json_response(r[0], r[1])
+
+
+@api_rest.route("api/v1/obj/gid/<path:object_global_id>", methods=['GET'])  # TODO REMOVE ME ????
+@token_required('read_only')
+def v1_object_global_id(object_global_id):
+    r = ail_objects.api_get_object_global_id(object_global_id)
+    return create_json_response(r[0], r[1])
 
+# @api_rest.route("api/v1/object/<object_type>/<object_subtype>/<path:object_id>", methods=['GET'])
+@api_rest.route("api/v1/obj/<object_type>/<path:object_id>", methods=['GET'])  # TODO REMOVE ME ????
+@token_required('read_only')
+def v1_object_type_id(object_type, object_id):
+    r = ail_objects.api_get_object_type_id(object_type, object_id)
+    return create_json_response(r[0], r[1])
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-# # # # # # # # # # # # # # #      TITLES       # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # #      TITLES       # # # # # # # # # # # # # # # # # # # TODO TO REVIEW
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
 @api_rest.route("api/v1/titles/download", methods=['GET'])
@@ -184,4 +210,13 @@ def objects_titles_download_unsafe():
                     all_titles[title_content].append(domain.get_id())
     return Response(json.dumps(all_titles), mimetype='application/json'), 200
 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # #      INVESTIGATIONS     # # # # # # # # # # # # # # #
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+@api_rest.route("api/v1/investigation/<investigation_uuid>", methods=['GET'])  # TODO options
+@token_required('read_only')
+def v1_investigation(investigation_uuid):
+    r = Investigations.api_get_investigation(investigation_uuid)
+    return create_json_response(r[0], r[1])
 
+# TODO CATCH REDIRECT