Merge branch 'master' into update-agent-server-parse

app/api/v2/__init__.py

@@ -42,4 +42,7 @@ def make_app(services):
     from .handlers.obfuscator_api import ObfuscatorApi
     ObfuscatorApi(services).add_routes(app)
 
+    from .handlers.fact_api import FactApi
+    FactApi(services).add_routes(app)
+
     return app

app/api/v2/handlers/base_object_api.py

@@ -1,4 +1,5 @@
 import abc
+import json
 
 from aiohttp import web
 
@@ -144,8 +145,9 @@ async def delete_on_disk_object(self, request: web.Request):
 
     async def _parse_common_data_from_request(self, request) -> (dict, dict, str, dict, dict):
         data = {}
-        if request.has_body:
-            data = await request.json()
+        raw_body = await request.read()
+        if raw_body:
+            data = json.loads(raw_body)
 
         obj_id = request.match_info.get(self.id_property, '')
         if obj_id:

app/api/v2/handlers/fact_api.py

@@ -0,0 +1,194 @@
+import aiohttp_apispec
+from aiohttp import web
+
+from app.api.v2.handlers.base_object_api import BaseObjectApi
+from app.api.v2.managers.fact_api_manager import FactApiManager
+from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema
+from app.objects.secondclass.c_fact import Fact, FactSchema, OriginType, WILDCARD_STRING
+from app.objects.secondclass.c_relationship import Relationship, RelationshipSchema
+
+
+class FactApi(BaseObjectApi):
+    def __init__(self, services):
+        super().__init__(description='adversary', obj_class=Fact, schema=FactSchema, ram_key='facts',
+                         id_property='adversary_id', auth_svc=services['auth_svc'])
+        self._api_manager = FactApiManager(data_svc=services['data_svc'], file_svc=services['file_svc'],
+                                           knowledge_svc=services['knowledge_svc'])
+
+    def add_routes(self, app: web.Application):
+        router = app.router
+        router.add_get('/facts', self.get_facts)
+        router.add_get('/relationships', self.get_relationships)
+        router.add_post('/facts', self.add_facts)
+        router.add_post('/relationships', self.add_relationships)
+        router.add_delete('/facts', self.delete_facts)
+        router.add_delete('/relationships', self.delete_relationships)
+        router.add_patch('/facts', self.update_facts)
+        router.add_patch('/relationships', self.update_relationships)
+
+    @aiohttp_apispec.docs(tags=['facts'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(FactSchema(many=True, partial=True))
+    async def get_facts(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        fact_data = await self._api_manager.extract_data(request)
+        if fact_data:
+            try:
+                store = await knowledge_svc_handle.get_facts(criteria=fact_data)
+                resp = await self._api_manager.verify_fact_integrity(store)
+                return web.json_response(dict(found=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue retrieving fact {fact_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['relationships'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(RelationshipSchema(many=True, partial=True))
+    async def get_relationships(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        relationship_data = await self._api_manager.extract_data(request)
+        if relationship_data:
+            try:
+                store = await knowledge_svc_handle.get_relationships(criteria=relationship_data)
+                resp = await self._api_manager.verify_relationship_integrity(store)
+                return web.json_response(dict(found=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue retrieving relationship {relationship_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['facts'])
+    @aiohttp_apispec.request_schema(FactSchema)
+    @aiohttp_apispec.response_schema(FactSchema)
+    async def add_facts(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        fact_data = await self._api_manager.extract_data(request)
+        if fact_data:
+            try:
+                new_fact = Fact.load(fact_data)
+                if 'source' not in fact_data:
+                    new_fact.source = WILDCARD_STRING
+                new_fact.source_type = OriginType.USER.name
+                await knowledge_svc_handle.add_fact(new_fact)
+                store = await knowledge_svc_handle.get_facts(criteria=dict(trait=fact_data['trait'],
+                                                                           value=fact_data['value'],
+                                                                           source=WILDCARD_STRING,
+                                                                           source_type=OriginType.USER.name))
+                resp = await self._api_manager.verify_fact_integrity(store)
+                return web.json_response(dict(added=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue saving fact {fact_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['relationships'])
+    @aiohttp_apispec.request_schema(RelationshipSchema)
+    @aiohttp_apispec.response_schema(RelationshipSchema)
+    async def add_relationships(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        relationship_data = await self._api_manager.extract_data(request)
+        if relationship_data:
+            try:
+                origin_target = WILDCARD_STRING
+                new_relationship = Relationship.load(relationship_data)
+                if 'origin' in relationship_data:
+                    origin_target = relationship_data['origin']
+                else:
+                    new_relationship.origin = origin_target
+                shorthand = new_relationship.shorthand
+                new_relationship.source.relationships = [shorthand]
+                new_relationship.source.source = origin_target
+                new_relationship.source.source_type = OriginType.USER.name
+                if 'target' in relationship_data:
+                    new_relationship.target.source = origin_target
+                    new_relationship.target.source_type = OriginType.USER.name
+                    new_relationship.target.relationships = [shorthand]
+                    await knowledge_svc_handle.add_fact(new_relationship.target)
+                await knowledge_svc_handle.add_fact(new_relationship.source)
+                await knowledge_svc_handle.add_relationship(new_relationship)
+
+                store = await knowledge_svc_handle.get_relationships(
+                    criteria=dict(source=new_relationship.source,
+                                  edge=new_relationship.edge if 'edge' in relationship_data else None,
+                                  target=new_relationship.target if 'target' in relationship_data else None,
+                                  origin=origin_target))
+                resp = await self._api_manager.verify_relationship_integrity(store)
+                return web.json_response(dict(added=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue saving relationship {relationship_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['facts'])
+    @aiohttp_apispec.response_schema(FactSchema)
+    async def delete_facts(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        fact_data = await self._api_manager.extract_data(request)
+        if fact_data:
+            try:
+                store = await knowledge_svc_handle.get_facts(criteria=fact_data)
+                await knowledge_svc_handle.delete_fact(criteria=fact_data)
+                resp = await self._api_manager.verify_fact_integrity(store)
+                return web.json_response(dict(removed=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue removing fact {fact_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['relationships'])
+    @aiohttp_apispec.response_schema(RelationshipSchema)
+    async def delete_relationships(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        relationship_data = await self._api_manager.extract_data(request)
+        if relationship_data:
+            try:
+                store = await knowledge_svc_handle.get_relationships(criteria=relationship_data)
+                await knowledge_svc_handle.delete_relationship(criteria=relationship_data)
+                resp = await self._api_manager.verify_relationship_integrity(store)
+                return web.json_response(dict(removed=resp))
+            except Exception as e:
+                self.log.warning(f'Encountered issue removing relationship {relationship_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['facts'])
+    @aiohttp_apispec.request_schema(FactSchema(partial=True))
+    @aiohttp_apispec.response_schema(FactSchema)
+    async def update_facts(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        fact_data = await self._api_manager.extract_data(request)
+        if fact_data:
+            try:
+                if 'criteria' in fact_data and 'updates' in fact_data:
+                    await knowledge_svc_handle.update_fact(criteria=fact_data['criteria'],
+                                                           updates=fact_data['updates'])
+                    temp = await self._api_manager.copy_object(fact_data['criteria'])
+                    for k in fact_data['updates']:
+                        temp[k] = fact_data['updates'][k]
+                    store = await knowledge_svc_handle.get_facts(criteria=temp)
+                    resp = await self._api_manager.verify_fact_integrity(store)
+                    return web.json_response(dict(updated=resp))
+                else:
+                    return web.json_response(dict(error="Need a 'criteria' to match on and 'updates' to apply."))
+            except Exception as e:
+                self.log.warning(f'Encountered issue updating fact {fact_data} - {e}')
+
+    @aiohttp_apispec.docs(tags=['relationships'])
+    @aiohttp_apispec.request_schema(RelationshipSchema(partial=True))
+    @aiohttp_apispec.response_schema(RelationshipSchema)
+    async def update_relationships(self, request: web.Request):
+        knowledge_svc_handle = self._api_manager.knowledge_svc
+        relationship_data = await self._api_manager.extract_data(request)
+        if relationship_data:
+            try:
+                if 'criteria' in relationship_data and 'updates' in relationship_data:
+                    await knowledge_svc_handle.update_relationship(criteria=relationship_data['criteria'],
+                                                                   updates=relationship_data['updates'])
+                    temp = await self._api_manager.copy_object(relationship_data['criteria'])
+                    for k in relationship_data['updates']:
+                        if isinstance(relationship_data['updates'][k], dict):
+                            handle = dict()
+                            if k in relationship_data['criteria'] and \
+                                    isinstance(relationship_data['criteria'][k], dict):
+                                handle = relationship_data['criteria'][k]
+                            for j in relationship_data['updates'][k]:
+                                handle[j] = relationship_data['updates'][k][j]
+                            temp[k] = handle
+                        else:
+                            temp[k] = relationship_data['updates'][k]
+                    store = await knowledge_svc_handle.get_relationships(criteria=temp)
+                    resp = await self._api_manager.verify_relationship_integrity(store)
+                    return web.json_response(dict(updated=resp))
+                else:
+                    return web.json_response(dict(error="Need a 'criteria' to match on and 'updates' to apply."))
+            except Exception as e:
+                self.log.warning(f'Encountered issue updating relationship {relationship_data} - {e}')

app/api/v2/handlers/operation_api.py

@@ -1,24 +1,35 @@
 import aiohttp_apispec
+
 from aiohttp import web
 
 from app.api.v2.handlers.base_object_api import BaseObjectApi
 from app.api.v2.managers.operation_api_manager import OperationApiManager
+from app.api.v2.responses import JsonHttpNotFound
 from app.api.v2.schemas.base_schemas import BaseGetAllQuerySchema, BaseGetOneQuerySchema
 from app.objects.c_operation import Operation, OperationSchema
+from app.objects.secondclass.c_link import LinkSchema
 
 
 class OperationApi(BaseObjectApi):
     def __init__(self, services):
         super().__init__(description='operation', obj_class=Operation, schema=OperationSchema, ram_key='operations',
                          id_property='id', auth_svc=services['auth_svc'])
-        self._api_manager = OperationApiManager(data_svc=services['data_svc'], file_svc=services['file_svc'])
+        self._api_manager = OperationApiManager(services)
 
     def add_routes(self, app: web.Application):
         router = app.router
         router.add_get('/operations', self.get_operations)
         router.add_get('/operations/{id}', self.get_operation_by_id)
+        router.add_post('/operations', self.create_operation)
+        router.add_patch('/operations/{id}', self.update_operation)
         router.add_delete('/operations/{id}', self.delete_operation)
         router.add_get('/operations/{id}/report', self.get_operation_report)
+        router.add_get('/operations/{id}/links', self.get_operation_links)
+        router.add_get('/operations/{id}/links/{link_id}', self.get_operation_link)
+        router.add_patch('/operations/{id}/links/{link_id}', self.update_operation_link)
+        router.add_post('/operations/{id}/potential-links', self.create_potential_link)
+        router.add_get('/operations/{id}/potential-links', self.get_potential_links)
+        router.add_get('/operations/{id}/potential-links/{paw}', self.get_potential_links_by_paw)
 
     @aiohttp_apispec.docs(tags=['operations'])
     @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
@@ -34,6 +45,20 @@ async def get_operation_by_id(self, request: web.Request):
         operation = await self.get_object(request)
         return web.json_response(operation)
 
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.request_schema(OperationSchema)
+    @aiohttp_apispec.response_schema(OperationSchema)
+    async def create_operation(self, request: web.Request):
+        operation = await self.create_object(request)
+        return web.json_response(operation.display)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.request_schema(OperationSchema(partial=True))
+    @aiohttp_apispec.response_schema(OperationSchema(partial=True))
+    async def update_operation(self, request: web.Request):
+        operation = await self.update_object(request)
+        return web.json_response(operation.display)
+
     @aiohttp_apispec.docs(tags=['operations'])
     @aiohttp_apispec.response_schema(OperationSchema)
     async def delete_operation(self, request: web.Request):
@@ -47,3 +72,76 @@ async def get_operation_report(self, request: web.Request):
         access = await self.get_request_permissions(request)
         report = await self._api_manager.get_operation_report(operation_id, access)
         return web.json_response(report)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(LinkSchema(many=True, partial=True))
+    async def get_operation_links(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        access = await self.get_request_permissions(request)
+        links = await self._api_manager.get_operation_links(operation_id, access)
+        return web.json_response(links)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema)
+    @aiohttp_apispec.response_schema(LinkSchema(partial=True))
+    async def get_operation_link(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        link_id = request.match_info.get('link_id')
+        access = await self.get_request_permissions(request)
+        link = await self._api_manager.get_operation_link(operation_id, link_id, access)
+        return web.json_response(link)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.request_schema(LinkSchema(partial=True))
+    @aiohttp_apispec.response_schema(LinkSchema)
+    async def update_operation_link(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        link_id = request.match_info.get('link_id')
+        access = await self.get_request_permissions(request)
+        data = await request.json()
+        link = await self._api_manager.update_operation_link(operation_id, link_id, data, access)
+        return web.json_response(link)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.request_schema(LinkSchema)
+    @aiohttp_apispec.response_schema(LinkSchema)
+    async def create_potential_link(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        access = await self.get_request_permissions(request)
+        data = await request.json()
+        potential_link = await self._api_manager.create_potential_link(operation_id, data, access)
+        return web.json_response(potential_link)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.querystring_schema(BaseGetAllQuerySchema)
+    @aiohttp_apispec.response_schema(LinkSchema(many=True, partial=True))
+    async def get_potential_links(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        access = await self.get_request_permissions(request)
+        potential_links = await self._api_manager.get_potential_links(operation_id, access)
+        return web.json_response(potential_links)
+
+    @aiohttp_apispec.docs(tags=['operations'])
+    @aiohttp_apispec.querystring_schema(BaseGetOneQuerySchema)
+    @aiohttp_apispec.response_schema(LinkSchema(partial=True))
+    async def get_potential_links_by_paw(self, request: web.Request):
+        operation_id = request.match_info.get('id')
+        paw = request.match_info.get('paw')
+        access = await self.get_request_permissions(request)
+        potential_links = await self._api_manager.get_potential_links(operation_id, access, paw)
+        return web.json_response(potential_links)
+
+    '''Overridden Methods'''
+    async def create_object(self, request: web.Request):
+        data = await request.json()
+        await self._error_if_object_with_id_exists(data.get(self.id_property))
+        access = await self.get_request_permissions(request)
+        return await self._api_manager.create_object_from_schema(self.schema, data, access)
+
+    async def update_object(self, request: web.Request):
+        data, access, obj_id, query, search = await self._parse_common_data_from_request(request)
+        obj = await self._api_manager.find_and_update_object(self.ram_key, data, search)
+        if not obj:
+            raise JsonHttpNotFound(f'{self.description.capitalize()} not found: {obj_id}')
+        return obj