Merge pull request #202 from NTIA/preselector

Preselector

configs/preselector_config.json

@@ -0,0 +1 @@
+{}

docker-compose.yml

@@ -28,6 +28,7 @@ services:
       dockerfile: docker/Dockerfile-api
       args:
         - BASE_IMAGE
+        - DEBUG
         - DOCKER_GIT_CREDENTIALS
     environment:
       - ADMIN_EMAIL

docker/Dockerfile-api

@@ -17,11 +17,13 @@ ENV PYTHONUNBUFFERED 1
 RUN mkdir -p /src
 WORKDIR /src
 COPY ./src/requirements.txt /src
+COPY ./src/requirements-dev.txt /src
 
 ARG DOCKER_GIT_CREDENTIALS
 RUN git config --global credential.helper store && echo "${DOCKER_GIT_CREDENTIALS}" > ~/.git-credentials
 RUN python3.7 -m pip install --upgrade pip
-RUN python3.7 -m pip install --no-cache-dir -r requirements.txt
+ARG DEBUG
+RUN if [ "$DEBUG" = true ]; then python3.7 -m pip install --no-cache-dir -r requirements-dev.txt ; else python3.7 -m pip install --no-cache-dir -r requirements.txt ; fi
 
 COPY ./src /src
 COPY ./gunicorn /gunicorn
@@ -37,7 +39,6 @@ RUN chmod +x /entrypoints/api_entrypoint.sh
 COPY ./configs /configs
 
 # Args are passed in via docker-compose during build time
-ARG DEBUG
 ARG DOMAINS
 ARG IPS
 ARG SECRET_KEY

src/actions/logger.py

@@ -23,7 +23,7 @@ class Logger(Action):
     def __init__(self, loglvl=LOGLVL_INFO):
         self.loglvl = loglvl
 
-    def __call__(self, schedule_entry_json, task_id, sensor_definition):
+    def __call__(self, schedule_entry_json, task_id):
         msg = "running test {name}/{tid}"
         schedule_entry_name = schedule_entry_json["name"]
         logger.log(

src/authentication/tests/test_jwt_auth.py

@@ -69,9 +69,8 @@ def test_no_token_unauthorized(live_server):
 def test_token_no_roles_unauthorized(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=[])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
     assert response.json()["detail"] == "User missing required role"
 
@@ -80,9 +79,8 @@ def test_token_no_roles_unauthorized(live_server, jwt_keys):
 def test_token_role_manager_accepted(live_server, jwt_keys):
     token_payload = get_token_payload()
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
 
@@ -106,9 +104,8 @@ def test_token_expired_1_day_forbidden(live_server, jwt_keys):
     current_datetime = datetime.now()
     token_payload = get_token_payload(exp=(current_datetime - one_day).timestamp())
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
     assert response.json()["detail"] == "Token is expired!"
 
@@ -119,9 +116,8 @@ def test_bad_private_key_forbidden(live_server):
     encoded = jwt.encode(
         token_payload, str(BAD_PRIVATE_KEY.decode("utf-8")), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
     assert response.json()["detail"] == "Unable to verify token!"
 
@@ -136,9 +132,8 @@ def test_bad_public_key_forbidden(settings, live_server, jwt_keys):
         encoded = jwt.encode(
             token_payload, str(jwt_keys.private_key), algorithm="RS256"
         )
-        utf8_bytes = encoded.decode("utf-8")
         client = RequestsClient()
-        response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+        response = client.get(f"{live_server.url}", headers=get_headers(encoded))
         assert response.status_code == 403
         assert response.json()["detail"] == "Unable to verify token!"
 
@@ -148,9 +143,8 @@ def test_token_expired_1_min_forbidden(live_server, jwt_keys):
     current_datetime = datetime.now()
     token_payload = get_token_payload(exp=(current_datetime - one_min).timestamp())
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
     assert response.json()["detail"] == "Token is expired!"
 
@@ -160,19 +154,17 @@ def test_token_expires_in_1_min_accepted(live_server, jwt_keys):
     current_datetime = datetime.now()
     token_payload = get_token_payload(exp=(current_datetime + one_min).timestamp())
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
 
 @pytest.mark.django_db
 def test_token_role_user_forbidden(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=["ROLE_USER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
     assert response.json()["detail"] == "User missing required role"
 
@@ -182,9 +174,8 @@ def test_token_role_user_required_role_accepted(settings, live_server, jwt_keys)
     settings.REQUIRED_ROLE = "ROLE_USER"
     token_payload = get_token_payload(authorities=["ROLE_USER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
 
@@ -194,9 +185,8 @@ def test_token_multiple_roles_accepted(live_server, jwt_keys):
         authorities=["ROLE_MANAGER", "ROLE_USER", "ROLE_ITS"]
     )
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
 
@@ -206,19 +196,17 @@ def test_token_mulitple_roles_forbidden(live_server, jwt_keys):
         authorities=["ROLE_SENSOR", "ROLE_USER", "ROLE_ITS"]
     )
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 403
 
 
 @pytest.mark.django_db
 def test_urls_unauthorized(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=["ROLE_USER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    headers = get_headers(utf8_bytes)
+    headers = get_headers(encoded)
 
     capabilities = reverse("capabilities", kwargs=V1)
     response = client.get(f"{live_server.url}{capabilities}", headers=headers)
@@ -253,9 +241,8 @@ def test_urls_unauthorized(live_server, jwt_keys):
 def test_urls_authorized(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=["ROLE_MANAGER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    headers = get_headers(utf8_bytes)
+    headers = get_headers(encoded)
 
     capabilities = reverse("capabilities", kwargs=V1)
     response = client.get(f"{live_server.url}{capabilities}", headers=headers)
@@ -292,25 +279,23 @@ def test_user_cannot_view_user_detail(live_server, jwt_keys):
     encoded = jwt.encode(
         sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
     sensor02_token_payload = get_token_payload(authorities=["ROLE_USER"])
     sensor02_token_payload["user_name"] = "sensor02"
     encoded = jwt.encode(
         sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
 
     sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"])
     kws = {"pk": sensor01_user.pk}
     kws.update(V1)
     user_detail = reverse("user-detail", kwargs=kws)
     response = client.get(
-        f"{live_server.url}{user_detail}", headers=get_headers(utf8_bytes)
+        f"{live_server.url}{user_detail}", headers=get_headers(encoded)
     )
     assert response.status_code == 403
 
@@ -321,22 +306,20 @@ def test_user_cannot_view_user_detail_role_change(live_server, jwt_keys):
     encoded = jwt.encode(
         sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
     token_payload = get_token_payload(authorities=["ROLE_USER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
 
     sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"])
     kws = {"pk": sensor01_user.pk}
     kws.update(V1)
     user_detail = reverse("user-detail", kwargs=kws)
     response = client.get(
-        f"{live_server.url}{user_detail}", headers=get_headers(utf8_bytes)
+        f"{live_server.url}{user_detail}", headers=get_headers(encoded)
     )
     assert response.status_code == 403
 
@@ -345,9 +328,8 @@ def test_user_cannot_view_user_detail_role_change(live_server, jwt_keys):
 def test_admin_can_view_user_detail(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=["ROLE_MANAGER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    headers = get_headers(utf8_bytes)
+    headers = get_headers(encoded)
     response = client.get(f"{live_server.url}", headers=headers)
     assert response.status_code == 200
 
@@ -365,25 +347,23 @@ def test_admin_can_view_other_user_detail(live_server, jwt_keys):
     encoded = jwt.encode(
         sensor01_token_payload, str(jwt_keys.private_key), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    response = client.get(f"{live_server.url}", headers=get_headers(utf8_bytes))
+    response = client.get(f"{live_server.url}", headers=get_headers(encoded))
     assert response.status_code == 200
 
     sensor02_token_payload = get_token_payload(authorities=["ROLE_MANAGER"])
     sensor02_token_payload["user_name"] = "sensor02"
     encoded = jwt.encode(
         sensor02_token_payload, str(jwt_keys.private_key), algorithm="RS256"
     )
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
 
     sensor01_user = User.objects.get(username=sensor01_token_payload["user_name"])
     kws = {"pk": sensor01_user.pk}
     kws.update(V1)
     user_detail = reverse("user-detail", kwargs=kws)
     response = client.get(
-        f"{live_server.url}{user_detail}", headers=get_headers(utf8_bytes)
+        f"{live_server.url}{user_detail}", headers=get_headers(encoded)
     )
     assert response.status_code == 200
 
@@ -392,9 +372,8 @@ def test_admin_can_view_other_user_detail(live_server, jwt_keys):
 def test_token_hidden(live_server, jwt_keys):
     token_payload = get_token_payload(authorities=["ROLE_MANAGER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
     client = RequestsClient()
-    headers = get_headers(utf8_bytes)
+    headers = get_headers(encoded)
     response = client.get(f"{live_server.url}", headers=headers)
     assert response.status_code == 200
 
@@ -416,10 +395,9 @@ def test_change_token_role_bad_signature(live_server, jwt_keys):
     """Make sure token modified after it was signed is rejected"""
     token_payload = get_token_payload(authorities=["ROLE_USER"])
     encoded = jwt.encode(token_payload, str(jwt_keys.private_key), algorithm="RS256")
-    utf8_bytes = encoded.decode("utf-8")
-    first_period = utf8_bytes.find(".")
-    second_period = utf8_bytes.find(".", first_period + 1)
-    payload = utf8_bytes[first_period + 1 : second_period]
+    first_period = encoded.find(".")
+    second_period = encoded.find(".", first_period + 1)
+    payload = encoded[first_period + 1 : second_period]
     payload_bytes = payload.encode("utf-8")
     # must be multiple of 4 for b64decode
     for i in range(len(payload_bytes) % 4):
@@ -431,8 +409,8 @@ def test_change_token_role_bad_signature(live_server, jwt_keys):
     payload_data["authorities"] = ["ROLE_MANAGER"]
     payload_data["userDetails"]["authorities"] = [{"authority": "ROLE_MANAGER"}]
     payload_str = json.dumps(payload_data)
-    encoded = base64.b64encode(payload_str.encode("utf-8"))
-    modified_payload = encoded.decode("utf-8")
+    modified_payload = base64.b64encode(payload_str.encode("utf-8"))
+    modified_payload = modified_payload.decode("utf-8")
     # remove padding
     if modified_payload.endswith("="):
         last_padded_index = len(modified_payload) - 1
@@ -442,11 +420,11 @@ def test_change_token_role_bad_signature(live_server, jwt_keys):
                 break
         modified_payload = modified_payload[: last_padded_index + 1]
     modified_token = (
-        utf8_bytes[:first_period]
+        encoded[:first_period]
         + "."
         + modified_payload
         + "."
-        + utf8_bytes[second_period + 1 :]
+        + encoded[second_period + 1 :]
     )
     client = RequestsClient()
     response = client.get(f"{live_server.url}", headers=get_headers(modified_token))

src/capabilities/__init__.py

@@ -1,48 +0,0 @@
-import copy
-
-from sensor import settings
-from sensor.settings import SENSOR_DEFINITION_FILE
-
-
-def load_from_json(fname):
-    import json
-    import logging
-
-    logger = logging.getLogger(__name__)
-
-    try:
-        with open(fname) as f:
-            return json.load(f)
-    except Exception:
-        logger.exception("Unable to load JSON file {}".format(fname))
-
-
-def get_sigmf_location():
-    from status.models import Location
-
-    try:
-        db_location = Location.objects.get(active=True)
-        return {
-            "x": db_location.longitude,
-            "y": db_location.latitude,
-            "z": db_location.height,
-            "description": db_location.description,
-        }
-    except Location.DoesNotExist:
-        return None
-
-
-_capabilities = {}
-_capabilities["sensor"] = load_from_json(SENSOR_DEFINITION_FILE)
-_capabilities["sensor"]["id"] = settings.FQDN
-
-
-def get_capabilities():
-    capabilities = copy.deepcopy(_capabilities)
-    location = get_sigmf_location()
-    if location:
-        capabilities["sensor"]["location"] = location
-    else:
-        if "location" in capabilities['sensor']:
-            del capabilities["sensor"]["location"]
-    return capabilities