update libs, fix small issue

compose/local/dask/Dockerfile

@@ -1,4 +1,4 @@
-FROM daskdev/dask:2024.3.1-py3.12
+FROM daskdev/dask:2024.4.1-py3.12
 ENV DEBIAN_FRONTEND noninteractive
 
 ARG local_folder=/uploads
@@ -47,7 +47,7 @@ RUN python setup.py build \
 # Workers should have similar reqs as django
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@a19dfb57a68cc7efc8926d9f6e7c24e5a2dbab9d#egg=volatility3 \
+RUN pip install uv==0.1.29 -e git+https://github.com/volatilityfoundation/volatility3.git@a19dfb57a68cc7efc8926d9f6e7c24e5a2dbab9d#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh

compose/local/dask/daskworkerinit.py

@@ -1,11 +1,16 @@
+# sourcery skip: use-fstring-for-formatting
 import logging
 import os
 import sys
 
 import django
 
-logger = logging.getLogger("distributed.utils_perf")
-logger.setLevel(logging.ERROR)
+logger_one = logging.getLogger("distributed.preloading")
+logger_one.setLevel(logging.ERROR)
+logger_two = logging.getLogger("distributed.utils")
+logger_two.setLevel(logging.ERROR)
+logger_three = logging.getLogger("distributed.utils_perf")
+logger_three.setLevel(logging.ERROR)
 
 os.environ["DATABASE_URL"] = "postgres://{}:{}@{}:{}/{}".format(
     os.environ["POSTGRES_USER"],

compose/local/django/Dockerfile

@@ -65,7 +65,7 @@ RUN /usr/local/go/bin/go build
 FROM common-base
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@a19dfb57a68cc7efc8926d9f6e7c24e5a2dbab9d#egg=volatility3 \
+RUN pip install uv==0.1.29 -e git+https://github.com/volatilityfoundation/volatility3.git@a19dfb57a68cc7efc8926d9f6e7c24e5a2dbab9d#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py

config/settings/base.py

@@ -67,7 +67,6 @@
     "django_file_form",
     "guardian",
     "widget_tweaks",
-    "django_json_widget",
     "django_admin_listfilter_dropdown",
     "django_admin_multiple_choice_list_filter",
 ]

orochi/utils/volatility_dask_elk.py

@@ -19,6 +19,7 @@
 import elasticsearch
 import magic
 import requests
+import volatility3.plugins
 import vt
 from asgiref.sync import sync_to_async
 from bs4 import BeautifulSoup
@@ -34,24 +35,6 @@
 )
 from regipy.plugins.plugin import PLUGINS
 from regipy.registry import RegistryHive
-
-import volatility3.plugins
-from orochi.website.defaults import (
-    DUMP_STATUS_COMPLETED,
-    DUMP_STATUS_CREATED,
-    DUMP_STATUS_ERROR,
-    DUMP_STATUS_MISSING_SYMBOLS,
-    DUMP_STATUS_UNZIPPING,
-    RESULT_STATUS_DISABLED,
-    RESULT_STATUS_EMPTY,
-    RESULT_STATUS_ERROR,
-    RESULT_STATUS_NOT_STARTED,
-    RESULT_STATUS_RUNNING,
-    RESULT_STATUS_SUCCESS,
-    RESULT_STATUS_UNSATISFIED,
-    SERVICE_VIRUSTOTAL,
-)
-from orochi.website.models import CustomRule, Dump, Result, Service
 from volatility3 import cli, framework
 from volatility3.cli.text_renderer import (
     JsonRenderer,
@@ -78,6 +61,23 @@
     ListRequirement,
 )
 
+from orochi.website.defaults import (
+    DUMP_STATUS_COMPLETED,
+    DUMP_STATUS_CREATED,
+    DUMP_STATUS_ERROR,
+    DUMP_STATUS_MISSING_SYMBOLS,
+    DUMP_STATUS_UNZIPPING,
+    RESULT_STATUS_DISABLED,
+    RESULT_STATUS_EMPTY,
+    RESULT_STATUS_ERROR,
+    RESULT_STATUS_NOT_STARTED,
+    RESULT_STATUS_RUNNING,
+    RESULT_STATUS_SUCCESS,
+    RESULT_STATUS_UNSATISFIED,
+    SERVICE_VIRUSTOTAL,
+)
+from orochi.website.models import CustomRule, Dump, Result, Service
+
 BANNER_REGEX = r'^"?Linux version (?P<kernel>\S+) (?P<build>.+) \(((?P<gcc>gcc.+)) #(?P<number>\d+)(?P<info>.+)$"?'
 
 COLOR_TIMELINER = {
@@ -492,7 +492,7 @@ def run_plugin(dump_obj, plugin_obj, params=None, user_pk=None, regipy_plugins=F
 
                 # CALCOLATE HASH AND CHECK FOR CLAMAV SIGNATURE
                 for x in json_data:
-                    filename = x["File output"].replace('"', "")
+                    filename = x.get("File output", "").replace('"', "")
                     down_path = f"{local_path}/{filename}"
                     if os.path.exists(down_path) and not os.path.isdir(down_path):
                         x["down_path"] = down_path

orochi/website/admin.py

@@ -68,6 +68,7 @@ class DumpAdmin(GuardedModelAdmin):
     list_display = ("name", "author", "index", "status")
     search_fields = ["author__name", "name", "index"]
     list_filter = ("author", "status", "created_at")
+    exclude = ("suggested_symbols_path", "regipy_plugins", "banner")
 
     def get_queryset(self, request):
         return super(DumpAdmin, self).get_queryset(request).prefetch_related("plugins")

orochi/website/views.py

@@ -39,6 +39,7 @@
 from guardian.shortcuts import assign_perm, get_objects_for_user, get_perms, remove_perm
 from pymisp import MISPEvent, MISPObject, PyMISP
 from pymisp.tools import FileObject
+from volatility3.framework import automagic, contexts
 
 from orochi.utils.download_symbols import Downloader
 from orochi.utils.plugin_install import plugin_install
@@ -84,7 +85,6 @@
     Service,
     UserPlugin,
 )
-from volatility3.framework import automagic, contexts
 
 COLOR_TEMPLATE = """
     <svg class="bd-placeholder-img rounded me-2" width="20" height="20"
@@ -162,11 +162,10 @@ def plugins(request):
                 DUMP_STATUS_DELETED,
             ]
         )
-        for dump in dumps:
-            if dump not in get_objects_for_user(request.user, "website.can_see"):
-                return JsonResponse({"status_code": 403, "error": "Unauthorized"})
+        dump_ok = get_objects_for_user(request.user, "website.can_see")
+        indexes_ok = [dump.index for dump in dumps if dump in dump_ok]
         results = (
-            Result.objects.filter(dump__index__in=indexes)
+            Result.objects.filter(dump__index__in=indexes_ok)
             .order_by("plugin__name")
             .distinct()
             .values_list("plugin__name", "plugin__comment")

requirements/base.txt

@@ -1,7 +1,7 @@
 # https://github.com/stub42/pytz
 pytz==2024.1
 # https://github.com/python-pillow/Pillow
-Pillow==10.2.0
+Pillow==10.3.0
 # https://github.com/hynek/argon2_cffi
 argon2-cffi==23.1.0
 # https://github.com/evansd/whitenoise
@@ -20,7 +20,7 @@ python-magic==0.4.27
 # Django
 # ------------------------------------------------------------------------------
 # https://www.djangoproject.com/
-django==5.0.3
+django==5.0.4
 # https://github.com/django/channels
 channels[daphne]==4.0.0
 # https://github.com/django/channels_redis
@@ -37,7 +37,7 @@ django-redis==5.4.0
 gunicorn==21.2.0
 
 
-# Django Rest Framework
+# Django Ninja
 # ------------------------------------------------------------------------------
 # https://github.com/adamchainz/django-cors-headers
 django-cors-headers==4.3.1
@@ -56,8 +56,6 @@ django-file-form==3.6.0
 django-guardian==2.4.0
 # https://github.com/jazzband/django-widget-tweaks
 django-widget-tweaks==1.5.0
-# https://github.com/jmrivas86/django-json-widget
-django-json-widget==1.1.1
 # https://github.com/mrts/django-admin-list-filter-dropdown
 django-admin-list-filter-dropdown==1.0.3
 # https://github.com/ctxis/django-admin-multiple-choice-list-filter
@@ -68,16 +66,16 @@ django-admin-multiple-choice-list-filter==0.1.1
 # https://github.com/elastic/elasticsearch-py
 elasticsearch==8.13.0
 # https://github.com/elastic/elasticsearch-dsl-py
-elasticsearch-dsl==8.12.0
+elasticsearch-dsl==8.13.0
 # https://github.com/jurismarches/luqum
 luqum==0.13.0
 
 # Dask & co
 # ------------------------------------------------------------------------------
 # https://github.com/dask/dask
-dask==2024.3.1
+dask==2024.4.1
 # https://github.com/dask/distributed
-distributed==2024.3.1
+distributed==2024.4.1
 # https://msgpack.org/ TO BE ALIGNED WITH SCHEDULER
 msgpack==1.0.8
 # https://github.com/python-lz4/python-lz4
@@ -90,13 +88,15 @@ cloudpickle==3.0.0
 toolz==0.12.1
 # https://pypi.org/project/tornado/
 tornado==6.4
+# https://pandas.pydata.org/
+pandas==2.2.1
 
 # Volatility
 # ------------------------------------------------------------------------------
 # https://github.com/volatilityfoundation/
-# -e git+https://github.com/volatilityfoundation/volatility3.git@55dd39f2ba60ffdd2126b7ea011940f0df42815a#egg=volatility3 volatility3
+# volatility3==2.5.2
 # https://github.com/VirusTotal/yara-python
-#yara-python==4.1.0
+# yara-python==4.5.0
 # https://github.com/Viicos/clamdpy
 clamdpy==0.1.0.post1
 # https://github.com/VirusTotal/vt-py
@@ -115,7 +115,7 @@ geoip2==4.8.0
 # Yara
 # ------------------------------------------------------------------------------
 # https://github.com/gitpython-developers/GitPython
-GitPython==3.1.42
+GitPython==3.1.43
 # https://github.com/frostming/marko
 marko==2.0.3
 
@@ -133,10 +133,10 @@ pefile==2023.2.7
 # misp export
 # ------------------------------------------------------------------------------
 # https://github.com/MISP/PyMISP
-pymisp==2.4.187
+pymisp==2.4.188
 
 # ldap
 # ------------------------------------------------------------------------------
 # https://github.com/python-ldap/python-ldap
 python-ldap==3.4.4
-django-auth-ldap==4.6.0
+django-auth-ldap==4.8.0

requirements/local.txt

@@ -7,7 +7,7 @@ mypy==1.9.0
 # https://github.com/typeddjango/django-stubs
 django-stubs==4.2.7
 # https://github.com/pytest-dev/pytest
-pytest==8.1.0
+pytest==8.1.1
 # https://github.com/Frozenball/pytest-sugar
 pytest-sugar==1.0.0
 
@@ -18,7 +18,7 @@ flake8==7.0.0
 # https://github.com/gforcada/flake8-isort
 flake8-isort==6.1.1
 # https://github.com/nedbat/coveragepy
-coverage==7.4.3
+coverage==7.4.4
 # https://github.com/ambv/black
 black==24.3.0
 # https://github.com/PyCQA/pylint-django