Merge pull request #34 from surface-security/develop

Release 1.1.0

.github/workflows/test.yml

@@ -4,8 +4,10 @@
 name: tests
 
 on:
-  push:
   pull_request:
+    branches:
+      - main
+      - develop
 
 jobs:
   style:

CODEOWNERS

@@ -0,0 +1 @@
+* @surface-security/core

dkron/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.0.5'
+__version__ = '1.1.0'
 
 # set default_app_config when using django earlier than 3.2
 try:

dkron/management/commands/run_dkron.py

@@ -5,10 +5,10 @@
 import shutil
 import tarfile
 from pathlib import Path
-
 from django.conf import settings
 
 from logbasecommand.base import LogBaseCommand
+from dkron import utils
 
 
 class Command(LogBaseCommand):
@@ -31,45 +31,17 @@ def add_arguments(self, parser):
         )
 
     def download_dkron(self):
-        """
-        this needs to map platform to the filenames used by dkron (goreleaser):
-
-        docker run --rm fopina/wine-python:3 -c 'import platform;print(platform.system(),platform.machine())'
-        Windows AMD64
-        python -c 'import platform;print(platform.system(),platform.machine())'
-        Darwin x86_64
-        docker run --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
-        Linux x86_64
-        docker run --platform linux/arm64 --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
-        Linux aarch64
-        docker run --platform linux/arm/v7 --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
-        Linux armv7l
-        """
         if settings.DKRON_BIN_DIR is None:
             bin_dir = Path(tempfile.mkdtemp())
         else:
             bin_dir = Path(settings.DKRON_BIN_DIR) / settings.DKRON_VERSION
 
-        system = platform.system().lower()
+        dl_url, system, _ = utils.dkron_binary_download_url()
         exe_path = bin_dir / ('dkron.exe' if system == 'windows' else 'dkron')
-        machine = platform.machine().lower()
-
-        if 'arm' in machine or 'aarch' in machine:
-            if '64' in machine:
-                machine = 'arm64'
-            else:
-                machine = 'armv7'
-        else:
-            machine = 'amd64'
 
         # check if download is required
         if not exe_path.is_file():
             os.makedirs(bin_dir, exist_ok=True)
-            dl_url = settings.DKRON_DOWNLOAD_URL_TEMPLATE.format(
-                version=settings.DKRON_VERSION,
-                system=system,
-                machine=machine,
-            )
             tarball = f'{exe_path}.tar.gz'
             self.log(f'Downloading {dl_url}')
             with requests.get(dl_url, stream=True) as r:
@@ -83,7 +55,6 @@ def download_dkron(self):
         return str(exe_path), str(bin_dir)
 
     def handle(self, *_, **options):
-        # TODO: check if there's any shutdown we should care before execv()
         exe_path, bin_dir = self.download_dkron()
 
         args = [exe_path, 'agent']
@@ -114,9 +85,10 @@ def handle(self, *_, **options):
         if settings.DKRON_WORKDIR:
             os.chdir(settings.DKRON_WORKDIR)
         if settings.DKRON_WEBHOOK_URL and settings.DKRON_TOKEN:
+            flag_name = '--webhook-url' if utils.dkron_binary_version() < (3, 2, 0) else '--webhook-endpoint'
             args.extend(
                 [
-                    '--webhook-url',
+                    flag_name,
                     settings.DKRON_WEBHOOK_URL,
                     '--webhook-payload',
                     f'{settings.DKRON_TOKEN}\n{{{{ .JobName }}}}\n{{{{ .Success }}}}',

dkron/management/commands/run_dkron_async_command.py

@@ -0,0 +1,24 @@
+import json
+from django.core.management import call_command
+from logbasecommand.base import LogBaseCommand
+
+import json
+import base64
+
+
+class Command(LogBaseCommand):
+    help = 'Hidden command'
+
+    def add_arguments(self, parser):
+        parser.add_argument('command', help='Run in server mode')
+        parser.add_argument('arguments', nargs='?', help='Port used by the web UI')
+
+    def handle(self, *_, **options):
+        args = []
+        kwargs = {}
+        if options['arguments']:
+            arguments = json.loads(base64.b64decode(options['arguments']))
+            args = arguments.get('args') or []
+            kwargs = arguments.get('kwargs') or {}
+
+        call_command(options['command'], *args, **kwargs, stdout=options.get('stdout'), stderr=options.get('stderr'))

dkron/migrations/0001_initial_20210729.py

@@ -8,7 +8,6 @@ def data_fix(apps, schema_editor):
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     replaces = [

dkron/migrations/0002_job_retries.py

@@ -4,7 +4,6 @@
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ('dkron', '0001_initial_20210729'),
     ]

dkron/utils.py

@@ -1,9 +1,13 @@
 from collections import defaultdict
 import logging
+import platform
 import time
 from typing import Iterator, Literal, Optional, Union
 import requests
 from functools import lru_cache
+import re
+import json
+import base64
 
 from django.conf import settings
 from django.core.management import call_command
@@ -13,6 +17,8 @@
 
 logger = logging.getLogger(__name__)
 
+UNKNOWN_DKRON_VERSION = (9999, 9, 9)
+
 
 class DkronException(Exception):
     def __init__(self, code, message) -> None:
@@ -48,6 +54,58 @@ def namespace_prefix():
     return f'{k}_'
 
 
+def dkron_binary_download_url():
+    """
+    Returns a tuple with (dkron binary download URL, system type, machine type)
+    """
+
+    # this needs to map platform to the filenames used by dkron (goreleaser):
+    #
+    # docker run --rm fopina/wine-python:3 -c 'import platform;print(platform.system(),platform.machine())'
+    # Windows AMD64
+    # python -c 'import platform;print(platform.system(),platform.machine())'
+    # Darwin x86_64
+    # docker run --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
+    # Linux x86_64
+    # docker run --platform linux/arm64 --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
+    # Linux aarch64
+    # docker run --platform linux/arm/v7 --rm python:3-alpine python -c 'import platform;print(platform.system(),platform.machine())'
+    # Linux armv7l
+
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if 'arm' in machine or 'aarch' in machine:
+        if '64' in machine:
+            machine = 'arm64'
+        else:
+            machine = 'armv7'
+    else:
+        machine = 'amd64'
+
+    dl_url = settings.DKRON_DOWNLOAD_URL_TEMPLATE.format(
+        version=settings.DKRON_VERSION,
+        system=system,
+        machine=machine,
+    )
+
+    return dl_url, system, machine
+
+
+@lru_cache
+def dkron_binary_version():
+    """
+    Return version of dkron binary in settings (based on DKRON_VERSION) as a standard version tuple
+    """
+    m = re.match(r'.*(\d+)\.(\d+)\.(\d+)', settings.DKRON_VERSION)
+    if m:
+        return tuple(map(int, m.groups()))
+    logger.warning(
+        'unable to identify dkron version from DKRON_VERSION="%s" - handling it as latest', settings.DKRON_VERSION
+    )
+    return UNKNOWN_DKRON_VERSION
+
+
 def add_namespace(job_name):
     if not job_name:
         return ''
@@ -226,51 +284,41 @@ def resync_jobs() -> Iterator[tuple[str, Literal["u", "d"], Optional[str]]]:
     import after_response
 
     @after_response.enable
-    def __run_async(command, *args, **kwargs) -> str:
-        return call_command(command, *args, **kwargs)
+    def __run_async(_command, *args, **kwargs) -> str:
+        return call_command(_command, *args, **kwargs)
 
 except ImportError:
 
-    def __run_async(command, *args, **kwargs):
+    def __run_async(_command, *args, **kwargs):
         raise DkronException('dkron is down and after_response is not installed')
 
 
-def __run_async_dkron(command, *args, **kwargs) -> tuple[str, str]:
-    final_command = f'python ./manage.py {command}'
-
-    # FIXME code very likely to NOT work in some cases :P
-    if args:
-        final_command += ' ' + ' '.join(map(str, args))
-    if kwargs:
-        for k in kwargs:
-            val = kwargs[k]
-            k = k.replace("_", "-")
-
-            if isinstance(val, bool):
-                if val is True:
-                    final_command += f' --{k}'
-            else:
-                if isinstance(val, (list, tuple)):
-                    for v in val:
-                        final_command += f' --{k} {v}'
-                else:
-                    final_command += f' --{k} {val}'
-
-    name = f'tmp_{command}_{time.time():.0f}'
+def __run_async_dkron(_command, *args, **kwargs) -> tuple[str, str]:
+    arguments = base64.b64encode(json.dumps({'args': args, 'kwargs': kwargs}).encode()).decode()
+    final_command = f'python ./manage.py run_dkron_async_command {_command} {arguments}'
+
+    name = f'tmp_{_command}_{time.time():.0f}'
+
+    if dkron_binary_version() >= (3, 2, 2):
+        # runoncreate was turned into asynchronous in https://github.com/distribworks/dkron/pull/1269
+        schedule = '@manually'
+        params = {'runoncreate': 'true'}
+    else:
+        schedule = f'@at {(timezone.now() + timezone.timedelta(seconds=5)).isoformat()}'
+        params = {}
+
     r = _post(
         'jobs',
         json={
             'name': add_namespace(name),
-            'schedule': f'@at {(timezone.now() + timezone.timedelta(seconds=5)).isoformat()}',
+            'schedule': schedule,
             'executor': 'shell',
             'tags': {'label': f'{settings.DKRON_JOB_LABEL}:1'} if settings.DKRON_JOB_LABEL else {},
             'metadata': {'temp': 'true'},
             'disabled': False,
             'executor_config': {'command': final_command},
         },
-        # FIXME: workaround for https://github.com/surface-security/django-dkron/issues/18
-        # if dkron fixes it, restore this (either based on dkron version or ignore the bug for old version...)
-        # params={'runoncreate': 'true'},
+        params=params,
     )
 
     if r.status_code != 201:
@@ -283,9 +331,9 @@ def job_executions(job_name):
     return f'{settings.DKRON_PATH}#/jobs/{add_namespace(job_name)}/show/executions'
 
 
-def run_async(command, *args, **kwargs) -> Union[tuple[str, str], str]:
+def run_async(_command, *args, **kwargs) -> Union[tuple[str, str], str]:
     try:
-        return __run_async_dkron(command, *args, **kwargs)
+        return __run_async_dkron(_command, *args, **kwargs)
     except requests.ConnectionError:
         # if dkron not available, use after_response
-        return __run_async.after_response(command, *args, **kwargs)
+        return __run_async.after_response(_command, *args, **kwargs)

setup.cfg

@@ -24,9 +24,9 @@ zip_safe = False
 include_package_data = True
 packages = find:
 # requires 3.9 due to django-notification-sender...
-python_requires = >=3.9, < 4
+python_requires = >=3.9
 install_requires =
-    Django >= 3.0
+    Django >= 3.0, < 4
     django-logbasecommand < 1
     django-notification-sender < 1
     requests > 2, < 3

testapp/testapp/settings.py

@@ -125,6 +125,8 @@
 DKRON_URL = 'http://localhost:8888/'
 DKRON_JOB_LABEL = 'testapp'
 DKRON_SERVER = True
+# to switch a different version than default (currently 3.1.10)
+# DKRON_VERSION= '3.2.2'
 
 # DKRON_PATH is meant to be setup in an nginx location before the main app, so it can re-use app authentication (and authz) to access dkron (which has no authentication)
 # as defined in nginx proxypass location + dashboard to avoid redirect such as: