From ace3d01da72edf13c53c139e7c7a3370765d4347 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 15:26:53 +0200 Subject: [PATCH 01/20] Optional Cron expression scheduling - inital commit --- django_q/admin.py | 63 ++++++++++++++++++--------------------------- django_q/cluster.py | 52 +++++++++++++++++++++++-------------- django_q/models.py | 37 +++++++++++++++++++++++--- poetry.lock | 34 +++++++++++++++++++++++- pyproject.toml | 3 ++- requirements.in | 1 + requirements.txt | 4 ++- 7 files changed, 131 insertions(+), 63 deletions(-) diff --git a/django_q/admin.py b/django_q/admin.py index a61fd24a..a1190cfe 100644 --- a/django_q/admin.py +++ b/django_q/admin.py @@ -10,14 +10,7 @@ class TaskAdmin(admin.ModelAdmin): """model admin for success tasks.""" - list_display = ( - 'name', - 'func', - 'started', - 'stopped', - 'time_taken', - 'group' - ) + list_display = ("name", "func", "started", "stopped", "time_taken", "group") def has_add_permission(self, request): """Don't allow adds.""" @@ -28,9 +21,9 @@ def get_queryset(self, request): qs = super(TaskAdmin, self).get_queryset(request) return qs.filter(success=True) - search_fields = ('name', 'func', 'group') + search_fields = ("name", "func", "group") readonly_fields = [] - list_filter = ('group',) + list_filter = ("group",) def get_readonly_fields(self, request, obj=None): """Set all fields readonly.""" @@ -50,21 +43,15 @@ def retry_failed(FailAdmin, request, queryset): class FailAdmin(admin.ModelAdmin): """model admin for failed tasks.""" - list_display = ( - 'name', - 'func', - 'started', - 'stopped', - 'short_result' - ) + list_display = ("name", "func", "started", "stopped", "short_result") def has_add_permission(self, request): """Don't allow adds.""" return False actions = [retry_failed] - search_fields = ('name', 'func') - list_filter = ('group',) + search_fields = ("name", "func") + list_filter = ("group",) readonly_fields = [] def get_readonly_fields(self, request, obj=None): @@ -76,31 +63,31 @@ class ScheduleAdmin(admin.ModelAdmin): """ model admin for schedules """ list_display = ( - 'id', - 'name', - 'func', - 'schedule_type', - 'repeats', - 'next_run', - 'last_run', - 'success' + "id", + "name", + "func", + "schedule_type", + "repeats", + "next_run", + "last_run", + "success", ) - list_filter = ('next_run', 'schedule_type') - search_fields = ('func',) - list_display_links = ('id', 'name') + # optional cron strings + try: + from croniter import croniter + except ImportError: + readonly_fields = ("cron",) + + list_filter = ("next_run", "schedule_type") + search_fields = ("func",) + list_display_links = ("id", "name") class QueueAdmin(admin.ModelAdmin): """ queue admin for ORM broker """ - list_display = ( - 'id', - 'key', - 'task_id', - 'name', - 'func', - 'lock' - ) + + list_display = ("id", "key", "task_id", "name", "func", "lock") def save_model(self, request, obj, form, change): obj.save(using=Conf.ORM) diff --git a/django_q/cluster.py b/django_q/cluster.py index 1b3c993d..0ad51d37 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -1,6 +1,5 @@ -import ast - # Standard +import ast import importlib import signal import socket @@ -9,9 +8,8 @@ from multiprocessing import Event, Process, Value, current_process from time import sleep -# external +# External import arrow - # Django from django import db from django.conf import settings @@ -29,6 +27,12 @@ from django_q.signing import SignedPackage, BadSignature from django_q.status import Stat, Status +# Optional +try: + from croniter import croniter +except ImportError: + croniter = None + class Cluster: def __init__(self, broker: Broker = None): @@ -103,10 +107,10 @@ def is_running(self) -> bool: @property def is_stopping(self) -> bool: return ( - self.stop_event - and self.start_event - and self.start_event.is_set() - and self.stop_event.is_set() + self.stop_event + and self.start_event + and self.start_event.is_set() + and self.stop_event.is_set() ) @property @@ -116,13 +120,13 @@ def has_stopped(self) -> bool: class Sentinel: def __init__( - self, - stop_event, - start_event, - cluster_id, - broker=None, - timeout=Conf.TIMEOUT, - start=True, + self, + stop_event, + start_event, + cluster_id, + broker=None, + timeout=Conf.TIMEOUT, + start=True, ): # Make sure we catch signals for the pool signal.signal(signal.SIGINT, signal.SIG_IGN) @@ -377,7 +381,7 @@ def monitor(result_queue: Queue, broker: Broker = None): def worker( - task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT + task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT ): """ Takes a task from the task queue, tries to execute it and puts the result back in the result queue @@ -552,9 +556,9 @@ def scheduler(broker: Broker = None): try: with db.transaction.atomic(using=Schedule.objects.db): for s in ( - Schedule.objects.select_for_update() - .exclude(repeats=0) - .filter(next_run__lt=timezone.now()) + Schedule.objects.select_for_update() + .exclude(repeats=0) + .filter(next_run__lt=timezone.now()) ): args = () kwargs = {} @@ -591,6 +595,16 @@ def scheduler(broker: Broker = None): next_run = next_run.shift(months=+3) elif s.schedule_type == s.YEARLY: next_run = next_run.shift(years=+1) + elif s.schedule_type == s.CRON: + if not croniter: + raise ImportError( + _( + "Please install croniter to enable cron expressions" + ) + ) + next_run = arrow.get( + croniter(s.cron, timezone.now()).get_next() + ) if Conf.CATCH_UP or next_run > arrow.utcnow(): break # arrow always returns a tz aware datetime, and we don't want diff --git a/django_q/models.py b/django_q/models.py index 86549ed0..7eac0549 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -1,16 +1,26 @@ +# Django from django import get_version +from django.core.exceptions import ValidationError +from django.db import models from django.template.defaultfilters import truncatechars - from django.urls import reverse +from django.utils import timezone from django.utils.html import format_html from django.utils.translation import gettext_lazy as _ -from django.db import models -from django.utils import timezone + +# External from picklefield import PickledObjectField from picklefield.fields import dbsafe_decode +# Local from django_q.signing import SignedPackage +# Optional +try: + from croniter import croniter +except ImportError: + croniter = None + class Task(models.Model): id = models.CharField(max_length=32, primary_key=True, editable=False) @@ -131,6 +141,18 @@ class Meta: proxy = True +# Optional Cron validator +def validate_cron(value): + if not value: + return + if not croniter: + raise ImportError(_("Please install croniter to enable cron expressions")) + try: + croniter.expand(value) + except ValueError as e: + raise ValidationError(e) + + class Schedule(models.Model): name = models.CharField(max_length=100, null=True, blank=True) func = models.CharField(max_length=256, help_text="e.g. module.tasks.function") @@ -152,6 +174,7 @@ class Schedule(models.Model): MONTHLY = "M" QUARTERLY = "Q" YEARLY = "Y" + CRON = "C" TYPE = ( (ONCE, _("Once")), (MINUTES, _("Minutes")), @@ -161,6 +184,7 @@ class Schedule(models.Model): (MONTHLY, _("Monthly")), (QUARTERLY, _("Quarterly")), (YEARLY, _("Yearly")), + (CRON, _("Cron")), ) schedule_type = models.CharField( max_length=1, choices=TYPE, default=TYPE[0][0], verbose_name=_("Schedule Type") @@ -174,6 +198,13 @@ class Schedule(models.Model): next_run = models.DateTimeField( verbose_name=_("Next Run"), default=timezone.now, null=True ) + cron = models.CharField( + max_length=100, + null=True, + blank=True, + validators=[validate_cron], + help_text=_("Cron expression"), + ) task = models.CharField(max_length=100, null=True, editable=False) def success(self): diff --git a/poetry.lock b/poetry.lock index 28d26b9a..55cee75d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -120,6 +120,18 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" version = "0.4.3" +[[package]] +category = "main" +description = "croniter provides iteration for datetime object with cron like format" +name = "croniter" +optional = true +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "0.3.34" + +[package.dependencies] +natsort = "*" +python-dateutil = "*" + [[package]] category = "main" description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." @@ -270,6 +282,18 @@ optional = false python-versions = ">=3.5" version = "8.4.0" +[[package]] +category = "main" +description = "Simple yet flexible natural sorting in Python." +name = "natsort" +optional = true +python-versions = ">=3.4" +version = "7.0.1" + +[package.extras] +fast = ["fastnumbers (>=2.0.0)"] +icu = ["PyICU (>=1.0.0)"] + [[package]] category = "dev" description = "Core utilities for Python packages" @@ -497,7 +521,7 @@ rollbar = ["django-q-rollbar"] sentry = [] [metadata] -content-hash = "d89c929e8b6951968228b1aab1aa5c0a7c014acceb3b986bbaa3e41df57d8b39" +content-hash = "6d89bff8bd465aa4a5facd812172a970bf5edebdd8099141054f72aa63260c2a" python-versions = ">=3.6" [metadata.files] @@ -545,6 +569,10 @@ colorama = [ {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"}, {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"}, ] +croniter = [ + {file = "croniter-0.3.34-py2.py3-none-any.whl", hash = "sha256:15597ef0639f8fbab09cbf8c277fa8c65c8b9dbe818c4b2212f95dbc09c6f287"}, + {file = "croniter-0.3.34.tar.gz", hash = "sha256:7186b9b464f45cf3d3c83a18bc2344cc101d7b9fd35a05f2878437b14967e964"}, +] django = [ {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"}, {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"}, @@ -638,6 +666,10 @@ more-itertools = [ {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"}, {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"}, ] +natsort = [ + {file = "natsort-7.0.1-py3-none-any.whl", hash = "sha256:d3fd728a3ceb7c78a59aa8539692a75e37cbfd9b261d4d702e8016639820f90a"}, + {file = "natsort-7.0.1.tar.gz", hash = "sha256:a633464dc3a22b305df0f27abcb3e83515898aa1fd0ed2f9726c3571a27258cf"}, +] packaging = [ {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, diff --git a/pyproject.toml b/pyproject.toml index 300f4c3c..76ee5726 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-q" -version = "1.2.4" +version = "1.3.0" description = "A multiprocessing distributed task queue for Django" authors = ["Ilan Steemers "] license = "MIT" @@ -52,6 +52,7 @@ django-redis = {version = "^4.12.1", optional = true} iron-mq = {version = "^0.9", optional = true} boto3 = {version = "^1.14.12", optional = true} pymongo = {version = "^3.10.1", optional = true} +croniter = {version = "^0.3.34", optional = true} [tool.poetry.dev-dependencies] pytest = "^5.4.2" diff --git a/requirements.in b/requirements.in index 7f91d4c0..fadec170 100644 --- a/requirements.in +++ b/requirements.in @@ -8,3 +8,4 @@ django-redis iron-mq boto3 pymongo +croniter diff --git a/requirements.txt b/requirements.txt index 1b3634d0..464bd5ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ boto3==1.14.12 # via -r requirements.in botocore==1.17.12 # via boto3, s3transfer certifi==2020.6.20 # via requests chardet==3.0.4 # via requests +croniter==0.3.34 # via -r requirements.in django-picklefield==3.0.1 # via -r requirements.in django-redis==4.12.1 # via -r requirements.in django==3.0.7 # via django-picklefield, django-redis @@ -20,9 +21,10 @@ idna==2.10 # via requests iron-core==1.2.0 # via iron-mq iron-mq==0.9 # via -r requirements.in jmespath==0.10.0 # via boto3, botocore +natsort==7.0.1 # via croniter psutil==5.7.0 # via -r requirements.in pymongo==3.10.1 # via -r requirements.in -python-dateutil==2.8.1 # via arrow, botocore, iron-core +python-dateutil==2.8.1 # via arrow, botocore, croniter, iron-core pytz==2020.1 # via django redis==3.5.3 # via -r requirements.in, django-redis requests==2.24.0 # via iron-core From 1a1597c050e6925aed1fb21a646d9433276ad891 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 17:14:20 +0200 Subject: [PATCH 02/20] Adds cron migration --- .../migrations/0011_auto_20200628_1055.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 django_q/migrations/0011_auto_20200628_1055.py diff --git a/django_q/migrations/0011_auto_20200628_1055.py b/django_q/migrations/0011_auto_20200628_1055.py new file mode 100644 index 00000000..f4997c39 --- /dev/null +++ b/django_q/migrations/0011_auto_20200628_1055.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.7 on 2020-06-28 10:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_q', '0010_auto_20200610_0856'), + ] + + operations = [ + migrations.AddField( + model_name='schedule', + name='cron', + field=models.CharField(blank=True, help_text='Cron expression', max_length=100, null=True), + ), + migrations.AlterField( + model_name='schedule', + name='schedule_type', + field=models.CharField(choices=[('O', 'Once'), ('I', 'Minutes'), ('H', 'Hourly'), ('D', 'Daily'), ('W', 'Weekly'), ('M', 'Monthly'), ('Q', 'Quarterly'), ('Y', 'Yearly'), ('C', 'Cron')], default='O', max_length=1, verbose_name='Schedule Type'), + ), + ] From 1906be92fd0138a2f2b3f52ffeba8358d259a045 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 19:52:24 +0200 Subject: [PATCH 03/20] Adds test for cron --- django_q/tests/test_scheduler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index 45679fae..9f1134dd 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -9,8 +9,8 @@ from django_q.brokers import get_broker from django_q.cluster import pusher, worker, monitor, scheduler from django_q.conf import Conf -from django_q.tasks import Schedule, fetch, schedule as create_schedule from django_q.queues import Queue +from django_q.tasks import Schedule, fetch, schedule as create_schedule @pytest.fixture @@ -91,8 +91,17 @@ def test_scheduler(broker, monkeypatch): schedule_type=Schedule.MINUTES, minutes=10) assert hasattr(minute_schedule, 'pk') is True + # Cron schedule + minute_schedule = create_schedule('django_q.tests.tasks.word_multiply', + 2, + word='django', + schedule_type=Schedule.CRON, + cron="0 22 * * 1-5") + assert hasattr(minute_schedule, 'pk') is True # All other types for t in Schedule.TYPE: + if t == Schedule.CRON: + continue schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, word='django', From 5509fbb4968e0a681d7a9cd2d2ca410ddc1a60fe Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 19:58:03 +0200 Subject: [PATCH 04/20] Fixes test for cron --- django_q/tests/test_scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index 9f1134dd..6b4d5ac4 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -92,12 +92,12 @@ def test_scheduler(broker, monkeypatch): minutes=10) assert hasattr(minute_schedule, 'pk') is True # Cron schedule - minute_schedule = create_schedule('django_q.tests.tasks.word_multiply', - 2, - word='django', - schedule_type=Schedule.CRON, - cron="0 22 * * 1-5") - assert hasattr(minute_schedule, 'pk') is True + cron_schedule = create_schedule('django_q.tests.tasks.word_multiply', + 2, + word='django', + schedule_type=Schedule.CRON, + cron="0 22 * * 1-5") + assert hasattr(cron_schedule, 'pk') is True # All other types for t in Schedule.TYPE: if t == Schedule.CRON: From d24e8298a13ae68da4ea45732f752ea02fa24cac Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 20:06:11 +0200 Subject: [PATCH 05/20] Fixes test for cron --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 69bc9b66..312a6607 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,5 @@ db.sqlite3 .venv .idea djq -node_modules \ No newline at end of file +node_modules +/c.cache/ From 5bf2cd49eb06eac271af549cd16c297f027c53aa Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 20:23:46 +0200 Subject: [PATCH 06/20] Fixes deprecation --- django_q/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 0ad51d37..3903c13c 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -212,7 +212,7 @@ def reincarnate(self, process): if process.timer.value == 0: # only need to terminate on timeout, otherwise we risk destabilizing the queues process.terminate() - logger.warn(_(f"reincarnated worker {process.name} after timeout")) + logger.warning(_(f"reincarnated worker {process.name} after timeout")) elif int(process.timer.value) == -2: logger.info(_(f"recycled worker {process.name}")) else: From a57f7e3f891b9ffdf1302edd81fbc420173655d7 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 20:23:56 +0200 Subject: [PATCH 07/20] Check for nodes --- django_q/brokers/disque.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/django_q/brokers/disque.py b/django_q/brokers/disque.py index 09c6c4b9..9a81abf3 100644 --- a/django_q/brokers/disque.py +++ b/django_q/brokers/disque.py @@ -1,8 +1,11 @@ import random +# External import redis from redis import Redis +# Django +from django.utils.translation import gettext_lazy as _ from django_q.brokers import Broker from django_q.conf import Conf @@ -52,6 +55,8 @@ def info(self) -> str: @staticmethod def get_connection(list_key: str = Conf.PREFIX) -> Redis: + if not Conf.DISQUE_NODES: + raise redis.exceptions.ConnectionError(_("No Disque nodes configured")) # randomize nodes random.shuffle(Conf.DISQUE_NODES) # find one that works @@ -67,4 +72,6 @@ def get_connection(list_key: str = Conf.PREFIX) -> Redis: return redis_client except redis.exceptions.ConnectionError: continue - raise redis.exceptions.ConnectionError("Could not connect to any Disque nodes") + raise redis.exceptions.ConnectionError( + _("Could not connect to any Disque nodes") + ) From a02ded0ade9ea35e6dab92d15b63d38b717b136a Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 28 Jun 2020 20:24:22 +0200 Subject: [PATCH 08/20] Get correct type name --- django_q/tests/test_scheduler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index 6b4d5ac4..e95650b7 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -100,7 +100,7 @@ def test_scheduler(broker, monkeypatch): assert hasattr(cron_schedule, 'pk') is True # All other types for t in Schedule.TYPE: - if t == Schedule.CRON: + if t[0] == Schedule.CRON: continue schedule = create_schedule('django_q.tests.tasks.word_multiply', 2, From 5fe5b5fe371172d4ea005ea43fdc11e06b4d0f1a Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 29 Jun 2020 11:38:50 +0200 Subject: [PATCH 09/20] Adds cron kwarg --- django_q/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/django_q/tasks.py b/django_q/tasks.py index c27eb8b4..27145f12 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -88,6 +88,7 @@ def schedule(func, *args, **kwargs): :param repeats: how many times to repeat. 0=never, -1=always. :param next_run: Next scheduled run. :type next_run: datetime.datetime + :param cron: optional cron expression :param kwargs: function keyword arguments. :return: the schedule object. :rtype: Schedule @@ -98,6 +99,7 @@ def schedule(func, *args, **kwargs): minutes = kwargs.pop("minutes", None) repeats = kwargs.pop("repeats", -1) next_run = kwargs.pop("next_run", timezone.now()) + cron = kwargs.pop("cron", None) # check for name duplicates instead of am unique constraint if name and Schedule.objects.filter(name=name).exists(): @@ -114,6 +116,7 @@ def schedule(func, *args, **kwargs): minutes=minutes, repeats=repeats, next_run=next_run, + cron=cron ) From c89c23a0cf7cb546328ca43f226398262c14394d Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 29 Jun 2020 14:01:36 +0200 Subject: [PATCH 10/20] Adds cron to docs --- README.rst | 6 ++++++ docs/install.rst | 7 +++++++ docs/schedules.rst | 22 ++++++++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3ce58053..22f6bddf 100644 --- a/README.rst +++ b/README.rst @@ -180,6 +180,12 @@ Admin page or directly from your code: repeats=24, next_run=arrow.utcnow().replace(hour=18, minute=0)) + # Use a cron expression + schedule('math.hypot', + 3, 4, + schedule_type=Schedule.CRON, + cron = '0 22 * * 1-5') + For more info check the `Schedules `__ documentation. diff --git a/docs/install.rst b/docs/install.rst index 12f21a69..e3b1363e 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -88,6 +88,13 @@ Optional +- `Croniter `__ is an optional package that is used to parse cron expressions for the scheduler:: + + $ pip install croniter + + + + Add-ons ------- - `django-q-rollbar `__ is a Rollbar error reporter:: diff --git a/docs/schedules.rst b/docs/schedules.rst index 781a3248..5e2a0804 100644 --- a/docs/schedules.rst +++ b/docs/schedules.rst @@ -45,6 +45,13 @@ You can manage them through the :ref:`admin_page` or directly from your code wit repeats=24, next_run=arrow.utcnow().replace(hour=18, minute=0)) + # Use a cron expression + schedule('math.hypot', + 3, 4, + schedule_type=Schedule.CRON, + cron = '0 22 * * 1-5') + + Missed schedules ---------------- @@ -99,8 +106,9 @@ Reference :param args: arguments for the scheduled function. :param str name: An optional name for your schedule. :param str hook: optional result hook function. Dotted strings only. - :param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or :attr:`Schedule.TYPE` + :param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or (C)ron :attr:`Schedule.TYPE` :param int minutes: Number of minutes for the Minutes type. + :param str cron: Cron expression for the Cron type. :param int repeats: Number of times to repeat schedule. -1=Always, 0=Never, n =n. :param datetime next_run: Next or first scheduled execution datetime. :param dict q_options: options passed to async_task for this schedule @@ -140,7 +148,7 @@ Reference .. py:attribute:: TYPE - :attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY` + :attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY`, :attr:`CRON` .. py:attribute:: minutes @@ -148,6 +156,10 @@ Reference The number of minutes the :attr:`MINUTES` schedule should use. Is ignored for other schedule types. + .. py:attribute:: cron + + A cron string describing the schedule. You need the optional `croniter` package installed for this. + .. py:attribute:: repeats Number of times to repeat the schedule. -1=Always, 0=Never, n =n. @@ -208,3 +220,9 @@ Reference `'Y'` only runs once a year. The same caution as with months apply; If you set this to february 29th, it will run on february 28th in the following years. + + .. py:attribute:: CRON + + `'C'` uses the optional `croniter` package to determine a schedule based a cron expression. + + From 85f57527d6d4e8a7546a67d40056e5b7f784f8c0 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 29 Jun 2020 15:28:26 +0200 Subject: [PATCH 11/20] Adds cron to readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 22f6bddf..d06a5b7d 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ Features - Multiprocessing worker pool - Asynchronous tasks -- Scheduled and repeated tasks +- Scheduled, cron and repeated tasks - Signed and compressed packages - Failure and success database or cache - Result hooks, groups and chains From 34293cfcad234f4ae6135bc3d77f9ca0e48eaae5 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 29 Jun 2020 19:08:08 +0200 Subject: [PATCH 12/20] Test cron validator --- django_q/tests/test_scheduler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index e95650b7..bb8187c0 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -98,6 +98,7 @@ def test_scheduler(broker, monkeypatch): schedule_type=Schedule.CRON, cron="0 22 * * 1-5") assert hasattr(cron_schedule, 'pk') is True + assert cron_schedule.full_clean() is None # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: From f0061dccd6c288727f91f549fcb6a1f7c76ac85d Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Mon, 29 Jun 2020 19:08:33 +0200 Subject: [PATCH 13/20] Moves optional import to root --- django_q/__init__.py | 10 ++++++++-- django_q/admin.py | 5 ++--- django_q/apps.py | 2 +- django_q/cluster.py | 7 +------ django_q/models.py | 7 +------ 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/django_q/__init__.py b/django_q/__init__.py index 30d0d2cc..7f95d38a 100644 --- a/django_q/__init__.py +++ b/django_q/__init__.py @@ -1,6 +1,12 @@ VERSION = (1, 2, 4) -default_app_config = 'django_q.apps.DjangoQConfig' +default_app_config = "django_q.apps.DjangoQConfig" -__all__ = ['conf', 'cluster', 'models', 'tasks'] +__all__ = ["conf", "cluster", "models", "tasks", "croniter"] + +# Optional Imports +try: + from croniter import croniter +except ImportError: + croniter = None diff --git a/django_q/admin.py b/django_q/admin.py index a1190cfe..2625728b 100644 --- a/django_q/admin.py +++ b/django_q/admin.py @@ -5,6 +5,7 @@ from django_q.conf import Conf from django_q.models import Success, Failure, Schedule, OrmQ from django_q.tasks import async_task +from django_q import croniter class TaskAdmin(admin.ModelAdmin): @@ -74,9 +75,7 @@ class ScheduleAdmin(admin.ModelAdmin): ) # optional cron strings - try: - from croniter import croniter - except ImportError: + if not croniter: readonly_fields = ("cron",) list_filter = ("next_run", "schedule_type") diff --git a/django_q/apps.py b/django_q/apps.py index 49432887..1f277595 100644 --- a/django_q/apps.py +++ b/django_q/apps.py @@ -4,7 +4,7 @@ class DjangoQConfig(AppConfig): - name = 'django_q' + name = "django_q" verbose_name = Conf.LABEL def ready(self): diff --git a/django_q/cluster.py b/django_q/cluster.py index 3903c13c..7e96c2e6 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -26,12 +26,7 @@ from django_q.signals import pre_execute from django_q.signing import SignedPackage, BadSignature from django_q.status import Stat, Status - -# Optional -try: - from croniter import croniter -except ImportError: - croniter = None +from django_q import croniter class Cluster: diff --git a/django_q/models.py b/django_q/models.py index 7eac0549..8ec59078 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -14,12 +14,7 @@ # Local from django_q.signing import SignedPackage - -# Optional -try: - from croniter import croniter -except ImportError: - croniter = None +from django_q import croniter class Task(models.Model): From dda0c3c44de2d0d8a6e35fa5c1eade31a6674ca2 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Tue, 30 Jun 2020 20:56:11 +0200 Subject: [PATCH 14/20] Test get_ids --- django_q/tests/test_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_q/tests/test_monitor.py b/django_q/tests/test_monitor.py index 5964a318..25a5dd0a 100644 --- a/django_q/tests/test_monitor.py +++ b/django_q/tests/test_monitor.py @@ -4,7 +4,7 @@ from django_q.tasks import async_task from django_q.brokers import get_broker from django_q.cluster import Cluster -from django_q.monitor import monitor, info +from django_q.monitor import monitor, info, get_ids from django_q.status import Stat from django_q.conf import Conf @@ -16,6 +16,7 @@ def test_monitor(monkeypatch): c = Cluster() c.start() stats = monitor(run_once=True) + assert get_ids() is True c.stop() assert len(stats) > 0 found_c = False From d46d08e94ec84f773cd9fb880bbe0c4ee4a70fd4 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Tue, 30 Jun 2020 21:19:15 +0200 Subject: [PATCH 15/20] Adds qinfo ids check --- django_q/tests/test_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_q/tests/test_commands.py b/django_q/tests/test_commands.py index 5a480ccd..e64a65b6 100644 --- a/django_q/tests/test_commands.py +++ b/django_q/tests/test_commands.py @@ -16,3 +16,4 @@ def test_qmonitor(): def test_qinfo(): call_command('qinfo') call_command('qinfo', config=True) + call_command('qinfo', ids=True) From b17fd2468213507c88a93a16f4eb134e301f99b0 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Tue, 30 Jun 2020 21:35:57 +0200 Subject: [PATCH 16/20] Test for no nodes config --- django_q/tests/test_brokers.py | 186 ++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 84 deletions(-) diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index c6f7c1c6..064ecf19 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -11,72 +11,72 @@ def test_broker(monkeypatch): broker = Broker() - broker.enqueue('test') + broker.enqueue("test") broker.dequeue() broker.queue_size() broker.lock_size() broker.purge_queue() - broker.delete('id') + broker.delete("id") broker.delete_queue() - broker.acknowledge('test') + broker.acknowledge("test") broker.ping() broker.info() # stats - assert broker.get_stat('test_1') is None - broker.set_stat('test_1', 'test', 3) - assert broker.get_stat('test_1') == 'test' - assert broker.get_stats('test:*')[0] == 'test' + assert broker.get_stat("test_1") is None + broker.set_stat("test_1", "test", 3) + assert broker.get_stat("test_1") == "test" + assert broker.get_stats("test:*")[0] == "test" # stats with no cache - monkeypatch.setattr(Conf, 'CACHE', 'not_configured') + monkeypatch.setattr(Conf, "CACHE", "not_configured") broker.cache = broker.get_cache() - assert broker.get_stat('test_1') is None - broker.set_stat('test_1', 'test', 3) - assert broker.get_stat('test_1') is None - assert broker.get_stats('test:*') is None + assert broker.get_stat("test_1") is None + broker.set_stat("test_1", "test", 3) + assert broker.get_stat("test_1") is None + assert broker.get_stats("test:*") is None def test_redis(monkeypatch): - monkeypatch.setattr(Conf, 'DJANGO_REDIS', None) + monkeypatch.setattr(Conf, "DJANGO_REDIS", None) broker = get_broker() assert broker.ping() is True assert broker.info() is not None - monkeypatch.setattr(Conf, 'REDIS', {'host': '127.0.0.1', 'port': 7799}) + monkeypatch.setattr(Conf, "REDIS", {"host": "127.0.0.1", "port": 7799}) broker = get_broker() with pytest.raises(Exception): broker.ping() - monkeypatch.setattr(Conf, 'REDIS', 'redis://127.0.0.1:7799') + monkeypatch.setattr(Conf, "REDIS", "redis://127.0.0.1:7799") broker = get_broker() with pytest.raises(Exception): broker.ping() def test_custom(monkeypatch): - monkeypatch.setattr(Conf, 'BROKER_CLASS', 'brokers.redis_broker.Redis') + monkeypatch.setattr(Conf, "BROKER_CLASS", "brokers.redis_broker.Redis") broker = get_broker() assert broker.ping() is True assert broker.info() is not None - assert broker.__class__.__name__ == 'Redis' + assert broker.__class__.__name__ == "Redis" def test_disque(monkeypatch): - monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7711']) + monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7711"]) # check broker - broker = get_broker(list_key='disque_test') + broker = get_broker(list_key="disque_test") assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # async_task - broker.enqueue('test') + broker.enqueue("test") assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] - assert task[1] == 'test' + assert task[1] == "test" broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test - monkeypatch.setattr(Conf, 'RETRY', 1) - broker.enqueue('test') + monkeypatch.setattr(Conf, "RETRY", 1) + broker.enqueue("test") assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 @@ -88,17 +88,17 @@ def test_disque(monkeypatch): sleep(1.5) assert broker.queue_size() == 0 # delete job - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.delete(task_id) assert broker.dequeue() is None # fail - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.fail(task_id) # bulk test for i in range(5): - broker.enqueue('test') - monkeypatch.setattr(Conf, 'BULK', 5) - monkeypatch.setattr(Conf, 'DISQUE_FASTACK', True) + broker.enqueue("test") + monkeypatch.setattr(Conf, "BULK", 5) + monkeypatch.setattr(Conf, "DISQUE_FASTACK", True) tasks = broker.dequeue() for task in tasks: assert task is not None @@ -106,35 +106,46 @@ def test_disque(monkeypatch): # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue - broker.enqueue('test') - broker.enqueue('test') + broker.enqueue("test") + broker.enqueue("test") broker.delete_queue() assert broker.queue_size() == 0 # connection test - monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7798', '127.0.0.1:7799']) + monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7798", "127.0.0.1:7799"]) + with pytest.raises(redis.exceptions.ConnectionError): + broker.get_connection() + # connection test with no nodes + monkeypatch.setattr(Conf, "DISQUE_NODES", None) with pytest.raises(redis.exceptions.ConnectionError): broker.get_connection() -@pytest.mark.skipif(not os.getenv('IRON_MQ_TOKEN'), - reason="requires IronMQ credentials") +@pytest.mark.skipif( + not os.getenv("IRON_MQ_TOKEN"), reason="requires IronMQ credentials" +) def test_ironmq(monkeypatch): - monkeypatch.setattr(Conf, 'IRON_MQ', {'token': os.getenv('IRON_MQ_TOKEN'), - 'project_id': os.getenv('IRON_MQ_PROJECT_ID')}) + monkeypatch.setattr( + Conf, + "IRON_MQ", + { + "token": os.getenv("IRON_MQ_TOKEN"), + "project_id": os.getenv("IRON_MQ_PROJECT_ID"), + }, + ) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None # initialize the queue - broker.enqueue('test') + broker.enqueue("test") # clear before we start broker.purge_queue() assert broker.queue_size() == 0 # async_task - broker.enqueue('test') + broker.enqueue("test") # dequeue task = broker.dequeue()[0] - assert task[1] == 'test' + assert task[1] == "test" broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test @@ -148,16 +159,16 @@ def test_ironmq(monkeypatch): # broker.acknowledge(task[0]) # sleep(3) # delete job - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.delete(task_id) assert broker.dequeue() is None # fail - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.fail(task_id) # bulk test for i in range(5): - broker.enqueue('test') - monkeypatch.setattr(Conf, 'BULK', 5) + broker.enqueue("test") + monkeypatch.setattr(Conf, "BULK", 5) tasks = broker.dequeue() for task in tasks: assert task is not None @@ -165,62 +176,69 @@ def test_ironmq(monkeypatch): # duplicate acknowledge broker.acknowledge(task[0]) # delete queue - broker.enqueue('test') - broker.enqueue('test') + broker.enqueue("test") + broker.enqueue("test") broker.purge_queue() assert broker.dequeue() is None broker.delete_queue() -@pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'), - reason="requires AWS credentials") +@pytest.mark.skipif( + not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials" +) def canceled_sqs(monkeypatch): - monkeypatch.setattr(Conf, 'SQS', {'aws_region': os.getenv('AWS_REGION'), - 'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'), - 'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')}) + monkeypatch.setattr( + Conf, + "SQS", + { + "aws_region": os.getenv("AWS_REGION"), + "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"), + "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"), + }, + ) # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None assert broker.queue_size() == 0 # async_task - broker.enqueue('test') + broker.enqueue("test") # dequeue task = broker.dequeue()[0] - assert task[1] == 'test' + assert task[1] == "test" broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test - monkeypatch.setattr(Conf, 'RETRY', 1) - broker.enqueue('test') + monkeypatch.setattr(Conf, "RETRY", 1) + broker.enqueue("test") sleep(2) # Sometimes SQS is not linear task = broker.dequeue() if not task: - pytest.skip('SQS being weird') + pytest.skip("SQS being weird") task = task[0] assert len(task) > 0 broker.acknowledge(task[0]) sleep(2) # delete job - monkeypatch.setattr(Conf, 'RETRY', 60) - broker.enqueue('test') + monkeypatch.setattr(Conf, "RETRY", 60) + broker.enqueue("test") sleep(1) task = broker.dequeue() if not task: - pytest.skip('SQS being weird') + pytest.skip("SQS being weird") task_id = task[0][0] broker.delete(task_id) assert broker.dequeue() is None # fail - broker.enqueue('test') + broker.enqueue("test") while task is None: task = broker.dequeue()[0] broker.fail(task[0]) # bulk test for i in range(10): - broker.enqueue('test') - monkeypatch.setattr(Conf, 'BULK', 12) + broker.enqueue("test") + monkeypatch.setattr(Conf, "BULK", 12) tasks = broker.dequeue() for task in tasks: assert task is not None @@ -229,31 +247,31 @@ def canceled_sqs(monkeypatch): broker.acknowledge(task[0]) assert broker.lock_size() == 0 # delete queue - broker.enqueue('test') + broker.enqueue("test") broker.purge_queue() broker.delete_queue() @pytest.mark.django_db def test_orm(monkeypatch): - monkeypatch.setattr(Conf, 'ORM', 'default') + monkeypatch.setattr(Conf, "ORM", "default") # check broker - broker = get_broker(list_key='orm_test') + broker = get_broker(list_key="orm_test") assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # async_task - broker.enqueue('test') + broker.enqueue("test") assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] - assert task[1] == 'test' + assert task[1] == "test" broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test - monkeypatch.setattr(Conf, 'RETRY', 1) - broker.enqueue('test') + monkeypatch.setattr(Conf, "RETRY", 1) + broker.enqueue("test") assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 @@ -265,16 +283,16 @@ def test_orm(monkeypatch): sleep(1.5) assert broker.queue_size() == 0 # delete job - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.delete(task_id) assert broker.dequeue() is None # fail - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.fail(task_id) # bulk test for i in range(5): - broker.enqueue('test') - monkeypatch.setattr(Conf, 'BULK', 5) + broker.enqueue("test") + monkeypatch.setattr(Conf, "BULK", 5) tasks = broker.dequeue() assert broker.lock_size() == Conf.BULK for task in tasks: @@ -285,32 +303,32 @@ def test_orm(monkeypatch): # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue - broker.enqueue('test') - broker.enqueue('test') + broker.enqueue("test") + broker.enqueue("test") broker.delete_queue() assert broker.queue_size() == 0 @pytest.mark.django_db def test_mongo(monkeypatch): - monkeypatch.setattr(Conf, 'MONGO', {'host': '127.0.0.1', 'port': 27017}) + monkeypatch.setattr(Conf, "MONGO", {"host": "127.0.0.1", "port": 27017}) # check broker - broker = get_broker(list_key='mongo_test') + broker = get_broker(list_key="mongo_test") assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # async_task - broker.enqueue('test') + broker.enqueue("test") assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] - assert task[1] == 'test' + assert task[1] == "test" broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test - monkeypatch.setattr(Conf, 'RETRY', 1) - broker.enqueue('test') + monkeypatch.setattr(Conf, "RETRY", 1) + broker.enqueue("test") assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 @@ -322,15 +340,15 @@ def test_mongo(monkeypatch): sleep(1.5) assert broker.queue_size() == 0 # delete job - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.delete(task_id) assert broker.dequeue() is None # fail - task_id = broker.enqueue('test') + task_id = broker.enqueue("test") broker.fail(task_id) # bulk test for i in range(5): - broker.enqueue('test') + broker.enqueue("test") tasks = [] for i in range(5): tasks.append(broker.dequeue()[0]) @@ -343,8 +361,8 @@ def test_mongo(monkeypatch): # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue - broker.enqueue('test') - broker.enqueue('test') + broker.enqueue("test") + broker.enqueue("test") broker.purge_queue() broker.delete_queue() assert broker.queue_size() == 0 From e6f82c51587198f6e07b4a6d115838160bfe939a Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Tue, 30 Jun 2020 22:00:31 +0200 Subject: [PATCH 17/20] test model unicode --- django_q/core_signing.py | 10 +++++----- django_q/management/commands/qmonitor.py | 10 +++++----- django_q/tests/test_cluster.py | 2 ++ django_q/tests/test_scheduler.py | 1 + 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/django_q/core_signing.py b/django_q/core_signing.py index e2ceb9d2..923ea289 100644 --- a/django_q/core_signing.py +++ b/django_q/core_signing.py @@ -24,11 +24,11 @@ def loads( - s, - key=None, - salt: str = "django.core.signing", - serializer=JSONSerializer, - max_age=None, + s, + key=None, + salt: str = "django.core.signing", + serializer=JSONSerializer, + max_age=None, ): """ Reverse of dumps(), raise BadSignature if signature fails. diff --git a/django_q/management/commands/qmonitor.py b/django_q/management/commands/qmonitor.py index bfb9f696..ca925a03 100644 --- a/django_q/management/commands/qmonitor.py +++ b/django_q/management/commands/qmonitor.py @@ -10,12 +10,12 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--run-once', - action='store_true', - dest='run_once', + "--run-once", + action="store_true", + dest="run_once", default=False, - help='Run once and then stop.', + help="Run once and then stop.", ) def handle(self, *args, **options): - monitor(run_once=options.get('run_once', False)) + monitor(run_once=options.get("run_once", False)) diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index 90930365..bd777587 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -142,6 +142,8 @@ def test_enqueue(broker, admin_user): # q_options and save opt_out test k = async_task('django_q.tests.tasks.get_user_id', admin_user, q_options={'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90}) + # test unicode + assert Task(name='Amalia').__unicode__()=='Amalia' # check if everything has a task id assert isinstance(a, str) assert isinstance(b, str) diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index bb8187c0..7cd700e3 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -99,6 +99,7 @@ def test_scheduler(broker, monkeypatch): cron="0 22 * * 1-5") assert hasattr(cron_schedule, 'pk') is True assert cron_schedule.full_clean() is None + assert cron_schedule.__unicode__() == 'django_q.tests.tasks.word_multiply' # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: From f48e26fbf9a3c6a72ad429d3ffe027dfd9aae0d9 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 1 Jul 2020 13:39:51 +0200 Subject: [PATCH 18/20] Updates django --- .travis.yml | 6 +++--- poetry.lock | 20 ++++++++++---------- requirements.txt | 6 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd2ab3a6..216295f9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python services: - - redis-server + - redis - mongodb python: @@ -9,8 +9,8 @@ python: - "3.8" env: - - DJANGO=3.0.7 - - DJANGO=2.2.13 + - DJANGO=3.0.8 + - DJANGO=2.2.14 sudo: true dist: xenial diff --git a/poetry.lock b/poetry.lock index 55cee75d..be271084 100644 --- a/poetry.lock +++ b/poetry.lock @@ -71,10 +71,10 @@ description = "The AWS SDK for Python" name = "boto3" optional = true python-versions = "*" -version = "1.14.12" +version = "1.14.14" [package.dependencies] -botocore = ">=1.17.12,<1.18.0" +botocore = ">=1.17.14,<1.18.0" jmespath = ">=0.7.1,<1.0.0" s3transfer = ">=0.3.0,<0.4.0" @@ -84,7 +84,7 @@ description = "Low-level, data-driven core of boto 3." name = "botocore" optional = true python-versions = "*" -version = "1.17.12" +version = "1.17.14" [package.dependencies] docutils = ">=0.10,<0.16" @@ -138,7 +138,7 @@ description = "A high-level Python Web framework that encourages rapid developme name = "django" optional = false python-versions = ">=3.6" -version = "3.0.7" +version = "3.0.8" [package.dependencies] asgiref = ">=3.2,<4.0" @@ -550,12 +550,12 @@ blessed = [ {file = "blessed-1.17.8.tar.gz", hash = "sha256:7671d057b2df6ddbefd809009fb08feb2f8d2d163d240b5e765088a90519b2f1"}, ] boto3 = [ - {file = "boto3-1.14.12-py2.py3-none-any.whl", hash = "sha256:7daad26a008c91dd7b82fde17d246d1fe6e4b3813426689ef8bac9017a277cfb"}, - {file = "boto3-1.14.12.tar.gz", hash = "sha256:2616351c98eec18d20a1d64b33355c86cd855ac96219d1b8428c9bfc590bde53"}, + {file = "boto3-1.14.14-py2.py3-none-any.whl", hash = "sha256:4c2f5f9f28930e236845e2cddbe01cb093ca96dc1f5c6e2b2b254722018a2268"}, + {file = "boto3-1.14.14.tar.gz", hash = "sha256:87beffba2360b8077413f2d473cb828d0a5bda513bd1d6fb7b137c57b686aeb6"}, ] botocore = [ - {file = "botocore-1.17.12-py2.py3-none-any.whl", hash = "sha256:45934d880378777cefeca727f369d1f5aebf6b254e9be58e7c77dd0b059338bb"}, - {file = "botocore-1.17.12.tar.gz", hash = "sha256:a94e0e2307f1b9fe3a84660842909cd2680b57a9fc9fb0c3a03b0afb2eadbe21"}, + {file = "botocore-1.17.14-py2.py3-none-any.whl", hash = "sha256:6a2e9768dad8ae9771302d5922b977dca6bb9693f9b6a5f6ed0e7ac375e2ca40"}, + {file = "botocore-1.17.14.tar.gz", hash = "sha256:96d668ae5246d236ea83e4586349552d6584e8b1551ae2fccc0bd4ed528a746f"}, ] certifi = [ {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"}, @@ -574,8 +574,8 @@ croniter = [ {file = "croniter-0.3.34.tar.gz", hash = "sha256:7186b9b464f45cf3d3c83a18bc2344cc101d7b9fd35a05f2878437b14967e964"}, ] django = [ - {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"}, - {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"}, + {file = "Django-3.0.8-py3-none-any.whl", hash = "sha256:5457fc953ec560c5521b41fad9e6734a4668b7ba205832191bbdff40ec61073c"}, + {file = "Django-3.0.8.tar.gz", hash = "sha256:31a5fbbea5fc71c99e288ec0b2f00302a0a92c44b13ede80b73a6a4d6d205582"}, ] django-picklefield = [ {file = "django-picklefield-3.0.1.tar.gz", hash = "sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287"}, diff --git a/requirements.txt b/requirements.txt index 464bd5ec..73467fdb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,14 +7,14 @@ arrow==0.15.7 # via -r requirements.in asgiref==3.2.10 # via django blessed==1.17.8 # via -r requirements.in -boto3==1.14.12 # via -r requirements.in -botocore==1.17.12 # via boto3, s3transfer +boto3==1.14.14 # via -r requirements.in +botocore==1.17.14 # via boto3, s3transfer certifi==2020.6.20 # via requests chardet==3.0.4 # via requests croniter==0.3.34 # via -r requirements.in django-picklefield==3.0.1 # via -r requirements.in django-redis==4.12.1 # via -r requirements.in -django==3.0.7 # via django-picklefield, django-redis +django==3.0.8 # via django-picklefield, django-redis docutils==0.15.2 # via botocore hiredis==1.0.1 # via -r requirements.in idna==2.10 # via requests From 532199fff3ed872662efbc8ced4f7fac2919ac02 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 1 Jul 2020 14:18:04 +0200 Subject: [PATCH 19/20] Trigger cron validation --- django_q/tasks.py | 8 ++++++-- django_q/tests/test_scheduler.py | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/django_q/tasks.py b/django_q/tasks.py index 27145f12..1e6ca3c8 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -106,7 +106,7 @@ def schedule(func, *args, **kwargs): raise IntegrityError("A schedule with the same name already exists.") # create and return the schedule - return Schedule.objects.create( + s = Schedule( name=name, func=func, hook=hook, @@ -116,8 +116,12 @@ def schedule(func, *args, **kwargs): minutes=minutes, repeats=repeats, next_run=next_run, - cron=cron + cron=cron, ) + # make sure we trigger validation + s.full_clean() + s.save() + return s def result(task_id, wait=0, cached=Conf.CACHED): diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index 7cd700e3..4625bfaa 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -3,6 +3,7 @@ import arrow import pytest +from django.core.exceptions import ValidationError from django.db import IntegrityError from django.utils import timezone @@ -100,6 +101,12 @@ def test_scheduler(broker, monkeypatch): assert hasattr(cron_schedule, 'pk') is True assert cron_schedule.full_clean() is None assert cron_schedule.__unicode__() == 'django_q.tests.tasks.word_multiply' + with pytest.raises(ValidationError): + cron_schedule = create_schedule('django_q.tests.tasks.word_multiply', + 2, + word='django', + schedule_type=Schedule.CRON, + cron="0 22 * * 1-12") # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: From f223cfcb30414f0a4c2c010f3f240d3a88972f2a Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 1 Jul 2020 14:25:51 +0200 Subject: [PATCH 20/20] Removing value check --- django_q/models.py | 2 -- django_q/tests/test_scheduler.py | 10 +++++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/django_q/models.py b/django_q/models.py index 8ec59078..f2ba852e 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -138,8 +138,6 @@ class Meta: # Optional Cron validator def validate_cron(value): - if not value: - return if not croniter: raise ImportError(_("Please install croniter to enable cron expressions")) try: diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py index 4625bfaa..50f6c5b8 100644 --- a/django_q/tests/test_scheduler.py +++ b/django_q/tests/test_scheduler.py @@ -102,11 +102,11 @@ def test_scheduler(broker, monkeypatch): assert cron_schedule.full_clean() is None assert cron_schedule.__unicode__() == 'django_q.tests.tasks.word_multiply' with pytest.raises(ValidationError): - cron_schedule = create_schedule('django_q.tests.tasks.word_multiply', - 2, - word='django', - schedule_type=Schedule.CRON, - cron="0 22 * * 1-12") + create_schedule('django_q.tests.tasks.word_multiply', + 2, + word='django', + schedule_type=Schedule.CRON, + cron="0 22 * * 1-12") # All other types for t in Schedule.TYPE: if t[0] == Schedule.CRON: