Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Scheduling jobs on status page (#1035)
* add job cache by sensor id in create forecasting jobs Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * fixes after review + split cache key by queue type and sensor/asset + use job_cache for scheduling jobs Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Return job data on GET assets/id/status + add tables for forecasting and scheduling jobs on status page Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Styling + annotations Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Fix for job/job.id url on status page Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Better redis connection exception handling Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Back to asset page button Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Review fixes + 'Created at' column to job tables + sort by this column Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Changelog Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Post review fixes Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Fix after spoiled main merge + fix annotations Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * Remove redundant file Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> * add exact enqueue time for users hovering, fix hovering info for job status Signed-off-by: Nicolas Höning <nicolas@seita.nl> --------- Signed-off-by: Nikolai Rozanov <nickolay.rozanov@gmail.com> Signed-off-by: Nicolas Höning <nicolas@seita.nl> Co-authored-by: Nicolas Höning <nicolas@seita.nl>
- Loading branch information
Showing
11 changed files
with
471 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
""" | ||
Logic around storing and retrieving jobs from redis cache. | ||
""" | ||
|
||
from __future__ import annotations | ||
|
||
import redis | ||
|
||
from redis.exceptions import ConnectionError | ||
from rq.job import Job, NoSuchJobError | ||
|
||
|
||
class NoRedisConfigured(Exception): | ||
def __init__(self, message="Redis not configured"): | ||
super().__init__(message) | ||
|
||
|
||
class JobCache: | ||
""" | ||
Class is used for storing jobs and retrieving them from redis cache. | ||
Need it to be able to get jobs for particular asset (and display them on status page). | ||
Keeps cache up to date by removing jobs that are not found in redis - were removed by TTL. | ||
Stores jobs by asset or sensor id, queue and asset or sensor type, cache key can look like this | ||
- forecasting:sensor:1 (forecasting jobs can be stored by sensor only) | ||
- scheduling:sensor:2 | ||
- scheduling:asset:3 | ||
""" | ||
|
||
def __init__(self, connection: redis.Redis): | ||
self.connection = connection | ||
|
||
def _get_cache_key( | ||
self, asset_or_sensor_id: int, queue: str, asset_or_sensor_type: str | ||
) -> str: | ||
return f"{queue}:{asset_or_sensor_type}:{asset_or_sensor_id}" | ||
|
||
def _check_redis_connection(self): | ||
try: | ||
self.connection.ping() # Check if the Redis connection is okay | ||
except (ConnectionError, ConnectionRefusedError): | ||
raise NoRedisConfigured | ||
|
||
def add( | ||
self, | ||
asset_or_sensor_id: int, | ||
job_id: str, | ||
queue: str = None, | ||
asset_or_sensor_type: str = None, | ||
): | ||
self._check_redis_connection() | ||
cache_key = self._get_cache_key(asset_or_sensor_id, queue, asset_or_sensor_type) | ||
self.connection.sadd(cache_key, job_id) | ||
|
||
def _get_job(self, job_id: str) -> Job: | ||
try: | ||
job = Job.fetch(job_id, connection=self.connection) | ||
except NoSuchJobError: | ||
return None | ||
return job | ||
|
||
def get( | ||
self, asset_or_sensor_id: int, queue: str, asset_or_sensor_type: str | ||
) -> list[Job]: | ||
self._check_redis_connection() | ||
|
||
job_ids_to_remove, jobs = list(), list() | ||
cache_key = self._get_cache_key(asset_or_sensor_id, queue, asset_or_sensor_type) | ||
for job_id in self.connection.smembers(cache_key): | ||
job_id = job_id.decode("utf-8") | ||
job = self._get_job(job_id) | ||
# remove job from cache if cant be found - was removed by TTL | ||
if job is None: | ||
job_ids_to_remove.append(job_id) | ||
continue | ||
jobs.append(job) | ||
if job_ids_to_remove: | ||
self.connection.srem(cache_key, *job_ids_to_remove) | ||
return jobs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.