Manually delete unused keys from redis activity cache

bookwyrm/settings.py

@@ -242,6 +242,7 @@
 # timeout for a query to an individual connector
 QUERY_TIMEOUT = env.int("INTERACTIVE_QUERY_TIMEOUT", env.int("QUERY_TIMEOUT", 5))
 
+CACHE_KEY_PREFIX = "django_cache"
 # Redis cache backend
 if env.bool("USE_DUMMY_CACHE", False):
     CACHES = {
@@ -258,6 +259,7 @@
         "default": {
             "BACKEND": "django_redis.cache.RedisCache",
             "LOCATION": REDIS_ACTIVITY_URL,
+            "KEY_PREFIX": CACHE_KEY_PREFIX,
             "OPTIONS": {
                 "CLIENT_CLASS": "django_redis.client.DefaultClient",
             },

bookwyrm/templates/settings/layout.html

@@ -85,6 +85,10 @@ <h2 class="menu-label">{% trans "System" %}</h2>
                 {% url 'settings-celery' as url %}
                 <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Celery status" %}</a>
             </li>
+            <li>
+                {% url 'settings-redis' as url %}
+                <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Redis status" %}</a>
+            </li>
             <li>
                 {% url 'settings-schedules' as url %}
                 <a href="{{ url }}"{% if url in request.path %} class="is-active" aria-selected="true"{% endif %}>{% trans "Scheduled tasks" %}</a>

bookwyrm/templates/settings/redis.html

@@ -0,0 +1,68 @@
+{% extends 'settings/layout.html' %}
+{% load humanize %}
+{% load i18n %}
+
+{% block title %}{% trans "Redis Status" %}{% endblock %}
+
+{% block header %}{% trans "Redis Status" %}{% endblock %}
+
+{% block panel %}
+
+{% if info %}
+<section class="block content">
+    <h2>{% trans "Info" %}</h2>
+    <div class="columns has-text-centered is-multiline">
+        <div class="column is-4">
+            <div class="notification">
+                <p class="header">{% trans "Used memory" %}</p>
+                <p class="title is-5">{{ info.used_memory_human }}</p>
+            </div>
+        </div>
+        <div class="column is-4">
+            <div class="notification">
+                <p class="header">{% trans "Total system memory" %}</p>
+                <p class="title is-5">{{ info.total_system_memory_human }}</p>
+            </div>
+        </div>
+        <div class="column is-4">
+            <div class="notification">
+                <p class="header">{% trans "Keys" %}</p>
+                <p class="title is-5">{{ info.db0.keys | intcomma }}</p>
+            </div>
+        </div>
+    </div>
+</section>
+
+<section>
+    {{ dead_key_count }}
+    <form name="erase-keys" method="POST" action="{% url 'settings-redis' %}">
+        {% csrf_token %}
+        <button type="submit" class="button">go</button>
+    </form>
+    <form name="erase-keys" method="POST" action="{% url 'settings-redis' %}">
+        {% csrf_token %}
+        <input type="hidden" name="dry_run" value="True">
+        <button type="submit" class="button">dry run</button>
+    </form>
+</section>
+{% else %}
+<div class="notification is-danger is-flex is-align-items-start">
+    <span class="icon icon-warning is-size-4 pr-3" aria-hidden="true"></span>
+    <span>
+        {% trans "Could not connect to Redis Activity" %}
+    </span>
+</div>
+
+{% endif %}
+
+{% if errors %}
+<div class="block content">
+    <h2>{% trans "Errors" %}</h2>
+    {% for error in errors %}
+    <pre>{{ error }}</pre>
+{% endfor %}
+
+</div>
+{% endif %}
+
+{% endblock %}

bookwyrm/urls.py

@@ -369,6 +369,7 @@
     re_path(
         r"^settings/celery/ping/?$", views.celery_ping, name="settings-celery-ping"
     ),
+    re_path(r"^settings/redis/?$", views.RedisStatus.as_view(), name="settings-redis"),
     re_path(
         r"^settings/schedules/(?P<task_id>\d+)?$",
         views.ScheduledTasks.as_view(),

bookwyrm/views/__init__.py

@@ -5,6 +5,7 @@
 from .admin.automod import AutoMod, automod_delete, run_automod
 from .admin.automod import schedule_automod_task, unschedule_automod_task
 from .admin.celery_status import CeleryStatus, celery_ping
+from .admin.redis import RedisStatus
 from .admin.schedule import ScheduledTasks
 from .admin.dashboard import Dashboard
 from .admin.federation import Federation, FederatedServer

bookwyrm/views/admin/redis.py

@@ -0,0 +1,63 @@
+""" redis cache status """
+from django.contrib.auth.decorators import login_required, permission_required
+from django.http import HttpResponse
+from django.template.response import TemplateResponse
+from django.utils.decorators import method_decorator
+from django.views import View
+import redis
+
+from bookwyrm import models, settings
+
+r = redis.from_url(settings.REDIS_ACTIVITY_URL)
+
+# pylint: disable= no-self-use
+@method_decorator(login_required, name="dispatch")
+@method_decorator(
+    permission_required("bookwyrm.edit_instance_settings", raise_exception=True),
+    name="dispatch",
+)
+class RedisStatus(View):
+    """Are your tasks running? Well you'd better go catch them"""
+
+    def get(self, request):
+        """See workers and active tasks"""
+        data = {"errors": []}
+        try:
+            data["info"] = r.info
+        # pylint: disable=broad-except
+        except Exception as err:
+            data["errors"].append(err)
+
+        return TemplateResponse(request, "settings/redis.html", data)
+
+    # pylint: disable=unused-argument
+    def post(self, request):
+        """Erase invalid keys"""
+        dry_run = request.POST.get("dry_run")
+        patterns = [":*:*"]  # this pattern is a django cache with no prefix
+        for user_id in models.User.objects.filter(
+            is_deleted=True, local=True
+        ).values_list("id", flat=True):
+            patterns.append(f"{user_id}-*")
+
+        deleted_count = 0
+        for pattern in patterns:
+            deleted_count += erase_keys(pattern, dry_run=dry_run)
+
+        if dry_run:
+            return HttpResponse(f"{deleted_count} keys identified for deletion")
+        return HttpResponse(f"{deleted_count} keys deleted")
+
+
+def erase_keys(pattern, count=1000, dry_run=False):
+    """Delete all redis activity keys according to a provided regex pattern"""
+    pipeline = r.pipeline()
+    key_count = 0
+    for keys in r.scan_iter(match=pattern, count=count):
+        key_count += len(keys)
+        if not dry_run:
+            for key in keys:
+                pipeline.delete(key)
+    if not dry_run:
+        pipeline.execute()
+    return key_count