Fix/scaleability/load datamodels

lektor/db.py

@@ -1,3 +1,4 @@
+import datetime
 import os
 import errno
 import hashlib
@@ -1111,6 +1112,57 @@ def _iter_datamodel_choices(datamodel_name, path, is_attachment=False):
         yield 'page'
     yield 'none'
 
+class DatabaseCache(object):
+
+    _cache_timeout_seconds = None # search for datamodels_cache_timeout_seconds for initial value
+    _cache_enable = None # search for datamodels_cache_enable
+    _cache_datamodels = None
+    _cache_last_update_datetime = None
+
+    @staticmethod
+    def init_cache_from_config(env):
+        lconfig = env.load_config()
+        cache_enabled = str(lconfig.values['LEKTOR_CACHE'].get('datamodels_cache_enable')).lower()
+        DatabaseCache._cache_enable = \
+            True if cache_enabled in ['true', 'yes', 'on', '1'] else False
+        DatabaseCache._cache_timeout_seconds = \
+            float(lconfig.values['LEKTOR_CACHE'].get('datamodels_cache_timeout_seconds'))
+
+    @staticmethod
+    def get_data_models(env):
+        if not DatabaseCache.is_cache_enabled(env):
+            return load_datamodels(env)
+
+        if DatabaseCache._cache_datamodels and DatabaseCache._cache_last_update_datetime:
+            now = datetime.datetime.now()
+            timedif = now - DatabaseCache._cache_last_update_datetime
+            seconds = timedif.total_seconds()
+            if seconds >= DatabaseCache._cache_timeout_seconds:
+                DatabaseCache._reload_cache(env)
+        else:
+            DatabaseCache._reload_cache(env)
+
+        return DatabaseCache._cache_datamodels
+
+    @staticmethod
+    def purge_cache():
+        DatabaseCache._cache_last_update_datetime = None
+        DatabaseCache._cache_datamodels = None
+
+    @staticmethod
+    def is_cache_enabled(env):
+        if DatabaseCache._cache_enable is None:
+            DatabaseCache.init_cache_from_config(env)
+
+        return DatabaseCache._cache_enable
+
+
+    @staticmethod
+    def _reload_cache(env):
+        DatabaseCache._cache_last_update_datetime = datetime.datetime.now()
+        DatabaseCache._cache_datamodels = load_datamodels(env)
+
+
 
 class Database(object):
 
@@ -1119,7 +1171,7 @@ def __init__(self, env, config=None):
         if config is None:
             config = env.load_config()
         self.config = config
-        self.datamodels = load_datamodels(env)
+        self.datamodels = DatabaseCache.get_data_models(env)
         self.flowblocks = load_flowblocks(env)
 
     def to_fs_path(self, path):

lektor/devserver.py

@@ -5,7 +5,7 @@
 
 from werkzeug.serving import run_simple, WSGIRequestHandler
 
-from lektor.db import Database
+from lektor.db import Database, DatabaseCache
 from lektor.builder import Builder, process_extra_flags
 from lektor.watcher import Watcher
 from lektor.reporter import CliReporter
@@ -55,7 +55,8 @@ def build(self, update_source_info_first=False):
     def run(self):
         with CliReporter(self.env, verbosity=self.verbosity):
             self.build(update_source_info_first=True)
-            for ts, _, _ in self.watcher:
+            for ts, eventtype, absolute_filename_w_path  in self.watcher:
+                DatabaseCache.purge_cache()
                 if self.last_build is None or ts > self.last_build:
                     self.build()
 

lektor/environment.py

@@ -73,6 +73,12 @@
     'ALTERNATIVES': OrderedDict(),
     'PRIMARY_ALTERNATIVE': None,
     'SERVERS': {},
+    'LEKTOR_CACHE': {
+        'datamodels_cache_enable': False,
+        'datamodels_cache_timeout_seconds': 30,
+    },
+
+
 }
 
 
@@ -95,7 +101,8 @@ def set_simple(target, source_path):
     set_simple(target='LESSC_EXECUTABLE',
                source_path='env.lessc_executable')
 
-    for section_name in ('ATTACHMENT_TYPES', 'PROJECT', 'PACKAGES', 'THEME_SETTINGS'):
+    for section_name in \
+            ('ATTACHMENT_TYPES', 'PROJECT', 'PACKAGES', 'THEME_SETTINGS', 'LEKTOR_CACHE'):
         section_config = inifile.section_as_dict(section_name.lower())
         config[section_name].update(section_config)
 

tests/test_config.py

@@ -1,3 +1,12 @@
+from lektor.db import DatabaseCache
+
+
 def test_custom_attachment_types(env):
     attachment_types = env.load_config().values['ATTACHMENT_TYPES']
     assert attachment_types['.foo'] == 'text'
+
+
+def test_database_cache_disabled_ondefault(env):
+    enabled = DatabaseCache.is_cache_enabled(env)
+    assert enabled is False
+    assert DatabaseCache._cache_datamodels is None