Add VFS Support #201

lektor/_compat.py

@@ -13,6 +13,7 @@
 
 
 PY2 = sys.version_info[0] == 2
+WIN = sys.platform.startswith('win')
 _identity = lambda x: x
 
 

lektor/assets.py

@@ -1,25 +1,29 @@
 import os
-import stat
 import posixpath
 
 from lektor.sourceobj import SourceObject
+from lektor.vfs import PathNotFound
 
 
-def get_asset(pad, filename, parent=None):
-    env = pad.db.env
+def get_asset(pad, filename, parent):
+    vfs = pad.db.vfs
+    path = vfs.join_path(parent.source_filename, filename)
 
-    if env.is_uninteresting_source_name(filename):
+    try:
+        record = vfs.describe_path(path)
+    except PathNotFound:
         return None
 
-    try:
-        st = os.stat(os.path.join(parent.source_filename, filename))
-    except OSError:
+    # Check if the asset is ignored.  We chop off the leading assets/
+    # in the path here as the db API assumes a path below that folder.
+    if pad.db.is_ignored_asset(path.split('/', 1)[-1]):
+        print('NO ASSET {}'.format(path))
         return None
-    if stat.S_ISDIR(st.st_mode):
-        return Directory(pad, filename, parent=parent)
 
+    if record.is_dir:
+        return Directory(pad, filename, parent=parent)
     ext = os.path.splitext(filename)[1]
-    cls = env.special_file_assets.get(ext, File)
+    cls = pad.db.env.special_file_assets.get(ext, File)
     return cls(pad, filename, parent=parent)
 
 
@@ -42,6 +46,10 @@ def __init__(self, pad, name, path=None, parent=None):
         self.name = name
         self.parent = parent
 
+    @property
+    def vfs(self):
+        return self.pad.db.vfs
+
     @property
     def url_name(self):
         name = self.name
@@ -58,7 +66,7 @@ def url_name(self):
     def url_path(self):
         if self.parent is None:
             return '/' + self.name
-        return posixpath.join(self.parent.url_path, self.url_name)
+        return self.pad.db.vfs.join_path(self.parent.url_path, self.url_name)
 
     @property
     def artifact_name(self):
@@ -95,9 +103,10 @@ class Directory(Asset):
 
     @property
     def children(self):
+        # TODO: optimize. this stats twice with get_child
         try:
-            files = os.listdir(self.source_filename)
-        except OSError:
+            files = self.vfs.list_dir(self.source_filename)
+        except PathNotFound:
             return
 
         for filename in files:

lektor/build_programs.py

@@ -245,7 +245,8 @@ def produce_artifacts(self):
 
     def build_artifact(self, artifact):
         with artifact.open('wb') as df:
-            with open(self.source.attachment_filename, 'rb') as sf:
+            vfs = self.source.pad.db.vfs
+            with vfs.open(self.source.attachment_filename, 'rb') as sf:
                 shutil.copyfileobj(sf, df)
 
 
@@ -259,7 +260,8 @@ def produce_artifacts(self):
 
     def build_artifact(self, artifact):
         with artifact.open('wb') as df:
-            with open(self.source.source_filename, 'rb') as sf:
+            vfs = self.source.pad.db.vfs
+            with vfs.open(self.source.source_filename, 'rb') as sf:
                 shutil.copyfileobj(sf, df)
 
 

lektor/builder.py

@@ -490,8 +490,7 @@ def checksum(self):
             if os.path.isdir(self.filename):
                 h.update(b'DIR\x00')
                 for filename in sorted(os.listdir(self.filename)):
-                    if self.env.is_uninteresting_source_name(filename):
-                        continue
+                    # XXX: skip ignored files?
                     if isinstance(filename, text_type):
                         filename = filename.encode('utf-8')
                     h.update(filename)

lektor/context.py

@@ -113,6 +113,11 @@ def env(self):
         """The environment of the context."""
         return self.pad.db.env
 
+    @property
+    def vfs(self):
+        """A reference to the virtualized file system."""
+        return self.pad.db.env.vfs
+
     @property
     def record(self):
         """If the source is a record it will be available here."""

lektor/databags.py

@@ -3,9 +3,9 @@
 import errno
 
 from collections import OrderedDict
-from inifile import IniFile
 
 from lektor.context import get_ctx
+from lektor.inifile import IniView
 from lektor.utils import iter_dotted_path_prefixes, resolve_dotted_value, \
      merge, decode_flat_data
 
@@ -16,7 +16,7 @@ def load_databag(filename):
             with open(filename, 'r') as f:
                 return json.load(f, object_pairs_hook=OrderedDict)
         elif filename.endswith('.ini'):
-            return decode_flat_data(IniFile(filename).items(),
+            return decode_flat_data(IniView(filename).items(),
                                     dict_cls=OrderedDict)
     except (OSError, IOError) as e:
         if e.errno != errno.ENOENT:

lektor/datamodel.py

@@ -1,15 +1,13 @@
-import errno
 import math
-import os
-
-from inifile import IniFile
 
 from lektor import types
 from lektor._compat import iteritems, itervalues
 from lektor.environment import Expression, FormatExpression, PRIMARY_ALT
 from lektor.i18n import get_i18n_block, load_i18n_block, generate_i18n_kvs
 from lektor.pagination import Pagination
 from lektor.utils import bool_from_string, slugify
+from lektor.inifile import IniFile
+from lektor.vfs import PathNotFound
 
 
 class ChildConfig(object):
@@ -572,28 +570,21 @@ def flowblock_from_data(env, block_data):
     )
 
 
-def iter_inis(path):
+def iter_inis(vfs, path):
     try:
-        for filename in os.listdir(path):
-            if not filename.endswith('.ini') or filename[:1] in '_.':
-                continue
-            fn = os.path.join(path, filename)
-            if os.path.isfile(fn):
-                base = filename[:-4]
-                base = base.encode('utf-8').decode('ascii', 'replace')
-                inifile = IniFile(fn)
-                yield base, inifile
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            raise
+        for dirent in vfs.iter_path(path):
+            if dirent.name.endswith('.ini') and \
+               dirent.is_file:
+                yield dirent.name[:-4], IniFile(vfs, dirent.path)
+    except PathNotFound:
+        pass
 
 
 def load_datamodels(env):
     """Loads the datamodels for a specific environment."""
-    path = os.path.join(env.root_path, 'models')
     data = {}
 
-    for model_id, inifile in iter_inis(path):
+    for model_id, inifile in iter_inis(env.vfs, 'models'):
         data[model_id] = datamodel_data_from_ini(model_id, inifile)
 
     rv = {}
@@ -628,10 +619,9 @@ def create_model(model_id):
 
 def load_flowblocks(env):
     """Loads all the flow blocks for a specific environment."""
-    path = os.path.join(env.root_path, 'flowblocks')
     rv = {}
 
-    for flowblock_id, inifile in iter_inis(path):
+    for flowblock_id, inifile in iter_inis(env.vfs, 'flowblocks'):
         rv[flowblock_id] = flowblock_from_data(env,
             flowblock_data_from_ini(flowblock_id, inifile))