fix RAM usage and clean stuff up (#305)

.github/workflows/codecov.yml

@@ -22,13 +22,13 @@ jobs:
   run:
     runs-on: ubuntu-latest
     env:
-      PYTHON: '3.7'
+      PYTHON: '3.9'
     steps:
     - uses: actions/checkout@master
     - name: Setup Python
       uses: actions/setup-python@master
       with:
-        python-version: 3.7
+        python-version: 3.9
     - name: Generate coverage report
       run: |
         sudo apt-get install ripgrep

archivy/click_web/resources/cmd_exec.py

@@ -158,7 +158,6 @@ def command_args(self, command_index) -> List[str]:
         commands_field_infos = sorted(commands_field_infos)
 
         for fi in commands_field_infos:
-
             # must be called mostly for saving and preparing file output.
             fi.before_script_execute()
 
@@ -336,7 +335,6 @@ def save(self):
             file.save(filename)
 
     def __str__(self):
-
         res = [super().__str__()]
         res.append(f"file_path: {self.file_path}")
         return ", ".join(res)

archivy/data.py

@@ -47,14 +47,26 @@ def get_by_id(dataobj_id):
     return results[0] if results else None
 
 
-def build_dir_tree(path, query_dir, load_content=True):
+def load_frontmatter(filepath):
+    count = 0
+    file = open(filepath, "r")
+    data = ""
+    line = "_"
+    while count != 2 and line:
+        line = file.readline()
+        if line in ["---\n", "---"]:
+            count += 1
+        data += line
+    data = frontmatter.loads(data)
+    return data
+
+
+def build_dir_tree(path, query_dir):
     """
     Builds a structured tree of directories and data objects.
 
     - **path**: name of the directory relative to the root directory.
     - **query_dir**: absolute path of the directory we're building the tree of.
-    - **load_content**: internal option to not save post contents in memory
-        if they're not going to be accessed.
     """
     datacont = Directory(path or "root")
     for filepath in query_dir.rglob("*"):
@@ -75,16 +87,12 @@ def build_dir_tree(path, query_dir, load_content=True):
                 current_dir.child_dirs[last_seg] = Directory(last_seg)
             current_dir = current_dir.child_dirs[last_seg]
         elif last_seg.endswith(".md"):
-            data = frontmatter.load(filepath)
-            if not load_content:
-                data.content = ""
+            data = load_frontmatter(filepath)
             current_dir.child_files.append(data)
     return datacont
 
 
-def get_items(
-    collections=[], path="", structured=True, json_format=False, load_content=True
-):
+def get_items(collections=[], path="", structured=True, json_format=False):
     """
     Gets all dataobjs.
 
@@ -107,9 +115,7 @@ def get_items(
     else:
         datacont = []
         for filepath in query_dir.rglob("*.md"):
-            data = frontmatter.load(filepath)
-            if not load_content:
-                data.content = ""
+            data = load_frontmatter(filepath)
             data["fullpath"] = str(filepath.parent.relative_to(query_dir))
             if len(collections) == 0 or any(
                 [collection == data["type"] for collection in collections]

archivy/models.py

@@ -32,6 +32,7 @@
 #     NOTE = 'note'
 #     PROCESSED_DATAOBJ = 'bookmark that has been processed'
 
+
 # TODO: use this as 'type' field
 # class DataobjType(Enum):
 #     BOOKMARK = 'bookmark'

archivy/routes.py

@@ -30,7 +30,7 @@
 
 @app.context_processor
 def pass_defaults():
-    dataobjs = data.get_items(load_content=False)
+    dataobjs = data.get_items()
     version = require("archivy")[0].version
     SEP = sep
     # check windows parsing for js (https://github.com/Uzay-G/archivy/issues/115)
@@ -72,7 +72,6 @@ def index():
     except FileNotFoundError:
         flash("Directory does not exist.", "error")
         return redirect("/")
-
     return render_template(
         "home.html",
         title=path or "root",
@@ -177,9 +176,7 @@ def show_tag(tag_name):
 def show_dataobj(dataobj_id):
     dataobj = data.get_item(dataobj_id)
     get_title_id_pairs = lambda x: (x["title"], x["id"])
-    titles = list(
-        map(get_title_id_pairs, data.get_items(structured=False, load_content=False))
-    )
+    titles = list(map(get_title_id_pairs, data.get_items(structured=False)))
     js_ext = ""
     if app.config["DATAOBJ_JS_EXTENSION"]:
         js_ext = (

archivy/tags.py

@@ -4,7 +4,6 @@
 from archivy import helpers, data
 from tinydb import Query, operations
 from archivy.search import query_ripgrep_tags
-from archivy.data import get_items
 
 
 def is_tag_format(tag_name):

requirements.txt

@@ -6,12 +6,12 @@
 # archivy/extensions.py: 6
 # archivy/models.py: 13
 # archivy/routes.py: 7
-Flask == 2.0.0
-werkzeug == 2.0.3
-jinja2 == 3.0.0
+Flask == 2.3.2
+werkzeug == 2.3.3
+jinja2 == 3.1.2
 
 # archivy/forms.py: 2
-Flask_WTF == 0.14.3
+Flask_WTF == 1.1.1
 
 # archivy/forms.py: 4
 WTForms == 2.3.1
@@ -53,7 +53,7 @@ validators == 0.15.0
 # archivy/__init__.py: 9
 # archivy/models.py: 13
 # archivy/routes.py: 10
-flask-login == 0.5.0
+flask-login == 0.6.2
 
 click_plugins
 html2text

tests/functional/test_routes.py

@@ -1,13 +1,11 @@
 import os
-import re
 
 from flask.testing import FlaskClient
 from flask import request
 from flask_login import current_user
 from responses import RequestsMock, GET
-from werkzeug.security import generate_password_hash
 
-from archivy.helpers import get_max_id, get_db
+from archivy.helpers import get_max_id
 from archivy.data import get_dirs, create_dir, get_items, get_item
 
 
@@ -129,7 +127,6 @@ def test_creating_bookmark_without_passing_path_saves_to_default_dir(
 
 
 def test_create_note(test_app, client: FlaskClient):
-
     note_data = {
         "title": "Testing the create route",
         "tags": "testing,note",
@@ -158,7 +155,7 @@ def test_logging_in(test_app, client: FlaskClient):
         follow_redirects=True,
     )
     assert resp.status_code == 200
-    assert request.path == "/"
+    assert resp.request.path == "/"
     assert current_user
 
 
@@ -169,7 +166,7 @@ def test_logging_in_with_invalid_creds(test_app, client: FlaskClient):
         follow_redirects=True,
     )
     assert resp.status_code == 200
-    assert request.path == "/login"
+    assert resp.request.path == "/login"
     assert b"Invalid credentials" in resp.data
 
 
@@ -184,7 +181,7 @@ def test_edit_user(test_app, client: FlaskClient):
         follow_redirects=True,
     )
 
-    assert request.path == "/"
+    assert resp.request.path == "/"
 
     client.delete("/logout")
 
@@ -194,7 +191,7 @@ def test_edit_user(test_app, client: FlaskClient):
         follow_redirects=True,
     )
     assert resp.status_code == 200
-    assert request.path == "/"
+    assert resp.request.path == "/"
     # check information has updated.
 
 
@@ -204,7 +201,7 @@ def test_logging_out(test_app, client: FlaskClient):
     client.delete("/logout")
 
     resp = client.get("/", follow_redirects=True)
-    assert request.path == "/login"
+    assert resp.request.path == "/login"
 
 
 def test_create_dir(test_app, client: FlaskClient):
@@ -217,7 +214,7 @@ def test_create_dir(test_app, client: FlaskClient):
     )
 
     assert resp.status_code == 200
-    assert request.args.get("path") == "testing"
+    assert resp.request.args.get("path") == "testing"
     assert "testing" in get_dirs()
     assert b"Folder successfully created" in resp.data
 
@@ -228,7 +225,7 @@ def test_creating_without_dirname_fails(test_app, client: FlaskClient):
     )
 
     assert resp.status_code == 200
-    assert request.path == "/"
+    assert resp.request.path == "/"
     assert b"Could not create folder." in resp.data
 
 
@@ -275,7 +272,6 @@ def test_backlinks_are_saved(
 
 
 def test_bookmark_with_long_title_gets_truncated(test_app, client, mocked_responses):
-
     long_title = "a" * 300
     # check that our mock title is indeed longer than the limit
     # and would cause an error, without our truncating
@@ -305,7 +301,6 @@ def test_move_data(test_app, note_fixture, client):
 
 
 def test_invalid_inputs_fail_move_data(test_app, note_fixture, client):
-
     resp = client.post("/dataobj/move/1", follow_redirects=True)
     assert b"No path specified." in resp.data
 
@@ -412,4 +407,4 @@ def test_bookmarklet_upload(test_app, client):
     )
     assert resp.status_code == 200
     assert title in str(resp.data)
-    assert "/dataobj/" in request.path
+    assert "/dataobj/" in resp.request.path

tests/integration/test_api.py

@@ -98,7 +98,6 @@ def test_get_bookmarks_with_empty_db(test_app, client: FlaskClient):
 
 
 def test_get_dataobjs(test_app, client: FlaskClient, bookmark_fixture):
-
     note_dict = {
         "type": "note",
         "title": "Nested Test Note",
@@ -119,7 +118,6 @@ def test_get_dataobjs(test_app, client: FlaskClient, bookmark_fixture):
     bookmark = response.json[0]
     assert bookmark["metadata"]["title"] == "Example"
     assert bookmark["metadata"]["id"] == 1
-    assert bookmark["content"].startswith("Lorem ipsum")
 
 
 def test_update_dataobj(test_app, client: FlaskClient, note_fixture):