From 1b76ca1060dae90fd50d0866a404eea7686d0be2 Mon Sep 17 00:00:00 2001 From: Uzay-G Date: Sun, 18 Jun 2023 11:38:51 -0400 Subject: [PATCH] fix RAM usage and clean stuff up (#305) --- .github/workflows/codecov.yml | 4 ++-- archivy/click_web/resources/cmd_exec.py | 2 -- archivy/data.py | 30 +++++++++++++++---------- archivy/models.py | 1 + archivy/routes.py | 7 ++---- archivy/tags.py | 1 - requirements.txt | 10 ++++----- tests/functional/test_routes.py | 23 ++++++++----------- tests/integration/test_api.py | 2 -- 9 files changed, 37 insertions(+), 43 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 0c3ada5b..b10f91db 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -22,13 +22,13 @@ jobs: run: runs-on: ubuntu-latest env: - PYTHON: '3.7' + PYTHON: '3.9' steps: - uses: actions/checkout@master - name: Setup Python uses: actions/setup-python@master with: - python-version: 3.7 + python-version: 3.9 - name: Generate coverage report run: | sudo apt-get install ripgrep diff --git a/archivy/click_web/resources/cmd_exec.py b/archivy/click_web/resources/cmd_exec.py index c92e9d07..88631ffb 100644 --- a/archivy/click_web/resources/cmd_exec.py +++ b/archivy/click_web/resources/cmd_exec.py @@ -158,7 +158,6 @@ def command_args(self, command_index) -> List[str]: commands_field_infos = sorted(commands_field_infos) for fi in commands_field_infos: - # must be called mostly for saving and preparing file output. fi.before_script_execute() @@ -336,7 +335,6 @@ def save(self): file.save(filename) def __str__(self): - res = [super().__str__()] res.append(f"file_path: {self.file_path}") return ", ".join(res) diff --git a/archivy/data.py b/archivy/data.py index c07534b3..9ecb2be9 100644 --- a/archivy/data.py +++ b/archivy/data.py @@ -47,14 +47,26 @@ def get_by_id(dataobj_id): return results[0] if results else None -def build_dir_tree(path, query_dir, load_content=True): +def load_frontmatter(filepath): + count = 0 + file = open(filepath, "r") + data = "" + line = "_" + while count != 2 and line: + line = file.readline() + if line in ["---\n", "---"]: + count += 1 + data += line + data = frontmatter.loads(data) + return data + + +def build_dir_tree(path, query_dir): """ Builds a structured tree of directories and data objects. - **path**: name of the directory relative to the root directory. - **query_dir**: absolute path of the directory we're building the tree of. - - **load_content**: internal option to not save post contents in memory - if they're not going to be accessed. """ datacont = Directory(path or "root") for filepath in query_dir.rglob("*"): @@ -75,16 +87,12 @@ def build_dir_tree(path, query_dir, load_content=True): current_dir.child_dirs[last_seg] = Directory(last_seg) current_dir = current_dir.child_dirs[last_seg] elif last_seg.endswith(".md"): - data = frontmatter.load(filepath) - if not load_content: - data.content = "" + data = load_frontmatter(filepath) current_dir.child_files.append(data) return datacont -def get_items( - collections=[], path="", structured=True, json_format=False, load_content=True -): +def get_items(collections=[], path="", structured=True, json_format=False): """ Gets all dataobjs. @@ -107,9 +115,7 @@ def get_items( else: datacont = [] for filepath in query_dir.rglob("*.md"): - data = frontmatter.load(filepath) - if not load_content: - data.content = "" + data = load_frontmatter(filepath) data["fullpath"] = str(filepath.parent.relative_to(query_dir)) if len(collections) == 0 or any( [collection == data["type"] for collection in collections] diff --git a/archivy/models.py b/archivy/models.py index daf16922..aaed747c 100644 --- a/archivy/models.py +++ b/archivy/models.py @@ -32,6 +32,7 @@ # NOTE = 'note' # PROCESSED_DATAOBJ = 'bookmark that has been processed' + # TODO: use this as 'type' field # class DataobjType(Enum): # BOOKMARK = 'bookmark' diff --git a/archivy/routes.py b/archivy/routes.py index 8c4d09ba..74fe3f4e 100644 --- a/archivy/routes.py +++ b/archivy/routes.py @@ -30,7 +30,7 @@ @app.context_processor def pass_defaults(): - dataobjs = data.get_items(load_content=False) + dataobjs = data.get_items() version = require("archivy")[0].version SEP = sep # check windows parsing for js (https://github.com/Uzay-G/archivy/issues/115) @@ -72,7 +72,6 @@ def index(): except FileNotFoundError: flash("Directory does not exist.", "error") return redirect("/") - return render_template( "home.html", title=path or "root", @@ -177,9 +176,7 @@ def show_tag(tag_name): def show_dataobj(dataobj_id): dataobj = data.get_item(dataobj_id) get_title_id_pairs = lambda x: (x["title"], x["id"]) - titles = list( - map(get_title_id_pairs, data.get_items(structured=False, load_content=False)) - ) + titles = list(map(get_title_id_pairs, data.get_items(structured=False))) js_ext = "" if app.config["DATAOBJ_JS_EXTENSION"]: js_ext = ( diff --git a/archivy/tags.py b/archivy/tags.py index f1765f85..53f207cd 100644 --- a/archivy/tags.py +++ b/archivy/tags.py @@ -4,7 +4,6 @@ from archivy import helpers, data from tinydb import Query, operations from archivy.search import query_ripgrep_tags -from archivy.data import get_items def is_tag_format(tag_name): diff --git a/requirements.txt b/requirements.txt index ddbfbfb5..56385a56 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,12 +6,12 @@ # archivy/extensions.py: 6 # archivy/models.py: 13 # archivy/routes.py: 7 -Flask == 2.0.0 -werkzeug == 2.0.3 -jinja2 == 3.0.0 +Flask == 2.3.2 +werkzeug == 2.3.3 +jinja2 == 3.1.2 # archivy/forms.py: 2 -Flask_WTF == 0.14.3 +Flask_WTF == 1.1.1 # archivy/forms.py: 4 WTForms == 2.3.1 @@ -53,7 +53,7 @@ validators == 0.15.0 # archivy/__init__.py: 9 # archivy/models.py: 13 # archivy/routes.py: 10 -flask-login == 0.5.0 +flask-login == 0.6.2 click_plugins html2text diff --git a/tests/functional/test_routes.py b/tests/functional/test_routes.py index 27497b85..cdda426f 100644 --- a/tests/functional/test_routes.py +++ b/tests/functional/test_routes.py @@ -1,13 +1,11 @@ import os -import re from flask.testing import FlaskClient from flask import request from flask_login import current_user from responses import RequestsMock, GET -from werkzeug.security import generate_password_hash -from archivy.helpers import get_max_id, get_db +from archivy.helpers import get_max_id from archivy.data import get_dirs, create_dir, get_items, get_item @@ -129,7 +127,6 @@ def test_creating_bookmark_without_passing_path_saves_to_default_dir( def test_create_note(test_app, client: FlaskClient): - note_data = { "title": "Testing the create route", "tags": "testing,note", @@ -158,7 +155,7 @@ def test_logging_in(test_app, client: FlaskClient): follow_redirects=True, ) assert resp.status_code == 200 - assert request.path == "/" + assert resp.request.path == "/" assert current_user @@ -169,7 +166,7 @@ def test_logging_in_with_invalid_creds(test_app, client: FlaskClient): follow_redirects=True, ) assert resp.status_code == 200 - assert request.path == "/login" + assert resp.request.path == "/login" assert b"Invalid credentials" in resp.data @@ -184,7 +181,7 @@ def test_edit_user(test_app, client: FlaskClient): follow_redirects=True, ) - assert request.path == "/" + assert resp.request.path == "/" client.delete("/logout") @@ -194,7 +191,7 @@ def test_edit_user(test_app, client: FlaskClient): follow_redirects=True, ) assert resp.status_code == 200 - assert request.path == "/" + assert resp.request.path == "/" # check information has updated. @@ -204,7 +201,7 @@ def test_logging_out(test_app, client: FlaskClient): client.delete("/logout") resp = client.get("/", follow_redirects=True) - assert request.path == "/login" + assert resp.request.path == "/login" def test_create_dir(test_app, client: FlaskClient): @@ -217,7 +214,7 @@ def test_create_dir(test_app, client: FlaskClient): ) assert resp.status_code == 200 - assert request.args.get("path") == "testing" + assert resp.request.args.get("path") == "testing" assert "testing" in get_dirs() assert b"Folder successfully created" in resp.data @@ -228,7 +225,7 @@ def test_creating_without_dirname_fails(test_app, client: FlaskClient): ) assert resp.status_code == 200 - assert request.path == "/" + assert resp.request.path == "/" assert b"Could not create folder." in resp.data @@ -275,7 +272,6 @@ def test_backlinks_are_saved( def test_bookmark_with_long_title_gets_truncated(test_app, client, mocked_responses): - long_title = "a" * 300 # check that our mock title is indeed longer than the limit # and would cause an error, without our truncating @@ -305,7 +301,6 @@ def test_move_data(test_app, note_fixture, client): def test_invalid_inputs_fail_move_data(test_app, note_fixture, client): - resp = client.post("/dataobj/move/1", follow_redirects=True) assert b"No path specified." in resp.data @@ -412,4 +407,4 @@ def test_bookmarklet_upload(test_app, client): ) assert resp.status_code == 200 assert title in str(resp.data) - assert "/dataobj/" in request.path + assert "/dataobj/" in resp.request.path diff --git a/tests/integration/test_api.py b/tests/integration/test_api.py index 9e8f1605..19e659a2 100644 --- a/tests/integration/test_api.py +++ b/tests/integration/test_api.py @@ -98,7 +98,6 @@ def test_get_bookmarks_with_empty_db(test_app, client: FlaskClient): def test_get_dataobjs(test_app, client: FlaskClient, bookmark_fixture): - note_dict = { "type": "note", "title": "Nested Test Note", @@ -119,7 +118,6 @@ def test_get_dataobjs(test_app, client: FlaskClient, bookmark_fixture): bookmark = response.json[0] assert bookmark["metadata"]["title"] == "Example" assert bookmark["metadata"]["id"] == 1 - assert bookmark["content"].startswith("Lorem ipsum") def test_update_dataobj(test_app, client: FlaskClient, note_fixture):