Quickfix memory usage due to broken filelocks on distributed systems (#…

…1743)

asreview/webapp/api/projects.py

@@ -1470,6 +1470,7 @@ def api_classify_instance(project, doc_id):  # noqa: F401
 
     retrain_model = False if is_prior == "1" else True
     prior = True if is_prior == "1" else False
+    file_lock_path = Path(project.project_path, "trainings.lock")
 
     if request.method == "POST":
         with open_state(project.project_path, read_only=False) as state:
@@ -1485,7 +1486,8 @@ def api_classify_instance(project, doc_id):  # noqa: F401
             elif label == -1:
                 state.delete_record_labeling_data(record_id)
 
-    if retrain_model:
+    # retrain only if there is no lock
+    if retrain_model and not file_lock_path.exists():
         # retrain model
         subprocess.Popen(
             [

asreview/webapp/app.py

@@ -71,6 +71,12 @@ def create_app(config_path=None):
     if config_fp := (config_path or app.config.get("CONFIG_PATH", None)):
         app.config.from_file(Path(config_fp).absolute(), load=tomllib.load, text=False)
 
+    # remove all lock files per project folder
+    for f in asreview_path().iterdir():
+        if f.is_dir():
+            # remove lock files
+            [lockfile.unlink(missing_ok=True) for lockfile in f.glob("*.lock")]
+
     # if there are no cors and config is in debug mode, add default cors
     if app.debug and not app.config.get("CORS_ORIGINS", None):
         app.config["CORS_ORIGINS"] = ["http://localhost:3000", "http://127.0.0.1:3000"]

asreview/webapp/entry_points/run_model.py

@@ -13,12 +13,8 @@
 # limitations under the License.
 
 import argparse
-import logging
 from pathlib import Path
 
-from filelock import FileLock
-from filelock import Timeout
-
 from asreview.models.balance import get_balance_model
 from asreview.models.classifiers import get_classifier
 from asreview.models.feature_extraction import get_feature_model
@@ -42,6 +38,7 @@ def run_model_entry_point(argv):
     args = parser.parse_args(argv)
 
     project = ASReviewProject(args.project_path)
+    file_lock_path = Path(project.project_path, "trainings.lock")
 
     try:
 
@@ -51,9 +48,7 @@ def run_model_entry_point(argv):
                 return
 
         # Lock so that only one training run is running at the same time.
-        lock = FileLock(Path(project.project_path, "training.lock"), timeout=0)
-
-        with lock:
+        with open(file_lock_path, "w"):
 
             with open_state(project) as state:
                 settings = state.settings
@@ -70,14 +65,13 @@ def run_model_entry_point(argv):
 
             reviewer.train()
 
-        project.update_review(status="review")
-
-    except Timeout:
-        logging.debug("Another iteration is training")
+            # Update status
+            project.update_review(status="review")
 
     except Exception as err:
         project.set_error(err, save_error_message=args.output_error)
         raise err
 
-    else:
-        project.update_review(status="review")
+    finally:
+        # Ensure removal of lock file
+        file_lock_path.unlink(missing_ok=True)