Adding support for custom tokens

CHANGELOG.md

@@ -15,6 +15,7 @@ Critical items to know are:
 
 
 ## [v3.x](https://github.com/expfactory/expfactory/tree/master) (master)
+ - ability to generate users from newline separated file with --tokens (3.2.15)
  - fixing bug with Dockerfile generation (3.2.14)
  - updating sqlalchemy (3.2.13)
  - updating flask to 1.1.4 (3.2.12)

expfactory/cli/__init__.py

@@ -72,6 +72,12 @@ def get_parser():
         type=int,
     )
 
+    users.add_argument(
+        "--tokens",
+        dest="token_file",
+        help="provide a filename with a single list (newline separated) of custom tokens to generate",
+    )
+
     users.add_argument(
         "--list",
         dest="list",

expfactory/cli/users.py

@@ -56,6 +56,13 @@ def main(args, parser, subparser):
             app.print_user(user)
         sys.exit(0)
 
+    # The user wants to create custom tokens
+    token_file = args.token_file
+    if token_file is not None:
+        users = app.tokens_from_file(token_file)
+        print("%s users existing or generated from file." % len(users))
+        sys.exit(0)
+
     # The user wants to manage user token
     action = None
     if args.revoke is not None:

expfactory/database/filesystem.py

@@ -30,11 +30,12 @@
 
 """
 from flask import session
-from expfactory.utils import write_json, mkdir_p
+from expfactory.utils import write_json, mkdir_p, read_file, token_regex
 from expfactory.defaults import EXPFACTORY_SUBID, EXPFACTORY_DATA
 from glob import glob
 import uuid
 import os
+import re
 import sys
 
 
@@ -48,7 +49,8 @@
 
 
 def generate_subid(self, token=None):
-    """assumes a flat (file system) database, organized by experiment id, and
+    """
+    assumes a flat (file system) database, organized by experiment id, and
     subject id, with data (json) organized by subject identifier
     """
 
@@ -61,14 +63,17 @@ def generate_subid(self, token=None):
 
 
 def list_users(self):
-    """list users, each associated with a filesystem folder"""
+    """
+    list users, each associated with a filesystem folder
+    """
     folders = glob("%s/*" % (self.database))
     folders.sort()
     return [self.print_user(x) for x in folders]
 
 
 def print_user(self, user):
-    """print a filesystem database user. A "database" folder that might end with
+    """
+    Print a filesystem database user. A "database" folder that might end with
     the participant status (e.g. _finished) is extracted to print in format
 
     [folder]                        [identifier][studyid]
@@ -96,16 +101,17 @@ def print_user(self, user):
 
 
 def generate_user(self, subid=None):
-    """generate a new user on the filesystem, still session based so we
+    """
+    Generate a new user on the filesystem, still session based so we
     create a new identifier. This function is called from the users new
     entrypoint, and it assumes we want a user generated with a token.
     since we don't have a database proper, we write the folder name to
     the filesystem
     """
     # Only generate token if subid being created
     if subid is None:
-        token = str(uuid.uuid4())
-        subid = self.generate_subid(token=token)
+        subid = str(uuid.uuid4())
+    subid = self.generate_subid(token=subid)
 
     if os.path.exists(self.data_base):  # /scif/data
         data_base = "%s/%s" % (self.data_base, subid)
@@ -117,7 +123,8 @@ def generate_user(self, subid=None):
 
 
 def finish_user(self, subid, ext="finished"):
-    """finish user will append "finished" (or other) to the data folder when
+    """
+    Finish user will append "finished" (or other) to the data folder when
     the user has completed (or been revoked from) the battery.
     For headless, this means that the session is ended and the token
     will not work again to rewrite the result. If the user needs to update
@@ -157,7 +164,8 @@ def finish_user(self, subid, ext="finished"):
 
 
 def restart_user(self, subid):
-    """restart user will remove any "finished" or "revoked" extensions from
+    """
+    Restart user will remove any "finished" or "revoked" extensions from
     the user folder to restart the session. This command always comes from
     the client users function, so we know subid does not start with the
     study identifer first
@@ -184,8 +192,34 @@ def restart_user(self, subid):
 # Tokens #######################################################################
 
 
+def tokens_from_file(self, token_file):
+    """
+    Generate one or more tokens from a newline separated file.
+    """
+    if not os.path.exists(token_file):
+        sys.exit(
+            "Tokens file %s does not exist. Are you sure it's bound to the container?"
+            % token_file
+        )
+    users = []
+    for token in read_file(token_file, readlines=True):
+        token = token.strip()
+
+        # Skip comments
+        if token.startswith("#") or not token:
+            continue
+        if not re.search(token_regex, token):
+            self.logger.warning(
+                "Token %s does not match regex requirements, skipping" % token
+            )
+
+        users.append(self.generate_user(token))
+    return users
+
+
 def validate_token(self, token):
-    """retrieve a subject based on a token. Valid means we return a participant
+    """
+    Retrieve a subject based on a token. Valid means we return a participant
     invalid means we return None
     """
     # A token that is finished or revoked is not valid
@@ -199,8 +233,10 @@ def validate_token(self, token):
 
 
 def refresh_token(self, subid):
-    """refresh or generate a new token for a user. If the user is finished,
-    this will also make the folder available again for using."""
+    """
+    Refresh or generate a new token for a user. If the user is finished,
+    this will also make the folder available again for using.
+    """
     if os.path.exists(self.data_base):  # /scif/data
         data_base = "%s/%s" % (self.data_base, subid)
         if os.path.exists(data_base):
@@ -215,13 +251,17 @@ def refresh_token(self, subid):
 
 
 def revoke_token(self, subid):
-    """revoke a presently active token, meaning append _revoked to it."""
+    """
+    Revoke a presently active token, meaning append _revoked to it.
+    """
     return self.finish_user(subid, ext="revoked")
 
 
 def save_data(self, session, exp_id, content):
-    """save data will obtain the current subid from the session, and save it
-    depending on the database type. Currently we just support flat files"""
+    """
+    Save data will obtain the current subid from the session, and save it
+    depending on the database type. Currently we just support flat files
+    """
 
     subid = session.get("subid")
 
@@ -255,7 +295,8 @@ def save_data(self, session, exp_id, content):
 
 
 def init_db(self):
-    """init_db for the filesystem ensures that the base folder (named
+    """
+    init_db for the filesystem ensures that the base folder (named
     according to the studyid) exists.
     """
     self.session = None

expfactory/database/relational.py

@@ -36,7 +36,7 @@
 
 from sqlalchemy.ext.declarative import declarative_base
 from expfactory.logger import bot
-from expfactory.utils import write_json
+from expfactory.utils import write_json, read_file, token_regex
 from expfactory.defaults import EXPFACTORY_SUBID, EXPFACTORY_DATA
 from glob import glob
 import os
@@ -58,15 +58,18 @@
 
 
 def generate_subid(self, token=None, return_user=False):
-    """generate a new user in the database, still session based so we
+    """
+    Generate a new user in the database, still session based so we
     create a new identifier.
     """
     from expfactory.database.models import Participant
 
+    # TODO: do we need to query here first instead to see if exists?
     if not token:
         p = Participant()
     else:
         p = Participant(token=token)
+
     self.session.add(p)
     self.session.commit()
     if return_user is True:
@@ -75,7 +78,9 @@ def generate_subid(self, token=None, return_user=False):
 
 
 def print_user(self, user):
-    """print a relational database user"""
+    """
+    Print a relational database user
+    """
     status = "active"
     token = user.token
 
@@ -91,7 +96,8 @@ def print_user(self, user):
 
 
 def list_users(self, user=None):
-    """list users, each having a model in the database. A headless experiment
+    """
+    List users, each having a model in the database. A headless experiment
     will use protected tokens, and interactive will be based on auto-
     incremented ids.
     """
@@ -108,7 +114,8 @@ def list_users(self, user=None):
 
 
 def generate_user(self):
-    """generate a new user in the database, still session based so we
+    """
+    Generate a new user in the database, still session based so we
     create a new identifier. This function is called from the users new
     entrypoint, and it assumes we want a user generated with a token.
     """
@@ -117,8 +124,10 @@ def generate_user(self):
 
 
 def finish_user(self, subid):
-    """finish user will remove a user's token, making the user entry not
-    accesible if running in headless model"""
+    """
+    Finish user will remove a user's token, making the user entry not
+    accesible if running in headless model
+    """
 
     p = self.revoke_token(subid)
     p.token = "finished"
@@ -127,7 +136,9 @@ def finish_user(self, subid):
 
 
 def restart_user(self, subid):
-    """restart a user, which means revoking and issuing a new token."""
+    """
+    Restart a user, which means revoking and issuing a new token.
+    """
     p = self.revoke_token(subid)
     p = self.refresh_token(subid)
     return p
@@ -136,8 +147,41 @@ def restart_user(self, subid):
 # Tokens #######################################################################
 
 
+def tokens_from_file(self, token_file):
+    """
+    Generate one or more tokens from a newline separated file.
+    """
+    from expfactory.database.models import Participant
+
+    if not os.path.exists(token_file):
+        sys.exit(
+            "Tokens file %s does not exist. Are you sure it's bound to the container?"
+            % token_file
+        )
+    users = []
+    for token in read_file(token_file, readlines=True):
+
+        token = token.strip()
+
+        # Skip comments
+        if token.startswith("#") or not token:
+            continue
+
+        if not re.search(token_regex, token):
+            self.logger.warning(
+                "Token %s does not match regex requirements, skipping" % token
+            )
+
+        # Only create if not created yet
+        p = Participant.query.filter(Participant.token == token).first()
+        if p is None:
+            users.append(self.generate_subid(token))
+    return users
+
+
 def validate_token(self, token):
-    """retrieve a subject based on a token. Valid means we return a participant
+    """
+    Retrieve a subject based on a token. Valid means we return a participant
     invalid means we return None
     """
     from expfactory.database.models import Participant
@@ -152,8 +196,10 @@ def validate_token(self, token):
 
 
 def revoke_token(self, subid):
-    """revoke a token by removing it. Is done at finish, and also available
-    as a command line option"""
+    """
+    Revoke a token by removing it. Is done at finish, and also available
+    as a command line option
+    """
     from expfactory.database.models import Participant
 
     p = Participant.query.filter(Participant.id == subid).first()
@@ -164,7 +210,9 @@ def revoke_token(self, subid):
 
 
 def refresh_token(self, subid):
-    """refresh or generate a new token for a user"""
+    """
+    Refresh or generate a new token for a user
+    """
     from expfactory.database.models import Participant
 
     p = Participant.query.filter(Participant.id == subid).first()
@@ -175,8 +223,10 @@ def refresh_token(self, subid):
 
 
 def save_data(self, session, exp_id, content):
-    """save data will obtain the current subid from the session, and save it
-    depending on the database type. Currently we just support flat files"""
+    """
+    Save data will obtain the current subid from the session, and save it
+    depending on the database type. Currently we just support flat files
+    """
     from expfactory.database.models import Participant, Result
 
     subid = session.get("subid")
@@ -222,7 +272,8 @@ def save_data(self, session, exp_id, content):
 
 
 def init_db(self):
-    """initialize the database, with the default database path or custom with
+    """
+    Initialize the database, with the default database path or custom with
     a format corresponding to the database type:
 
     Examples:

expfactory/database/sqlite.py

@@ -90,9 +90,9 @@ def save_data(self, session, exp_id, content):
 
 
 def init_db(self):
-    """initialize the database, with the default database path or custom of
+    """
+    Initialize the database, with the default database path or custom of
     the format sqlite:////scif/data/expfactory.db
-
     """
 
     # Database Setup, use default if uri not provided

expfactory/server.py

@@ -131,6 +131,7 @@ def finish_experiment(self, session, exp_id):
 EFServer.validate_token = validate_token
 EFServer.refresh_token = refresh_token
 EFServer.revoke_token = revoke_token
+EFServer.tokens_from_file = tokens_from_file
 
 # User Actions
 EFServer.list_users = list_users