fix(auth): add scripts to import users to auth0 (#755)

Follow up of #705. May contribute to #669.

Usage, from project root dir:

```sh
$ mongoexport -d ${dbname} -c user --type=json --out ./prod-users.json-lines -u ${dbuser} -p ${dbpassword}
$ node ./scripts/auth0/prepare-import-batches.js # => create files: `prod-users-*.for-auth0.json`
$ ./scripts/auth0/import-prod-users.sh
```

.gitignore

@@ -1,4 +1,5 @@
 .env
+.env-prod
 .token
 .port
 .idea

env-vars-testing.conf

@@ -23,8 +23,9 @@ ALGOLIA_API_KEY=""
 DISABLE_DATADOG="true"
 LOG_REQ_THRESHOLD_MS="5000"
 
-# experimental: use Auth0 as indentity provider, instead of checking password hashes ourselves
+# use Auth0 as indentity provider, instead of checking password hashes ourselves
 AUTH0_ISSUER_BASE_URL=""
 AUTH0_CLIENT_ID=""
+AUTH0_SECRET_ID=""
 AUTH0_SECRET="" # to generate with $ openssl rand -hex 32
 # dont forget to run $ ngrok tcp 27117, to expose local user accounts to Auth0

scripts/auth0/import-prod-get-error.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e # stop an any error
+
+source ./.env-prod # loads AUTH0_ISSUER_BASE_URL, AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET
+
+JOB_ID=$1
+
+DATABASE_CONNECTION_ID="con_Ilnbm841MAIXvpeS"
+
+echo "🔑 Reading token from scripts/auth0/.token ..." # from https://manage.auth0.com/dashboard/eu/openwhyd/apis/management/explorer
+TOKEN="$( cat scripts/auth0/.token )"
+
+echo "🔬 Fetching import status for job ${JOB_ID}..."
+ERRORS=$( curl --request GET \
+  --url "${AUTH0_ISSUER_BASE_URL}/api/v2/jobs/${JOB_ID}/errors" \
+  --header "authorization: Bearer ${TOKEN}" \
+  --header "content-type: application/json" \
+  --form "connection_id=${DATABASE_CONNECTION_ID}" )
+echo "=> Errors: ${ERRORS}"
+echo "${ERRORS}" | jq .

scripts/auth0/import-prod-users.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# This script imports prod users to our prod Auth0 account.
+# Based on procedure: https://auth0.com/docs/manage-users/user-migration/bulk-user-imports
+
+# Usage, from project root dir:
+# 1. $ mongoexport -d ${dbname} -c user --type=json --out ./prod-users.json-lines -u ${dbuser} -p ${dbpassword}
+# 2. $ node ./scripts/auth0/prepare-import-batches.js # => create files: `prod-users-*.for-auth0.json`
+# 3. $ ./scripts/auth0/import-prod-users.sh
+
+set -e # stop an any error
+
+source ./.env-prod # loads AUTH0_ISSUER_BASE_URL, AUTH0_CLIENT_ID, AUTH0_CLIENT_SECRET
+
+TIMESTAMP="$( date +'%Y-%m-%d_%H-%M' )"
+DATABASE_CONNECTION_ID="con_Ilnbm841MAIXvpeS"
+
+echo "🔑 Reading token from scripts/auth0/.token ..." # from https://manage.auth0.com/dashboard/eu/openwhyd/apis/management/explorer
+TOKEN="$( cat scripts/auth0/.token )"
+
+# Iterate through all files matching the "prod-users-*.json" pattern
+for USERS_FILE in prod-users-*.json; do
+
+  echo "\n🚚 Importing users from ${USERS_FILE}..."
+  RESPONSE=$( curl --request POST \
+    --url "${AUTH0_ISSUER_BASE_URL}/api/v2/jobs/users-imports" \
+    --header "authorization: Bearer ${TOKEN}" \
+    --form users=@${USERS_FILE} \
+    --form connection_id="${DATABASE_CONNECTION_ID}" \
+    --form external_id="import-test-users-${TIMESTAMP}" \
+    --form upsert="true" )
+  echo "=> Response: ${RESPONSE}"
+
+  echo "\n🧗‍♀️ Waiting for import job to complete..."
+  sleep 240
+
+  echo "\n🔬 Fetching import status..."
+  JOB_ID=$( echo "${RESPONSE}" | jq --raw-output ".id" )
+  echo "=> Job id: ${JOB_ID}"
+  ERRORS=$( curl --request GET \
+    --url "${AUTH0_ISSUER_BASE_URL}/api/v2/jobs/${JOB_ID}/errors" \
+    --header "authorization: Bearer ${TOKEN}" \
+    --header "content-type: application/json" \
+    --form "connection_id=${DATABASE_CONNECTION_ID}" )
+  echo "=> Errors: ${ERRORS}"
+  echo "${ERRORS}" | jq .
+
+  echo \n"🧗‍♀️ Will start next import in 30 seconds..."
+  sleep 30
+
+done
+
+echo "✅ Done!"

scripts/auth0/prepare-import-batches.js

@@ -0,0 +1,91 @@
+// usage:
+// 1. $ mongoexport -d ${dbname} -c user --type=json --out ./prod-users.json-lines -u ${dbuser} -p ${dbpassword}
+// 2. $ node ./scripts/auth0/prepare-import-batches.js
+// 3. $ ./import-prod-users.sh
+
+const fs = require('fs');
+const readline = require('readline');
+
+// Replace 'your-text-file.txt' with the path to your text file
+const inputFile = 'prod-users.json-lines';
+const outputFile = 'prod-users-NUMBER.for-auth0.json';
+const MAX_BYTES_PER_BATCH = 498 * 1000; // ≤ 500KB, file size limit for a bulk import
+
+let currentBatchNumber = 0;
+let currentBatchBytes = 0;
+let currentBatchUsers = [];
+let nextGeneratedUsername = 0;
+
+const convertUser = (user) => {
+  const defaultUsername =
+    !user.handle || !user.name ? `_auto_${nextGeneratedUsername++}` : null;
+  return {
+    user_id: user._id.$oid,
+    email: user.email,
+    username: user.handle || defaultUsername,
+    name: user.name || defaultUsername,
+    custom_password_hash: {
+      algorithm: 'md5',
+      hash: {
+        encoding: 'hex',
+        value: user.pwd,
+      },
+    },
+  };
+};
+
+async function* readLinesGenerator(filePath) {
+  const fileStream = fs.createReadStream(filePath);
+  const rl = readline.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity,
+  });
+
+  try {
+    for await (const line of rl) {
+      yield line;
+    }
+  } catch (error) {
+    console.error('Error:', error);
+  } finally {
+    rl.close();
+  }
+}
+
+// Create the generator
+const linesGenerator = readLinesGenerator(inputFile);
+
+// Use the generator to retrieve lines
+(async () => {
+  try {
+    for await (const line of linesGenerator) {
+      const user = JSON.parse(line);
+      if (
+        currentBatchBytes +
+          Buffer.byteLength(JSON.stringify(user), 'utf8') +
+          1 >=
+        MAX_BYTES_PER_BATCH
+      ) {
+        await dumpCurrentBatch();
+        currentBatchUsers = [];
+      }
+      currentBatchUsers.push(convertUser(user));
+      currentBatchBytes = Buffer.byteLength(
+        JSON.stringify(currentBatchUsers),
+        'utf8',
+      );
+    }
+
+    await dumpCurrentBatch();
+    console.warn(`✅ done`);
+  } catch (error) {
+    console.error('❌ Error:', error);
+  }
+})();
+
+async function dumpCurrentBatch() {
+  const filename = outputFile.replace('NUMBER', currentBatchNumber);
+  ++currentBatchNumber;
+  await fs.promises.writeFile(filename, JSON.stringify(currentBatchUsers));
+  console.warn(`🟢 wrote ${filename}`);
+}

scripts/restart.sh

@@ -17,8 +17,10 @@ GIT_REPOSITORY_URL="git@github.com:openwhyd/openwhyd.git"
 
 echo "👋  Restarting Openwhyd server ${VERSION} on port ${PORT}, with ${NODE}..."
 cd ${ROOT_DIR}
+pwd
 source env-vars-local.sh
+echo "AUTH0_ISSUER_BASE_URL=${AUTH0_ISSUER_BASE_URL}"
 DD_GIT_COMMIT_SHA="${GIT_COMMIT_SHA}" \
 DD_GIT_REPOSITORY_URL="${GIT_REPOSITORY_URL}" \
 WHYD_PORT=${PORT} \
-  npx --yes pm2 restart app.js --interpreter=${NODE} --update-env
+  npx --yes pm2 restart app.js --interpreter=${NODE} --update-env --env production