Sqlite3 support

install/databases.js

@@ -7,6 +7,7 @@ const questions = {
 	redis: require('../src/database/redis').questions,
 	mongo: require('../src/database/mongo').questions,
 	postgres: require('../src/database/postgres').questions,
+	sqlite3: require('../src/database/sqlite3').questions,
 };
 
 module.exports = async function (config) {
@@ -35,6 +36,11 @@ async function getDatabaseConfig(config) {
 			return config;
 		}
 		return await prompt.get(questions.postgres);
+	} else if (config.database === 'sqlite3') {
+		if (config['sqlite3:path']) {
+			return config;
+		}
+		return await prompt.get(questions.sqlite3);
 	}
 	throw new Error(`unknown database : ${config.database}`);
 }
@@ -74,6 +80,10 @@ function saveDatabaseConfig(config, databaseConfig) {
 			database: databaseConfig['postgres:database'],
 			ssl: databaseConfig['postgres:ssl'],
 		};
+	} else if (config.database === 'sqlite3') {
+		config.sqlite3 = {
+			path: databaseConfig['sqlite3:path'],
+		};
 	} else {
 		throw new Error(`unknown database : ${config.database}`);
 	}

install/package.json

@@ -37,6 +37,8 @@
         "autoprefixer": "10.4.14",
         "bcryptjs": "2.4.3",
         "benchpressjs": "2.5.1",
+        "better-sqlite3": "8.3.0",
+        "better-sqlite3-session-store": "0.1.0",
         "body-parser": "1.20.2",
         "bootbox": "6.0.0",
         "bootstrap": "5.2.3",

src/database/sqlite3.js

@@ -0,0 +1,198 @@
+'use strict';
+
+const winston = require('winston');
+const nconf = require('nconf');
+const session = require('express-session');
+const semver = require('semver');
+
+const connection = require('./sqlite3/connection');
+
+const sqlite3Module = module.exports;
+
+sqlite3Module.questions = [
+	{
+		name: 'sqlite3:path',
+		description: 'Fully qualified path to database file',
+		default: nconf.get('sqlite3:path') || '/var/local/nodebb.sqlite',
+	},
+];
+
+sqlite3Module.init = async function () {
+	try {
+		sqlite3Module.db = connection.connect(nconf.get('sqlite3'));
+		checkUpgrade(sqlite3Module.db);
+	} catch (err) {
+		winston.error(`NodeBB could not create the Sqlite3 database. Sqlite3 returned the following error: ${err.message}`);
+		throw err;
+	}
+};
+
+function checkUpgrade(db) {
+	const res = db.prepare(`
+SELECT EXISTS(SELECT * 
+								FROM sqlite_master 
+							 WHERE type='view' 
+							   AND name='legacy_object_live') a`).get();
+	if (res.a) {
+		return;
+	}
+
+	db.exec(`
+BEGIN;
+CREATE TABLE "legacy_object" (
+	"_key" TEXT NOT NULL
+		PRIMARY KEY,
+	"type" TEXT NOT NULL
+		CHECK( "type" IN ('hash', 'zset', 'set', 'list', 'string') ),
+	"expireAt" TEXT DEFAULT NULL,
+	UNIQUE ( "_key", "type" )
+);
+CREATE TABLE "legacy_hash" (
+	"_key" TEXT NOT NULL
+		PRIMARY KEY,
+	"data" TEXT NOT NULL,
+	"type" TEXT NOT NULL
+		DEFAULT 'hash'
+		CHECK ( "type" = 'hash' ),
+	CONSTRAINT "fk__legacy_hash__key"
+		FOREIGN KEY ("_key", "type")
+		REFERENCES "legacy_object"("_key", "type")
+		ON UPDATE CASCADE
+		ON DELETE CASCADE
+);
+CREATE TABLE "legacy_zset" (
+	"_key" TEXT NOT NULL,
+	"value" TEXT NOT NULL,
+	"score" REAL NOT NULL,
+	"type" TEXT NOT NULL
+		DEFAULT 'zset'
+		CHECK ( "type" = 'zset' ),
+	PRIMARY KEY ("_key", "value"),
+	CONSTRAINT "fk__legacy_zset__key"
+		FOREIGN KEY ("_key", "type")
+		REFERENCES "legacy_object"("_key", "type")
+		ON UPDATE CASCADE
+		ON DELETE CASCADE
+);
+CREATE TABLE "legacy_set" (
+	"_key" TEXT NOT NULL,
+	"member" TEXT NOT NULL,
+	"type" TEXT NOT NULL
+		DEFAULT 'set'
+		CHECK ( "type" = 'set' ),
+	PRIMARY KEY ("_key", "member"),
+	CONSTRAINT "fk__legacy_set__key"
+		FOREIGN KEY ("_key", "type")
+		REFERENCES "legacy_object"("_key", "type")
+		ON UPDATE CASCADE
+		ON DELETE CASCADE
+);
+CREATE TABLE "legacy_list" (
+	"_key" TEXT NOT NULL
+		PRIMARY KEY,
+	"array" TEXT NOT NULL,
+	"type" TEXT NOT NULL
+		DEFAULT 'list'
+		CHECK ( "type" = 'list' ),
+	CONSTRAINT "fk__legacy_list__key"
+		FOREIGN KEY ("_key", "type")
+		REFERENCES "legacy_object"("_key", "type")
+		ON UPDATE CASCADE
+		ON DELETE CASCADE
+);
+CREATE TABLE "legacy_string" (
+	"_key" TEXT NOT NULL
+		PRIMARY KEY,
+	"data" TEXT NOT NULL,
+	"type" TEXT NOT NULL
+		DEFAULT 'string'
+		CHECK ( "type" = 'string' ),
+	CONSTRAINT "fk__legacy_string__key"
+		FOREIGN KEY ("_key", "type")
+		REFERENCES "legacy_object"("_key", "type")
+		ON UPDATE CASCADE
+		ON DELETE CASCADE
+);
+CREATE VIEW "legacy_object_live" AS
+SELECT "_key", "type"
+  FROM "legacy_object"
+ WHERE "expireAt" IS NULL
+    OR "expireAt" > CURRENT_TIMESTAMP;
+COMMIT;
+	`);
+}
+
+sqlite3Module.createSessionStore = async function (options) {
+	const meta = require('../meta');
+	const db = connection.connect(options);
+	const sessionStore = require('better-sqlite3-session-store')(session);
+	return new sessionStore({
+		client: db,
+		expired: {
+			clear: true,
+			intervalMs: meta.getSessionTTLSeconds() * 1000
+		}		
+	});
+};
+
+sqlite3Module.createIndices = function (callback) {
+	if (!sqlite3Module.db) {
+		winston.warn('[database/createIndices] database not initialized');
+		return callback();
+	}
+
+	const db = sqlite3Module.db;
+	winston.info('[database] Checking database indices.');
+	try {
+		db.exec(`
+CREATE INDEX IF NOT EXISTS "idx__legacy_zset__key__score" ON "legacy_zset"("_key" ASC, "score" DESC);	
+CREATE INDEX IF NOT EXISTS "idx__legacy_object__expireAt" ON "legacy_object"("expireAt" ASC);
+		`);	
+	} catch (err) {
+		winston.error(`Error creating index ${err.message}`);
+		return callback(err);
+	}
+	winston.info('[database] Checking database indices done!');
+	callback();
+};
+
+sqlite3Module.checkCompatibility = function (callback) {
+	const sqlite3Pkg = require('better-sqlite3/package.json');
+	sqlite3Module.checkCompatibilityVersion(sqlite3Pkg.version, callback);
+};
+
+sqlite3Module.checkCompatibilityVersion = function (version, callback) {
+	if (semver.lt(version, '8.3.0')) {
+		return callback(new Error('The `better-sqlite3` package is out-of-date, please run `./nodebb setup` again.'));
+	}
+
+	callback();
+};
+
+sqlite3Module.info = async function (db) {
+	if (!db) {
+		db = connection.connect(nconf.get('sqlite3'));
+	}
+	sqlite3Module.db = sqlite3Module.db || db;
+	const res = db.prepare(`
+		SELECT true "sqlite3",
+			sqlite_version() "version"
+	`).get();
+	return {
+		...res,
+		raw: JSON.stringify(res, null, 4),
+	};
+};
+
+sqlite3Module.close = async function () {
+	sqlite3Module.db.close();
+};
+
+require('./sqlite3/main')(sqlite3Module);
+require('./sqlite3/hash')(sqlite3Module);
+require('./sqlite3/sets')(sqlite3Module);
+require('./sqlite3/sorted')(sqlite3Module);
+require('./sqlite3/list')(sqlite3Module);
+require('./sqlite3/transaction')(sqlite3Module);
+
+require('../promisify')(sqlite3Module, ['client', 'sessionStore', 'pool', 'transaction']);

src/database/sqlite3/connection.js

@@ -0,0 +1,91 @@
+'use strict';
+
+const winston = require('winston');
+
+const connection = module.exports;
+
+connection.connect = function (options) {
+	const Sqlite3 = require('better-sqlite3');
+	const db = new Sqlite3(options.path, {});
+	addFunctions(db);
+	return db;
+};
+
+function addFunctions(db) {
+  db.function('json_inc', { deterministic: true }, (json, name, amount) => {
+    const object = (json) ? JSON.parse(json) : {};
+    object[name] = object[name] ? object[name] + amount : amount;
+    return JSON.stringify(object);
+  });
+  db.function('json_set', { deterministic: true }, (json, name, jsonValue) => {
+    const object = (json) ? JSON.parse(json) : {};
+    const value = (jsonValue) ? JSON.parse(jsonValue) : null;
+    object[name] = value;
+    return JSON.stringify(object);
+  });
+  db.function('json_get', { deterministic: true }, (json, name) => {
+    const object = (json) ? JSON.parse(json) : {};
+    return JSON.stringify(object[name] ?? null);
+  });
+  db.function('json_gather', { deterministic: true, varargs: true }, (json, ...names) => {
+    const object = (json) ? JSON.parse(json) : {};
+    const res = {};
+    for (const name of names) {
+      res[name] = object[name] ?? null;
+    }
+    return JSON.stringify(res);
+  });
+  db.function('json_delete', { deterministic: true, varargs: true }, (json, ...names) => {
+    const object = (json) ? JSON.parse(json) : {};
+    for (const name of names) {
+      delete object[name];
+    }
+    return JSON.stringify(object);
+  });
+  db.function('json_has', { deterministic: true, varargs: true }, (json, ...names) => {
+    const object = (json) ? JSON.parse(json) : {};
+    for (const name of names) {
+      if (object[name] != undefined) {
+        return 1;
+      }
+    }
+    return 0;
+  }); 
+  db.function('json_keys', { deterministic: true }, (json) => {
+    const object = (json) ? JSON.parse(json) : {};
+    const keys = Object.keys(object);
+    return JSON.stringify(keys);
+  });
+  db.function('json_merge', { deterministic: true }, (json, jsonAddendum) => {
+    const object = (json) ? JSON.parse(json) : {};
+    const addendum = (jsonAddendum) ? JSON.parse(jsonAddendum) : {};
+    Object.assign(object, addendum);
+    return JSON.stringify(object);
+  });
+  db.function('json_array_append', { deterministic: true }, (json, jsonAddendum) => {
+    const array = (json) ? JSON.parse(json) : [];
+    const addendum = (jsonAddendum) ? JSON.parse(jsonAddendum) : [];
+    for (const item of addendum) {
+      array.push(item);
+    }
+    return JSON.stringify(array);
+  });
+  db.function('json_array_remove', { deterministic: true }, (json, jsonRemoval) => {
+    const array = (json) ? JSON.parse(json) : [];
+    const removal = (jsonRemoval) ? JSON.parse(jsonRemoval) : [];
+    const list = array.filter(i => !removal.includes(i));
+    return JSON.stringify(list);
+  });
+  db.function('json_array_slice', { deterministic: true }, (json, start, stop) => {
+    const array = (json) ? JSON.parse(json) : [];
+    const end = (stop > 0) ? stop : array.length + stop;
+    const list = array.slice(start, end);
+    return JSON.stringify(list);
+  });
+  db.function('json_array_length', { deterministic: true }, (json) => {
+    const array = (json) ? JSON.parse(json) : [];
+    return array.length;
+  });
+}
+
+require('../../promisify')(connection);