feat: use transformer.js for embeddings

jest.config.electron.e2e.ts

@@ -5,7 +5,15 @@ const jestConfig = {
 	roots: ["<rootDir>/src/electron"],
 	testMatch: ["**/*.test.e2e.ts"],
 	transform: {
-		"^.+\\.(ts|js)$": ["@swc/jest"],
+		"^.+\\.(ts)$": ["@swc/jest"],
+		"^.+\\.(js)$": [
+			"@swc/jest",
+			{
+				jsc: {
+					target: "es5",
+				},
+			},
+		],
 	},
 	moduleNameMapper: {
 		"@/(.*)": "<rootDir>/src/electron/future/$1",

package.json

@@ -34,6 +34,7 @@
     "electron-dl": "3.5.2",
     "electron-serve": "1.3.0",
     "electron-store": "^8.2.0",
+    "onnxruntime-node": "^1.17.0",
     "sharp": "0.33.2"
   },
   "devDependencies": {
@@ -81,6 +82,7 @@
     "@types/uuid": "^9.0.8",
     "@typescript-eslint/eslint-plugin": "^7.1.1",
     "@typescript-eslint/parser": "^7.1.1",
+    "@xenova/transformers": "github:xenova/transformers.js#v3",
     "axios": "^1.6.7",
     "codemirror": "^5.65.16",
     "conventional-changelog-cli": "^4.1.0",
@@ -151,8 +153,8 @@
     "tslib": "2.6.2",
     "type-fest": "4.12.0",
     "typescript": "^5.4.2",
-    "uuid": "^9.0.1",
     "use-debounce": "^10.0.0",
+    "uuid": "^9.0.1",
     "webpack": "^5.90.3"
   },
   "optionalDependencies": {

src/electron/future/langchain/__tests__/custom-hugging-face-transformers-embeddings.test.e2e.ts

@@ -0,0 +1,88 @@
+import path from "node:path";
+
+import { env } from "@xenova/transformers";
+
+const originalImplementation = Array.isArray;
+// @ts-expect-error we just want to mock this
+Array.isArray = jest.fn(type => {
+	if (
+		type &&
+		type.constructor &&
+		(type.constructor.name === "Float32Array" || type.constructor.name === "BigInt64Array")
+	) {
+		return true;
+	}
+
+	return originalImplementation(type);
+});
+
+import { CustomHuggingFaceTransformersEmbeddings } from "../custom-hugging-face-transformers-embeddings";
+
+describe("CustomHuggingFaceEmbeddings", () => {
+	let embeddings: any;
+
+	beforeAll(() => {
+		env.localModelPath = path.join(process.cwd(), "models");
+
+		embeddings = new CustomHuggingFaceTransformersEmbeddings({
+			modelName: "Xenova/all-MiniLM-L6-v2",
+			maxTokens: 128,
+			stripNewLines: true,
+		});
+	});
+
+	it("should create embeddings for a given text", async () => {
+		const text = "Hello, world!";
+		const result = await embeddings.embedQuery(text);
+
+		expect(Array.isArray(result)).toBeTruthy();
+		expect(result.length).toBeGreaterThan(0);
+	});
+
+	it("should create embeddings for multiple documents", async () => {
+		const texts = ["Hello, world!", "Goodbye, world!"];
+		const results = await embeddings.embedDocuments(texts);
+
+		// Check that results is an array of arrays
+		expect(Array.isArray(results)).toBeTruthy();
+		expect(results.length).toEqual(texts.length);
+
+		// Check each result to ensure it's an array and not empty
+		for (const [index, embedding] of results.entries()) {
+			expect(Array.isArray(embedding)).toBeTruthy();
+			expect(embedding.length).toBeGreaterThan(0);
+		}
+	});
+
+	describe("without maxTokens defined", () => {
+		let defaultEmbeddings: any;
+
+		beforeAll(() => {
+			defaultEmbeddings = new CustomHuggingFaceTransformersEmbeddings({
+				modelName: "Xenova/all-MiniLM-L6-v2",
+				stripNewLines: true,
+			});
+		});
+
+		it("should create embeddings for a given text using default settings", async () => {
+			const text = "Hello, world!";
+			const result = await defaultEmbeddings.embedQuery(text);
+
+			expect(Array.isArray(result)).toBeTruthy();
+			expect(result.length).toBeGreaterThan(0);
+		});
+
+		it("should create embeddings for multiple documents using default settings", async () => {
+			const texts = ["Hello, world!", "Goodbye, world!"];
+			const results = await defaultEmbeddings.embedDocuments(texts);
+
+			expect(Array.isArray(results)).toBeTruthy();
+			expect(results.length).toEqual(texts.length);
+
+			for (const [index, embedding] of results.entries()) {
+				expect(Array.isArray(embedding)).toBeTruthy();
+				expect(embedding.length).toBeGreaterThan(0);
+			}
+		});
+	});
+});