diff --git a/.prettierrc b/.prettierrc index 7c28b83a..a5cd8192 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,5 @@ { "tabWidth": 2, "useTabs": false, - "semi": false + "semi": true } diff --git a/lib/inlineWorker.ts b/lib/inlineWorker.ts new file mode 100644 index 00000000..e493125d --- /dev/null +++ b/lib/inlineWorker.ts @@ -0,0 +1,37 @@ +// vite-plugin-inline-worker-dev.ts +import { Plugin } from 'vite' +import fs from 'fs/promises' +import { build, transform } from 'esbuild' + +export default function InlineWorkerDevPlugin(): Plugin { + return { + name: 'vite:inline-worker-dev', + async load(id) { + if (id.includes('?inlineWorker')) { + const [cleanPath] = id.split('?') + console.log('cleanPath', cleanPath) + const code = await fs.readFile(cleanPath, 'utf-8') + const result = await build({ + entryPoints: [cleanPath], + bundle: true, + write: false, + platform: 'browser', + format: 'iife', + target: 'esnext', + }) + + const workerCode = result.outputFiles[0].text + + const workerBlobCode = ` + const code = ${JSON.stringify(workerCode)}; + export default function InlineWorker() { + const blob = new Blob([code], { type: 'application/javascript' }); + return new Worker(URL.createObjectURL(blob), { type: 'module' }); + } + ` + return workerBlobCode + } + return null + } + } +} diff --git a/src/declarations.d.ts b/src/declarations.d.ts index f89205fc..deb52700 100644 --- a/src/declarations.d.ts +++ b/src/declarations.d.ts @@ -5,6 +5,11 @@ declare module '*.png'; declare module '*.html'; declare module '*.svelte'; +declare module '*?inlineWorker' { + const value: () => Worker; + export default value; +} + declare module "*.png?base64" { const value: string; export default value; diff --git a/src/manifests/manifest.json b/src/manifests/manifest.json index 82f6c37e..9ccf3f2f 100644 --- a/src/manifests/manifest.json +++ b/src/manifests/manifest.json @@ -32,15 +32,7 @@ ], "web_accessible_resources": [ { - "resources": ["*/*"], - "matches": ["*://*/*"] - }, - { - "resources": ["resources/*"], - "matches": ["*://*/*"] - }, - { - "resources": ["seqta/utils/migration/migrate.html"], + "resources": ["*/*", "resources/*", "seqta/utils/migration/migrate.html", "plugins/built-in/globalSearch/*"], "matches": ["*://*/*"] } ] diff --git a/src/plugins/built-in/globalSearch/SearchBar.svelte b/src/plugins/built-in/globalSearch/SearchBar.svelte index a9838714..6d0a3bfc 100644 --- a/src/plugins/built-in/globalSearch/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/SearchBar.svelte @@ -3,12 +3,12 @@ import { settingsState } from '@/seqta/utils/listeners/SettingsState' import { fade, scale } from 'svelte/transition'; import { circOut, quintOut } from 'svelte/easing'; - import { type StaticCommandItem } from './commands'; - import type { CombinedResult } from './types'; + import { type StaticCommandItem } from './core/commands'; + import type { CombinedResult } from './core/types'; import { createSearchIndexes, performSearch as doSearch } from './searchUtils'; import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from './highlightUtils'; import Fuse from 'fuse.js'; - import Calculator from './Calculator.svelte'; + import Calculator from './components/Calculator.svelte'; import { actionMap } from './indexing/actions'; import type { IndexItem, HydratedIndexItem } from './indexing/types'; diff --git a/src/plugins/built-in/globalSearch/client-vector-search-docs.md b/src/plugins/built-in/globalSearch/client-vector-search-docs.md index 97fab159..3af71ef8 100644 --- a/src/plugins/built-in/globalSearch/client-vector-search-docs.md +++ b/src/plugins/built-in/globalSearch/client-vector-search-docs.md @@ -20,64 +20,63 @@ Our goal is to build a super simple, fast vector search that works with couple h We'll initially keep things super simple and sub 100ms ### TODOs + - [ ] add HNSW index that works on node and browser env, don't rely on hnsw binder libs - [ ] add a proper testing suite and ci/cd for the lib - [ ] simple health tests - [ ] mock the @xenova/transformers for jest, it's not happy with it - [ ] performance tests, recall, memory usage, cpu usage etc. - ## Installation ```bash npm i client-vector-search ``` - ## Quickstart This library provides a plug-and-play solution for embedding and vector search. It's designed to be easy to use, efficient, and versatile. Here's a quick start guide: - ```ts - import { getEmbedding, EmbeddingIndex } from 'client-vector-search'; +import { getEmbedding, EmbeddingIndex } from "client-vector-search"; - // getEmbedding is an async function, so you need to use 'await' or '.then()' to get the result - const embedding = await getEmbedding("Apple"); // Returns embedding as number[] +// getEmbedding is an async function, so you need to use 'await' or '.then()' to get the result +const embedding = await getEmbedding("Apple"); // Returns embedding as number[] - // Each object should have an 'embedding' property of type number[] - const initialObjects = [ +// Each object should have an 'embedding' property of type number[] +const initialObjects = [ { id: 1, name: "Apple", embedding: embedding }, { id: 2, name: "Banana", embedding: await getEmbedding("Banana") }, - { id: 3, name: "Cheddar", embedding: await getEmbedding("Cheddar")}, - { id: 4, name: "Space", embedding: await getEmbedding("Space")}, - { id: 5, name: "database", embedding: await getEmbedding("database")}, - ]; - const index = new EmbeddingIndex(initialObjects); // Creates an index + { id: 3, name: "Cheddar", embedding: await getEmbedding("Cheddar") }, + { id: 4, name: "Space", embedding: await getEmbedding("Space") }, + { id: 5, name: "database", embedding: await getEmbedding("database") }, +]; +const index = new EmbeddingIndex(initialObjects); // Creates an index - // The query should be an embedding of type number[] - const queryEmbedding = await getEmbedding('Fruit'); // Query embedding - const results = await index.search(queryEmbedding, { topK: 5 }); // Returns top similar objects +// The query should be an embedding of type number[] +const queryEmbedding = await getEmbedding("Fruit"); // Query embedding +const results = await index.search(queryEmbedding, { topK: 5 }); // Returns top similar objects - // specify the storage type - await index.saveIndex('indexedDB'); - const results = await index.search([1, 2, 3], { - topK: 5, - useStorage: 'indexedDB', - // storageOptions: { // use only if you overrode the defaults - // indexedDBName: 'clientVectorDB', - // indexedDBObjectStoreName: 'ClientEmbeddingStore', - // }, - }); +// specify the storage type +await index.saveIndex("indexedDB"); +const results = await index.search([1, 2, 3], { + topK: 5, + useStorage: "indexedDB", + // storageOptions: { // use only if you overrode the defaults + // indexedDBName: 'clientVectorDB', + // indexedDBObjectStoreName: 'ClientEmbeddingStore', + // }, +}); - console.log(results); +console.log(results); - await index.deleteIndexedDB(); // if you overrode default, specify db name +await index.deleteIndexedDB(); // if you overrode default, specify db name ``` ## Trouble-shooting ### NextJS + To use it inside NextJS projects you'll need to update the `next.config.js` file to include the following: ```js @@ -120,26 +119,31 @@ Until we have a reference documentation, you can find all the methods and their Let's get started! ### Step 1: Generate Embeddings for String + Generate embeddings for a given string using the `getEmbedding` method. ```ts const embedding = await getEmbedding("Apple"); // Returns embedding as number[] ``` + > **Note**: `getEmbedding` is asynchronous; make sure to use `await`. --- ### Step 2: Calculate Cosine Similarity + Calculate the cosine similarity between two embeddings. ```ts const similarity = cosineSimilarity(embedding1, embedding2, 6); ``` + > **Note**: Both embeddings should be of the same length. --- ### Step 3: Create an Index + Create an index with an initial array of objects. Each object must have an 'embedding' property. ```ts @@ -150,26 +154,37 @@ const index = new EmbeddingIndex(initialObjects); --- ### Step 4: Add to Index + Add an object to the index. ```ts -const objectToAdd = { id: 6, name: 'Cat', embedding: await getEmbedding('Cat') }; +const objectToAdd = { + id: 6, + name: "Cat", + embedding: await getEmbedding("Cat"), +}; index.add(objectToAdd); ``` --- ### Step 5: Update Index + Update an existing object in the index. ```ts -const vectorToUpdate = { id: 6, name: 'Dog', embedding: await getEmbedding('Dog') }; +const vectorToUpdate = { + id: 6, + name: "Dog", + embedding: await getEmbedding("Dog"), +}; index.update({ id: 6 }, vectorToUpdate); ``` --- ### Step 6: Remove from Index + Remove an object from the index. ```ts @@ -179,6 +194,7 @@ index.remove({ id: 6 }); --- ### Step 7: Retrieve from Index + Retrieve an object from the index. ```ts @@ -188,16 +204,18 @@ const vector = index.get({ id: 1 }); --- ### Step 8: Search the Index + Search the index with a query embedding. ```ts -const queryEmbedding = await getEmbedding('Fruit'); +const queryEmbedding = await getEmbedding("Fruit"); const results = await index.search(queryEmbedding, { topK: 5 }); ``` --- ### Step 9: Print the Index + Print the entire index to the console. ```ts @@ -207,18 +225,23 @@ index.printIndex(); --- ### Step 10: Save Index to IndexedDB (for browser) + Save the index to a persistent IndexedDB database. Note ```ts -await index.saveIndex("indexedDB", { DBName: "clientVectorDB", objectStoreName:"ClientEmbeddingStore"}) +await index.saveIndex("indexedDB", { + DBName: "clientVectorDB", + objectStoreName: "ClientEmbeddingStore", +}); ``` --- ### Important: Search in indexedDB + Perform a search operation in the IndexedDB. -```ts +````ts const results = await index.search(queryEmbedding, { topK: 5, useStorage: "indexedDB", @@ -235,30 +258,36 @@ To delete an entire database. ```ts await IndexedDbManager.deleteIndexedDB("clientVectorDB"); -``` +```` --- ### Delete Object Store + To delete an object store from a database. ```ts -await IndexedDbManager.deleteIndexedDBObjectStore("clientVectorDB", "ClientEmbeddingStore"); +await IndexedDbManager.deleteIndexedDBObjectStore( + "clientVectorDB", + "ClientEmbeddingStore", +); ``` --- ### Retrieve All Objects + To retrieve all objects from a specific object store. ```ts -const allObjects = await IndexedDbManager.getAllObjectsFromIndexedDB("clientVectorDB", "ClientEmbeddingStore"); +const allObjects = await IndexedDbManager.getAllObjectsFromIndexedDB( + "clientVectorDB", + "ClientEmbeddingStore", +); ``` - - - # THE MAIN INDEX.TS FILE THAT YOU ARE IMPORTING FROM + ```index.ts const DEFAULT_TOP_K = 3; @@ -594,4 +623,4 @@ export class EmbeddingIndex { return objects; } } -``` \ No newline at end of file +``` diff --git a/src/plugins/built-in/globalSearch/commands.ts b/src/plugins/built-in/globalSearch/commands.ts deleted file mode 100644 index f5b6ae75..00000000 --- a/src/plugins/built-in/globalSearch/commands.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { settingsState } from '@/seqta/utils/listeners/SettingsState'; -import { loadHomePage } from '@/seqta/utils/Loaders/LoadHomePage'; - -export interface BaseCommandItem { - id: string; - text: string; - category: string; - icon: string; - action: () => void; - keywords?: string[]; - priority?: number; -} - -export interface StaticCommandItem extends BaseCommandItem { - keybind?: string[]; - keybindLabel?: string[]; -} - -const staticCommands: StaticCommandItem[] = [ - { - id: 'home', - icon: '\ueb4c', - category: 'navigation', - text: 'Home', - keybind: ['alt+h'], - keybindLabel: ['Alt', 'H'], - action: () => { - window.location.hash = '?page=/home'; - loadHomePage(); - }, - priority: 4 - }, - { - id: 'messages', - icon: '\uebfd', - category: 'navigation', - text: 'Direct Messages', - keybind: ['alt+m'], - keybindLabel: ['Alt', 'M'], - action: () => { - window.location.hash = '?page=/messages'; - }, - priority: 4 - }, - { - id: 'timetable', - icon: '\ue9cd', - category: 'navigation', - text: 'Timetable', - keybind: ['alt+t'], - keybindLabel: ['Alt', 'T'], - action: () => { - window.location.hash = '?page=/timetable'; - }, - priority: 4 - }, - { - id: 'assessments', - icon: '\ueac3', - category: 'navigation', - text: 'Assessments', - keybind: ['alt+a'], - keybindLabel: ['Alt', 'A'], - action: () => { - window.location.hash = '?page=/assessments'; - }, - priority: 4 - }, - { - id: 'toggle-dark-mode', - icon: '\uecfe', - category: 'action', - text: 'Toggle Dark Mode', - action: () => settingsState.DarkMode = !settingsState.DarkMode, - priority: 2, - keywords: ['theme', 'appearance'] - } -]; - -/** - * Returns the predefined list of static commands. - */ -export const getStaticCommands = (): StaticCommandItem[] => { - return [...staticCommands]; -}; \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte b/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte index 1d5dde71..89daf445 100644 --- a/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte +++ b/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte @@ -1,7 +1,7 @@