From 1f3354c47bc1fab73c40f6e529db512aec6cc033 Mon Sep 17 00:00:00 2001 From: SethBurkart123 Date: Mon, 31 Mar 2025 22:54:03 +1100 Subject: [PATCH 01/45] feat: add global search UI --- package.json | 1 + .../built-in/globalSearch/SearchBar.svelte | 202 ++++++++++++++++++ src/plugins/built-in/globalSearch/index.ts | 93 ++++++++ src/plugins/built-in/globalSearch/styles.css | 68 ++++++ src/plugins/index.ts | 3 + 5 files changed, 367 insertions(+) create mode 100644 src/plugins/built-in/globalSearch/SearchBar.svelte create mode 100644 src/plugins/built-in/globalSearch/index.ts create mode 100644 src/plugins/built-in/globalSearch/styles.css diff --git a/package.json b/package.json index 3cd068bf..1ccada5d 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "dompurify": "^3.2.4", "embla-carousel-autoplay": "^8.5.2", "embla-carousel-svelte": "^8.5.2", + "flexsearch": "^0.8.147", "fuse.js": "^7.1.0", "idb": "^8.0.2", "localforage": "^1.10.0", diff --git a/src/plugins/built-in/globalSearch/SearchBar.svelte b/src/plugins/built-in/globalSearch/SearchBar.svelte new file mode 100644 index 00000000..82de5623 --- /dev/null +++ b/src/plugins/built-in/globalSearch/SearchBar.svelte @@ -0,0 +1,202 @@ + + +{#if commandPalleteOpen} + +{/if} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/index.ts b/src/plugins/built-in/globalSearch/index.ts new file mode 100644 index 00000000..2d5116d5 --- /dev/null +++ b/src/plugins/built-in/globalSearch/index.ts @@ -0,0 +1,93 @@ +import type { Plugin } from '@/plugins/core/types'; +import { BasePlugin } from '@/plugins/core/settings'; +import { booleanSetting, defineSettings, Setting, stringSetting } from '@/plugins/core/settingsHelpers'; +//import FlexSearch from 'flexsearch'; +import renderSvelte from '@/interface/main'; +import SearchBar from './SearchBar.svelte'; +import styles from './styles.css?inline'; +import { unmount } from 'svelte'; + +// Plugin settings +const settings = defineSettings({ + searchHotkey: stringSetting({ + default: 'ctrl+k', + title: 'Search Hotkey', + description: 'Keyboard shortcut to open the search (cmd on Mac)', + }), + showRecentFirst: booleanSetting({ + default: true, + title: 'Show Recent First', + description: 'Sort dynamic content by most recent first', + }) +}); + +class GlobalSearchPlugin extends BasePlugin { + @Setting(settings.searchHotkey) + searchHotkey!: string; + + @Setting(settings.showRecentFirst) + showRecentFirst!: boolean; +} + +const settingsInstance = new GlobalSearchPlugin(); + +const globalSearchPlugin: Plugin = { + id: 'global-search', + name: 'Global Search', + description: 'Quick search for everything in SEQTA', + version: '1.0.0', + settings: settingsInstance.settings, + disableToggle: true, + + // Add some basic styles for our search UI + styles: styles, + + run: async (api) => { + let app: any; + + // Create search button + api.seqta.onMount('#title', (titleElement) => { + // Create search button + const searchButton = document.createElement('div'); + searchButton.className = 'search-trigger'; + searchButton.innerHTML = ` + + + + + +

Quick search...

+ ⌘K + `; + + // Add button before the title + titleElement.appendChild(searchButton); + + // Create shadow DOM for Svelte component + const searchRoot = document.createElement('div'); + document.body.appendChild(searchRoot); + const searchRootShadow = searchRoot.attachShadow({ mode: 'open' }); + + // Mount Svelte component in shadow DOM + app = renderSvelte(SearchBar, searchRootShadow); + + // Handle click on search button + searchButton.addEventListener('click', () => { + // @ts-ignore + window.setCommandPalleteOpen(true); + }); + }); + + // Clean up + return () => { + const searchButton = document.querySelector('.search-trigger'); + const searchRoot = document.querySelector('.global-search-root'); + if (searchButton) searchButton.remove(); + if (searchRoot) searchRoot.remove(); + + unmount(app); + }; + } +}; + +export default globalSearchPlugin; \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/styles.css b/src/plugins/built-in/globalSearch/styles.css new file mode 100644 index 00000000..3a9911be --- /dev/null +++ b/src/plugins/built-in/globalSearch/styles.css @@ -0,0 +1,68 @@ +.search-trigger { + display: flex; + align-items: center; + justify-content: center; + height: 32px; + margin-left: 10px; + border-radius: 8px; + cursor: pointer; + transition: all 0.2s ease; + margin-right: auto !important; + padding: 3px 12px; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); + backdrop-filter: blur(4px); + + svg { + opacity: 0.8; + } + + p { + font-size: 14px; + margin-left: 8px; + margin-right: 16px; + height: 100%; + margin-bottom: 0; + line-height: 32px; + font-weight: 400; + } +} + +/* Light mode styles */ +.search-trigger { + background-color: rgba(248, 250, 252, 0.9) !important; + border: 1px solid rgba(0, 0, 0, 0.1) !important; + color: #555 !important; + + p { + color: #555 !important; + } + + svg { + color: #555; + } +} + +.search-trigger:hover { + background-color: rgba(248, 250, 252, 1) !important; + color: #333 !important; +} + +/* Dark mode styles */ +.dark .search-trigger { + background-color: rgba(30, 41, 59, 0.7) !important; + border: 1px solid rgba(255, 255, 255, 0.1) !important; + color: #aaa !important; + + p { + color: #aaa !important; + } + + svg { + color: #aaa; + } +} + +.dark .search-trigger:hover { + background-color: rgba(30, 41, 59, 0.9) !important; + color: #eee !important; +} \ No newline at end of file diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 5332e966..539600d5 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -6,6 +6,8 @@ import notificationCollectorPlugin from './built-in/notificationCollector'; import themesPlugin from './built-in/themes'; import animatedBackgroundPlugin from './built-in/animatedBackground'; import assessmentsAveragePlugin from './built-in/assessmentsAverage'; +import globalSearchPlugin from './built-in/globalSearch'; + // Initialize plugin manager const pluginManager = PluginManager.getInstance(); @@ -15,6 +17,7 @@ pluginManager.registerPlugin(animatedBackgroundPlugin); pluginManager.registerPlugin(assessmentsAveragePlugin); pluginManager.registerPlugin(notificationCollectorPlugin); pluginManager.registerPlugin(timetablePlugin); +pluginManager.registerPlugin(globalSearchPlugin); //pluginManager.registerPlugin(testPlugin); export { init as Monofile } from './monofile'; From adbba730c46b936fc1a479b73a1e4303492eee2c Mon Sep 17 00:00:00 2001 From: SethBurkart123 Date: Mon, 31 Mar 2025 23:03:45 +1100 Subject: [PATCH 02/45] style: search styling improvements --- .../built-in/globalSearch/SearchBar.svelte | 4 ++-- src/plugins/built-in/globalSearch/styles.css | 19 ++++--------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/plugins/built-in/globalSearch/SearchBar.svelte b/src/plugins/built-in/globalSearch/SearchBar.svelte index 82de5623..4a7546dc 100644 --- a/src/plugins/built-in/globalSearch/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/SearchBar.svelte @@ -153,14 +153,14 @@ role="button" tabindex="0">
-
+
{'\ueca5'}
diff --git a/src/plugins/built-in/globalSearch/styles.css b/src/plugins/built-in/globalSearch/styles.css index 3a9911be..42b1942d 100644 --- a/src/plugins/built-in/globalSearch/styles.css +++ b/src/plugins/built-in/globalSearch/styles.css @@ -19,7 +19,7 @@ p { font-size: 14px; margin-left: 8px; - margin-right: 16px; + margin-right: 48px; height: 100%; margin-bottom: 0; line-height: 32px; @@ -29,7 +29,7 @@ /* Light mode styles */ .search-trigger { - background-color: rgba(248, 250, 252, 0.9) !important; + background-color: rgba(248, 250, 252, 0.05) !important; border: 1px solid rgba(0, 0, 0, 0.1) !important; color: #555 !important; @@ -42,14 +42,8 @@ } } -.search-trigger:hover { - background-color: rgba(248, 250, 252, 1) !important; - color: #333 !important; -} - -/* Dark mode styles */ .dark .search-trigger { - background-color: rgba(30, 41, 59, 0.7) !important; + background-color: rgba(0, 0, 0, 0.03) !important; border: 1px solid rgba(255, 255, 255, 0.1) !important; color: #aaa !important; @@ -60,9 +54,4 @@ svg { color: #aaa; } -} - -.dark .search-trigger:hover { - background-color: rgba(30, 41, 59, 0.9) !important; - color: #eee !important; -} \ No newline at end of file +} \ No newline at end of file From 068e4ab7786807263f63e41f11792ddbe4e89c1b Mon Sep 17 00:00:00 2001 From: SethBurkart123 Date: Mon, 31 Mar 2025 23:06:56 +1100 Subject: [PATCH 03/45] style: further UI tweaks --- src/plugins/built-in/globalSearch/SearchBar.svelte | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/plugins/built-in/globalSearch/SearchBar.svelte b/src/plugins/built-in/globalSearch/SearchBar.svelte index 4a7546dc..9346b4b5 100644 --- a/src/plugins/built-in/globalSearch/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/SearchBar.svelte @@ -134,7 +134,7 @@ transition:fade={{ duration: 150 }} >
-
commandPalleteOpen = false} onkeydown={(e) => e.key === 'Escape' && (commandPalleteOpen = false)} role="button" @@ -166,11 +166,11 @@
{#if filteredItems.length > 0} -
    +
      {#each filteredItems as item, i (item.text)}
    • + + \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/dynamicSearch.ts b/src/plugins/built-in/globalSearch/dynamicSearch.ts index 230cc6ac..7955b1d1 100644 --- a/src/plugins/built-in/globalSearch/dynamicSearch.ts +++ b/src/plugins/built-in/globalSearch/dynamicSearch.ts @@ -1,30 +1,30 @@ +import type { SvelteComponent } from 'svelte'; +import type { HydratedIndexItem } from './indexing/types'; + export interface DynamicContentItem { - id: string; + id: string; text: string; category: string; - icon: string; - action: () => void; - keywords?: string[]; - contentType: 'message' | 'course' | 'assessment' | 'other'; content: string; dateAdded: number; - metadata?: Record; - priority?: number; + metadata: Record; + actionId: string; + renderComponentId: string; + renderComponent?: typeof SvelteComponent; } -let dynamicItems: DynamicContentItem[] = []; +let dynamicItems: HydratedIndexItem[] = []; /** * Loads a new set of dynamic items. */ -export const loadDynamicItems = (items: DynamicContentItem[]) => { - dynamicItems = [...items]; - console.log(`Loaded ${items.length} dynamic items.`); -}; +export function loadDynamicItems(items: HydratedIndexItem[]) { + dynamicItems = items; +} /** * Returns all currently loaded dynamic items. */ -export const getAllDynamicItems = (): DynamicContentItem[] => { - return [...dynamicItems]; -}; \ No newline at end of file +export function getDynamicItems(): HydratedIndexItem[] { + return dynamicItems; +} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/index.ts b/src/plugins/built-in/globalSearch/index.ts index 016c11d0..c7adce94 100644 --- a/src/plugins/built-in/globalSearch/index.ts +++ b/src/plugins/built-in/globalSearch/index.ts @@ -5,8 +5,9 @@ import renderSvelte from '@/interface/main'; import SearchBar from './SearchBar.svelte'; import styles from './styles.css?inline'; import { unmount } from 'svelte'; -import { type DynamicContentItem, loadDynamicItems } from './dynamicSearch'; +import { loadDynamicItems } from './dynamicSearch'; import { waitForElm } from '@/seqta/utils/waitForElm'; +import { runIndexing, loadAllStoredItems } from './indexing/indexer'; const settings = defineSettings({ searchHotkey: stringSetting({ @@ -24,6 +25,11 @@ const settings = defineSettings({ title: 'Transparency Effects', description: 'Enable transparency effects for the search bar', }), + runIndexingOnLoad: booleanSetting({ + default: true, + title: 'Index on Page Load', + description: 'Run content indexing when SEQTA loads', + }), }); class GlobalSearchPlugin extends BasePlugin { @@ -35,11 +41,14 @@ class GlobalSearchPlugin extends BasePlugin { @Setting(settings.transparencyEffects) transparencyEffects!: boolean; + + @Setting(settings.runIndexingOnLoad) + runIndexingOnLoad!: boolean; } const settingsInstance = new GlobalSearchPlugin(); -const createSampleDynamicData = (): DynamicContentItem[] => { +/* const createSampleDynamicData = (): DynamicContentItem[] => { const sampleMessages = [ { id: 'message_1', @@ -86,6 +95,14 @@ const createSampleDynamicData = (): DynamicContentItem[] => { ]; return [...sampleMessages, ...sampleCourses, ...sampleAssessments]; +}; */ + +// Update dynamic items directly from the indexer without conversion +const updateDynamicItemsFromIndex = async () => { + const indexedItems = await loadAllStoredItems(); + loadDynamicItems(indexedItems); + console.log(`Loaded ${indexedItems.length} indexed items into search.`); + window.dispatchEvent(new CustomEvent('dynamic-items-updated')); }; const globalSearchPlugin: Plugin = { @@ -99,15 +116,20 @@ const globalSearchPlugin: Plugin = { run: async (api) => { let app: any; - - const dynamicData = createSampleDynamicData(); - loadDynamicItems(dynamicData); + + // Run initial indexing and update dynamic items + if (api.settings.runIndexingOnLoad) { + setTimeout(async () => { + await runIndexing(); + await updateDynamicItemsFromIndex(); + }, 2000); // Delay initial indexing to let page load + } const mountSearchBar = (titleElement: Element) => { if (titleElement.querySelector('.search-trigger')) { return; } - // Create search button + const searchButton = document.createElement('div'); searchButton.className = 'search-trigger'; searchButton.innerHTML = ` @@ -120,26 +142,24 @@ const globalSearchPlugin: Plugin = { ⌘K `; - // Add button before the title titleElement.appendChild(searchButton); - // Create shadow DOM for Svelte component const searchRoot = document.createElement('div'); document.body.appendChild(searchRoot); const searchRootShadow = searchRoot.attachShadow({ mode: 'open' }); console.log('adding event listener to search button'); - // Handle click on search button + searchButton.addEventListener('click', () => { console.log('search button clicked'); - // @ts-ignore + // @ts-ignore - Intentionally adding to window window.setCommandPalleteOpen(true); }); - // Mount Svelte component in shadow DOM try { app = renderSvelte(SearchBar, searchRootShadow, { transparencyEffects: api.settings.transparencyEffects ? true : false, + showRecentFirst: api.settings.showRecentFirst }); } catch (error) { console.error('Error rendering Svelte component:', error); diff --git a/src/plugins/built-in/globalSearch/indexing/actions.ts b/src/plugins/built-in/globalSearch/indexing/actions.ts new file mode 100644 index 00000000..fe2f2ae4 --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/actions.ts @@ -0,0 +1,40 @@ +import type { IndexItem } from './types'; + +interface MessageMetadata { + messageId: number; + author: string; + senderId: number; + senderType: string; + timestamp: string; + hasAttachments: boolean; + attachmentCount: number; + read: boolean; +} + +interface AssessmentMetadata { + assessmentId?: number; + messageId?: number; + subject?: string; + term?: string; + programmeId?: number; + metaclassId?: number; + timestamp: string; + isMessageBased?: boolean; + author?: string; +} + +type ActionHandler = (item: IndexItem & { metadata: T }) => void; + +export const actionMap: Record> = { + message: ((item: IndexItem & { metadata: MessageMetadata }) => { + window.location.hash = `#?page=/messages&id=${item.metadata.messageId}`; + }) as ActionHandler, + + assessment: ((item: IndexItem & { metadata: AssessmentMetadata }) => { + if (item.metadata.isMessageBased) { + window.location.hash = `#?page=/messages&id=${item.metadata.messageId}`; + } else { + window.location.hash = `#?page=/assessments&id=${item.metadata.assessmentId}`; + } + }) as ActionHandler +}; \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/indexing/db.ts b/src/plugins/built-in/globalSearch/indexing/db.ts new file mode 100644 index 00000000..6241e5b4 --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/db.ts @@ -0,0 +1,198 @@ +const DB_NAME = 'betterseqta-index'; +const META_STORE = 'meta'; +const VERSION_KEY = 'betterseqta-index-version'; + +let dbPromise: Promise | null = null; + +// Get the current version from localStorage or start at 1 +function getCurrentVersion(): number { + const storedVersion = localStorage.getItem(VERSION_KEY); + return storedVersion ? parseInt(storedVersion, 10) : 1; +} + +// Update the version in localStorage +function updateVersion(version: number) { + localStorage.setItem(VERSION_KEY, version.toString()); +} + +function openDB(): Promise { + if (dbPromise) return dbPromise; + + const currentVersion = getCurrentVersion(); + + dbPromise = new Promise((resolve, reject) => { + let request: IDBOpenDBRequest; + + try { + request = indexedDB.open(DB_NAME, currentVersion); + } catch (e) { + // If there's a version error, try to delete the database and start fresh + console.warn('Database version conflict, recreating database...'); + indexedDB.deleteDatabase(DB_NAME); + localStorage.removeItem(VERSION_KEY); + request = indexedDB.open(DB_NAME, 1); + updateVersion(1); + } + + request.onupgradeneeded = (event) => { + const db = request.result; + const existingStores = Array.from(db.objectStoreNames); + + // Always ensure META_STORE exists + if (!existingStores.includes(META_STORE)) { + db.createObjectStore(META_STORE); + } + + // Update version in localStorage to match the database + updateVersion(event.newVersion || 1); + }; + + request.onsuccess = () => resolve(request.result); + + request.onerror = () => { + console.error('Error opening database:', request.error); + // If there's an error, try to recover by deleting and recreating + indexedDB.deleteDatabase(DB_NAME); + localStorage.removeItem(VERSION_KEY); + reject(request.error); + }; + }); + + return dbPromise; +} + +async function getStore(store: string, mode: IDBTransactionMode = 'readonly') { + const db = await openDB(); + + // Create store dynamically if needed + if (!db.objectStoreNames.contains(store)) { + db.close(); + await upgradeDB(store); + return getStore(store, mode); + } + + const tx = db.transaction(store, mode); + return tx.objectStore(store); +} + +function upgradeDB(newStore: string): Promise { + return new Promise((resolve, reject) => { + const currentVersion = getCurrentVersion(); + const newVersion = currentVersion + 1; + + // Close any existing connections + if (dbPromise) { + dbPromise.then(db => db.close()); + dbPromise = null; + } + + const request = indexedDB.open(DB_NAME, newVersion); + + request.onupgradeneeded = (event) => { + const db = request.result; + if (!db.objectStoreNames.contains(newStore)) { + db.createObjectStore(newStore); + } + // Update version in localStorage + updateVersion(event.newVersion || newVersion); + }; + + request.onsuccess = () => { + dbPromise = Promise.resolve(request.result); + resolve(); + }; + + request.onerror = () => { + console.error('Error upgrading database:', request.error); + reject(request.error); + }; + }); +} + +export async function getAll(store: string): Promise { + try { + const s = await getStore(store); + return new Promise((resolve, reject) => { + const req = s.getAll(); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in getAll for store ${store}:`, error); + return []; + } +} + +export async function get(store: string, key: string): Promise { + try { + const s = await getStore(store); + return new Promise((resolve, reject) => { + const req = s.get(key); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in get for store ${store}, key ${key}:`, error); + return null; + } +} + +export async function put(store: string, value: any, key?: string): Promise { + try { + const s = await getStore(store, 'readwrite'); + return new Promise((resolve, reject) => { + const req = key ? s.put(value, key) : s.put(value); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in put for store ${store}:`, error); + throw error; + } +} + +export async function remove(store: string, key: string): Promise { + try { + const s = await getStore(store, 'readwrite'); + return new Promise((resolve, reject) => { + const req = s.delete(key); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in remove for store ${store}, key ${key}:`, error); + throw error; + } +} + +export async function clear(store: string): Promise { + try { + const s = await getStore(store, 'readwrite'); + return new Promise((resolve, reject) => { + const req = s.clear(); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in clear for store ${store}:`, error); + throw error; + } +} + +// Helper function to reset the database if needed +export async function resetDatabase(): Promise { + if (dbPromise) { + const db = await dbPromise; + db.close(); + dbPromise = null; + } + + return new Promise((resolve, reject) => { + const req = indexedDB.deleteDatabase(DB_NAME); + req.onsuccess = () => { + localStorage.removeItem(VERSION_KEY); + resolve(); + }; + req.onerror = () => reject(req.error); + }); +} diff --git a/src/plugins/built-in/globalSearch/indexing/indexer.ts b/src/plugins/built-in/globalSearch/indexing/indexer.ts new file mode 100644 index 00000000..6bd21c0b --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/indexer.ts @@ -0,0 +1,158 @@ +import { getAll, put, clear, remove } from './db'; +import { jobs } from './jobs'; +import { renderComponentMap } from './renderComponents'; +import type { IndexItem, HydratedIndexItem, Job, JobContext } from './types'; + +const META_STORE = 'meta'; +const LOCK_KEY = 'bsq-indexer-lock'; +const HEARTBEAT_INTERVAL = 10000; +const LOCK_TIMEOUT = 20000; + +let heartbeatTimer: ReturnType | null = null; + +function shouldRun(job: Job, lastRun?: number): boolean { + const now = Date.now(); + + if (job.frequency === 'pageLoad') return true; + if (!lastRun) return true; + + if (job.frequency.type === 'interval') { + return now - lastRun >= job.frequency.ms; + } + + if (job.frequency.type === 'expiry') { + return now - lastRun >= job.frequency.afterMs; + } + + return false; +} + +function getLastRunMeta(jobId: string): Promise { + return getAll(META_STORE).then(metaItems => { + const match = metaItems.find((m: any) => m.jobId === jobId); + return match?.lastRun; + }); +} + +async function updateLastRunMeta(jobId: string): Promise { + await put(META_STORE, { jobId, lastRun: Date.now() }, jobId); +} + +function shouldIndex(): boolean { + const last = parseInt(localStorage.getItem(LOCK_KEY) || '0', 10); + return isNaN(last) || Date.now() - last > LOCK_TIMEOUT; +} + +function startHeartbeat() { + localStorage.setItem(LOCK_KEY, `${Date.now()}`); + heartbeatTimer = setInterval(() => { + localStorage.setItem(LOCK_KEY, `${Date.now()}`); + }, HEARTBEAT_INTERVAL); +} + +function stopHeartbeat() { + if (heartbeatTimer) clearInterval(heartbeatTimer); + localStorage.removeItem(LOCK_KEY); +} + +function dispatchProgress(completed: number, total: number, indexing: boolean) { + const event = new CustomEvent('indexing-progress', { + detail: { completed, total, indexing } + }); + window.dispatchEvent(event); +} + +export async function loadAllStoredItems(): Promise { + const all: HydratedIndexItem[] = []; + + for (const jobId in jobs) { + const items = await getAll(jobId); + const job = jobs[jobId]; + const renderComponent = renderComponentMap[job.renderComponentId]; + + for (const item of items) { + all.push({ + ...item, + renderComponent, + }); + } + } + + return all; +} + +export async function runIndexing(): Promise { + if (!shouldIndex()) { + console.debug('%c[Indexer] Skipping indexing (another tab has the lock)', 'color: gray'); + return; + } + + startHeartbeat(); + console.debug('%c[Indexer] Starting indexing...', 'color: green'); + + const jobIds = Object.keys(jobs); + let completedJobs = 0; + dispatchProgress(completedJobs, jobIds.length, true); + + for (const jobId of jobIds) { + const job = jobs[jobId]; + const lastRun = await getLastRunMeta(jobId); + + if (!shouldRun(job, lastRun)) { + console.debug(`%c[Indexer] Skipping job "${jobId}" (not due)`, 'color: gray'); + completedJobs++; + dispatchProgress(completedJobs, jobIds.length, true); + continue; + } + + const getStoredItems = async () => await getAll(jobId); + const setStoredItems = async (items: IndexItem[]) => { + await clear(jobId); + await Promise.all(items.map(i => put(jobId, i, i.id))); + }; + const addItem = async (item: IndexItem) => { + await put(jobId, item, item.id); + }; + const removeItem = async (id: string) => { + await remove(jobId, id); + }; + + const ctx: JobContext = { + getStoredItems, + setStoredItems, + addItem, + removeItem, + }; + + console.debug(`%c[Indexer] Running job "${jobId}"...`, 'color: #4ea1ff'); + + try { + const newItems = await job.run(ctx); + const stored = await getStoredItems(); + + let merged = mergeItems(stored, newItems); + if (job.purge) merged = job.purge(merged); + + await setStoredItems(merged); + await updateLastRunMeta(jobId); + + console.debug(`%c[Indexer] ✅ ${job.label}: ${newItems.length} items indexed`, 'color: #00c46f'); + } catch (err) { + console.debug(`%c[Indexer] ❌ ${job.label} failed:`, 'color: red'); + console.error(err); + } + + completedJobs++; + dispatchProgress(completedJobs, jobIds.length, true); + } + + stopHeartbeat(); + dispatchProgress(completedJobs, jobIds.length, false); +} + +function mergeItems(existing: IndexItem[], incoming: IndexItem[]): IndexItem[] { + const map = new Map(); + for (const item of existing) map.set(item.id, item); + for (const item of incoming) map.set(item.id, item); + return Array.from(map.values()); +} diff --git a/src/plugins/built-in/globalSearch/indexing/jobs.ts b/src/plugins/built-in/globalSearch/indexing/jobs.ts new file mode 100644 index 00000000..7da9ac7a --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/jobs.ts @@ -0,0 +1,341 @@ +import type { Job } from './types'; +import type { IndexItem } from './types'; + +interface MessageNotification { + notificationID: number; + type: 'message'; + message: { + subtitle: string; + messageID: number; + title: string; + }; + timestamp: string; +} + +interface AssessmentNotification { + notificationID: number; + type: 'coneqtassessments'; + coneqtAssessments: { + programmeID: number; + metaclassID: number; + subtitle: string; + term: string; + title: string; + assessmentID: number; + subjectCode: string; + }; + timestamp: string; +} + +type Notification = MessageNotification | AssessmentNotification; + +interface MessageListResponse { + payload: { + hasMore: boolean; + messages: { + date: string; + attachments: boolean; + attachmentCount: number; + read: number; + sender: string; + sender_id: number; + sender_type: string; + subject: string; + id: number; + participants: Array<{ + name: string; + photo: string; + type: string; + }>; + }[]; + ts: string; + }; + status: string; +} + +interface MessageContentResponse { + payload: { + date: string; + blind: boolean; + read: boolean; + subject: string; + sender_type: string; + sender_id: number; + starred: boolean; + contents: string; + sender: string; + files: any[]; + id: number; + participants: Array<{ + read: number; + name: string; + photo: string; + id: number; + type: string; + }>; + }; + status: string; +} + +// Helper to strip HTML tags from text +function stripHtmlTags(html: string): string { + return html.replace(/<[^>]*>/g, ''); +} + +// Helper to fetch messages with pagination +async function fetchMessages(offset: number = 0, limit: number = 100): Promise { + const response = await fetch(`${location.origin}/seqta/student/load/message`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json; charset=utf-8' + }, + body: JSON.stringify({ + searchValue: "", + sortBy: "date", + sortOrder: "desc", + action: "list", + label: "inbox", + offset, + limit, + datetimeUntil: null + }) + }); + + return await response.json(); +} + +// Helper to fetch message content +async function fetchMessageContent(messageId: number): Promise { + const response = await fetch(`${location.origin}/seqta/student/load/message`, { + method: 'POST', + credentials: 'include', + headers: { + 'Content-Type': 'application/json; charset=utf-8' + }, + body: JSON.stringify({ + action: "message", + id: messageId + }) + }); + + return await response.json(); +} + +// Helper to fetch notifications +async function fetchNotifications(): Promise { + const response = await fetch(`${location.origin}/seqta/student/heartbeat?`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json; charset=utf-8' + }, + body: JSON.stringify({ + timestamp: "1970-01-01 00:00:00.0", + hash: "#?page=/notifications", + }) + }); + + const json = await response.json(); + return json.notifications ?? []; +} + +export const jobs: Record = { + messages: { + id: 'messages', + label: 'Messages', + renderComponentId: 'message', + frequency: { type: 'expiry', afterMs: 1000 * 60 * 5 }, // every 5 minutes + + run: async (ctx) => { + // Get existing items first + const existing = await ctx.getStoredItems(); + const existingIds = new Set(existing.map(i => i.id)); + const newItems: IndexItem[] = []; + let offset = 0; + const limit = 100; + let hasMore = true; + let consecutiveExisting = 0; + + // Fetch all messages with pagination + while (hasMore) { + try { + const response = await fetchMessages(offset, limit); + + if (response.status !== "200") { + console.error('Failed to fetch messages:', response); + break; + } + + const messages = response.payload.messages; + hasMore = response.payload.hasMore; + + // Process each message + for (const message of messages) { + const id = message.id.toString(); + + // Skip if we already have this message + if (existingIds.has(id)) { + consecutiveExisting++; + // If we've found 20 consecutive existing messages, assume we've caught up + if (consecutiveExisting >= 20) { + console.debug('[Messages Job] Found 20 consecutive existing messages, stopping fetch'); + hasMore = false; + break; + } + continue; + } + + // Reset consecutive counter when we find a new message + consecutiveExisting = 0; + + try { + // Fetch message content + const contentResponse = await fetchMessageContent(message.id); + + if (contentResponse.status !== "200") { + console.error('Failed to fetch message content:', contentResponse); + continue; + } + + const content = stripHtmlTags(contentResponse.payload.contents); + + newItems.push({ + id, + text: message.subject, + category: 'messages', + content: `From: ${message.sender}\n\n${content}`, + dateAdded: new Date(message.date).getTime(), + metadata: { + messageId: message.id, + author: message.sender, + senderId: message.sender_id, + senderType: message.sender_type, + timestamp: message.date, + hasAttachments: message.attachments, + attachmentCount: message.attachmentCount, + read: message.read === 1 + }, + actionId: 'message', + renderComponentId: 'message' + }); + + // Add to existingIds as we process to prevent duplicates in the same run + existingIds.add(id); + } catch (error) { + console.error('Error fetching message content:', error); + continue; + } + } + + offset += limit; + + // If we've processed 500 messages and haven't found any existing ones, + // assume these are all new (first run) and stop here to avoid overwhelming + if (offset >= 500 && consecutiveExisting === 0) { + console.debug('[Messages Job] Processed 500 new messages, stopping for now'); + hasMore = false; + break; + } + } catch (error) { + console.error('Error fetching messages:', error); + break; + } + + // Small delay to avoid overwhelming the server + await new Promise(resolve => setTimeout(resolve, 100)); + } + + console.debug(`[Messages Job] Found ${newItems.length} new messages`); + return newItems; + }, + + purge: (items) => { + // Keep messages from the last 30 days + const cutoff = Date.now() - (30 * 24 * 60 * 60 * 1000); + return items.filter(i => i.dateAdded >= cutoff); + } + }, + + assessments: { + id: 'assessments', + label: 'Assessments', + renderComponentId: 'assessment', + frequency: { type: 'expiry', afterMs: 1000 * 60 * 15 }, // every 15 minutes + + run: async (ctx) => { + const notifications = await fetchNotifications(); + const assessmentNotifications = notifications.filter((n): n is (MessageNotification | AssessmentNotification) => + n.type === 'coneqtassessments' || + (n.type === 'message' && n.message.title.toLowerCase().includes('assessment')) + ); + + const existing = await ctx.getStoredItems(); + const existingIds = new Set(existing.map(i => i.id)); + const newItems: IndexItem[] = []; + + for (const notification of assessmentNotifications) { + const id = notification.notificationID.toString(); + if (existingIds.has(id)) continue; + + if (notification.type === 'coneqtassessments') { + const { coneqtAssessments: assessment } = notification; + newItems.push({ + id, + text: assessment.title, + category: 'assessments', + content: assessment.subtitle, + dateAdded: new Date(notification.timestamp).getTime(), + metadata: { + assessmentId: assessment.assessmentID, + subject: assessment.subjectCode, + term: assessment.term, + programmeId: assessment.programmeID, + metaclassId: assessment.metaclassID, + timestamp: notification.timestamp + }, + actionId: 'assessment', + renderComponentId: 'assessment' + }); + } else { + // Handle message-based assessments + const { message } = notification; + newItems.push({ + id, + text: message.title, + category: 'assessments', + content: `From: ${message.subtitle}`, + dateAdded: new Date(notification.timestamp).getTime(), + metadata: { + messageId: message.messageID, + author: message.subtitle, + timestamp: notification.timestamp, + isMessageBased: true + }, + actionId: 'assessment', + renderComponentId: 'assessment' + }); + } + } + + return newItems; + }, + + purge: (items) => { + // Keep assessments from the current year + const date = new Date(); + date.setMonth(0); // January + date.setDate(1); + date.setHours(0); + date.setMinutes(0); + date.setSeconds(0); + const cutoff = date.getTime(); + return items.filter(i => i.dateAdded >= cutoff); + } + }, + + // We can add more job types here as needed: + // - notices + // - timetable changes + // - homework + // etc. +}; diff --git a/src/plugins/built-in/globalSearch/indexing/renderComponents.ts b/src/plugins/built-in/globalSearch/indexing/renderComponents.ts new file mode 100644 index 00000000..603c5d5c --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/renderComponents.ts @@ -0,0 +1,10 @@ +import type { SvelteComponent } from 'svelte'; +import AssessmentComponent from '../components/AssessmentItem.svelte'; +// import other components as needed + +export const renderComponentMap: Record = { + assessment: AssessmentComponent as unknown as typeof SvelteComponent, + // messages: MessageComponent, + // subject: SubjectComponent, + // etc... +}; diff --git a/src/plugins/built-in/globalSearch/indexing/types.ts b/src/plugins/built-in/globalSearch/indexing/types.ts new file mode 100644 index 00000000..c33f3a61 --- /dev/null +++ b/src/plugins/built-in/globalSearch/indexing/types.ts @@ -0,0 +1,37 @@ +import type { SvelteComponent } from 'svelte'; + +export interface IndexItem { + id: string; + text: string; + category: string; + content: string; + dateAdded: number; + metadata: Record; + actionId: string; + renderComponentId: string; +} + +export interface HydratedIndexItem extends IndexItem { + renderComponent: typeof SvelteComponent; +} + +export type Frequency = + | 'pageLoad' + | { type: 'interval'; ms: number } + | { type: 'expiry'; afterMs: number }; + +export interface JobContext { + getStoredItems: () => Promise; + setStoredItems: (items: IndexItem[]) => Promise; + addItem: (item: IndexItem) => Promise; + removeItem: (id: string) => Promise; +} + +export interface Job { + id: string; + label: string; + frequency: Frequency; + renderComponentId: string; + run: (ctx: JobContext) => Promise; + purge?: (items: IndexItem[]) => IndexItem[]; +} diff --git a/src/plugins/built-in/globalSearch/searchUtils.ts b/src/plugins/built-in/globalSearch/searchUtils.ts index 9a7db5c4..c8ef3b96 100644 --- a/src/plugins/built-in/globalSearch/searchUtils.ts +++ b/src/plugins/built-in/globalSearch/searchUtils.ts @@ -1,9 +1,11 @@ import Fuse, { type FuseResult } from 'fuse.js'; import { getStaticCommands, type StaticCommandItem } from './commands'; -import { type DynamicContentItem, getAllDynamicItems } from './dynamicSearch'; +import { type DynamicContentItem, getDynamicItems } from './dynamicSearch'; import type { CombinedResult } from './types'; +import type { HydratedIndexItem } from './indexing/types'; -export function prepareDynamicItems(items: DynamicContentItem[]): DynamicContentItem[] { +// This function is likely no longer needed as items are pre-processed by the indexer +/* export function prepareDynamicItems(items: DynamicContentItem[]): DynamicContentItem[] { return items.map(item => { const preparedItem = { ...item }; @@ -12,18 +14,15 @@ export function prepareDynamicItems(items: DynamicContentItem[]): DynamicContent item.text, item.content, item.category, - item.keywords?.join(' ') || '', - ...Object.values(item.metadata || {}) - .filter(value => typeof value === 'string') ].filter(Boolean).join(' '); return preparedItem; }); -} +} */ export function createSearchIndexes() { const commands = getStaticCommands(); - const dynamicItems = prepareDynamicItems(getAllDynamicItems()); + const dynamicItems = getDynamicItems(); // Returns HydratedIndexItem[] const commandOptions = { keys: ['text', 'category', 'keywords'], @@ -35,13 +34,15 @@ export function createSearchIndexes() { useExtendedSearch: false }; + // Keys for dynamic items remain the same structurally const dynamicOptions = { keys: [ - 'text', - 'content', - 'category', - 'keywords', - 'allContent' + 'text', + 'content', + 'category', + 'metadata.author', // Example: Include specific metadata if needed + 'metadata.subject', // Example: Include specific metadata if needed + // 'keywords', // Keywords are not currently part of IndexItem, add if needed ], includeScore: true, includeMatches: true, @@ -53,7 +54,7 @@ export function createSearchIndexes() { return { commandsFuse: new Fuse(commands, commandOptions) as Fuse, - dynamicContentFuse: new Fuse(dynamicItems, dynamicOptions) as Fuse, + dynamicContentFuse: new Fuse(dynamicItems, dynamicOptions) as Fuse, commands, dynamicItems }; @@ -61,7 +62,7 @@ export function createSearchIndexes() { export function searchCommands( commandsFuse: Fuse, - query: string, + query: string, commandIdToItemMap: Map, limit = 10 ): CombinedResult[] { @@ -69,6 +70,8 @@ export function searchCommands( if (!query.trim()) { return Array.from(commandIdToItemMap.values()) + .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)) // Sort by priority when no query + .slice(0, limit) // Limit results even when no query .map(item => ({ id: item.id, type: 'command' as const, @@ -95,21 +98,25 @@ export function searchCommands( } export function searchDynamicItems( - dynamicContentFuse: Fuse, - query: string, - dynamicIdToItemMap: Map, - limit = 10 + dynamicContentFuse: Fuse, + query: string, + dynamicIdToItemMap: Map, + limit = 10, + sortByRecent: boolean = true // Added option to control sorting ): CombinedResult[] { if (!dynamicContentFuse) return []; if (!query.trim()) { - return Array.from(dynamicIdToItemMap.values()) - .sort((a, b) => b.dateAdded - a.dateAdded) + let items = Array.from(dynamicIdToItemMap.values()); + if (sortByRecent) { + items = items.sort((a, b) => b.dateAdded - a.dateAdded); + } + return items .slice(0, limit) .map(item => ({ id: item.id, type: 'dynamic' as const, - score: 80, + score: 80, // Assign a default score for non-searched items item })); } @@ -117,12 +124,12 @@ export function searchDynamicItems( const now = Date.now(); const searchResults = dynamicContentFuse.search(query, { limit }); - return searchResults.map((result: FuseResult) => { + return searchResults.map((result: FuseResult) => { const item = result.item; const fuseScore = 10 * (1 - (result.score || 0.5)); const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); - const recencyBoost = 1 / (ageInDays + 1); - const score = fuseScore + recencyBoost + (item.priority ?? 0); + const recencyBoost = sortByRecent ? (1 / (ageInDays + 1)) : 0; // Apply boost only if sorting by recent + const score = fuseScore + recencyBoost; return { id: item.id, @@ -135,14 +142,15 @@ export function searchDynamicItems( } export function performSearch( - query: string, + query: string, commandsFuse: Fuse, - dynamicContentFuse: Fuse, + dynamicContentFuse: Fuse, commandIdToItemMap: Map, - dynamicIdToItemMap: Map + dynamicIdToItemMap: Map, + showRecentFirst: boolean // Pass sorting preference ): CombinedResult[] { const commandResults = searchCommands(commandsFuse, query, commandIdToItemMap); - const dynamicResults = searchDynamicItems(dynamicContentFuse, query, dynamicIdToItemMap); + const dynamicResults = searchDynamicItems(dynamicContentFuse, query, dynamicIdToItemMap, 10, showRecentFirst); const results = [...commandResults, ...dynamicResults]; results.sort((a, b) => b.score - a.score); diff --git a/src/plugins/built-in/globalSearch/types.ts b/src/plugins/built-in/globalSearch/types.ts index 7cd24c5a..fa71dc04 100644 --- a/src/plugins/built-in/globalSearch/types.ts +++ b/src/plugins/built-in/globalSearch/types.ts @@ -1,5 +1,5 @@ import type { StaticCommandItem } from './commands'; -import type { DynamicContentItem } from './dynamicSearch'; +import type { HydratedIndexItem } from './indexing/types'; export interface MatchIndices { readonly 0: number; @@ -16,7 +16,7 @@ export interface CombinedResult { id: string; type: 'command' | 'dynamic'; score: number; - item: StaticCommandItem | DynamicContentItem; + item: StaticCommandItem | HydratedIndexItem; matches?: readonly FuseResultMatch[]; } From 07aa9524aae735e69be2938f0a1185bf7fcf42e1 Mon Sep 17 00:00:00 2001 From: SethBurkart123 Date: Tue, 1 Apr 2025 23:14:45 +1100 Subject: [PATCH 14/45] feat: early vector search testing --- package.json | 1 + .../built-in/globalSearch/SearchBar.svelte | 11 +- .../globalSearch/client-vector-search-docs.md | 597 ++++++++++++++++++ src/plugins/built-in/globalSearch/commands.ts | 10 +- .../components/AssessmentItem.svelte | 22 +- .../built-in/globalSearch/highlightUtils.ts | 43 ++ .../built-in/globalSearch/indexing/indexer.ts | 16 + .../built-in/globalSearch/indexing/jobs.ts | 8 - .../built-in/globalSearch/searchUtils.ts | 73 ++- .../built-in/globalSearch/vectorSearch.ts | 86 +++ 10 files changed, 833 insertions(+), 34 deletions(-) create mode 100644 src/plugins/built-in/globalSearch/client-vector-search-docs.md create mode 100644 src/plugins/built-in/globalSearch/vectorSearch.ts diff --git a/package.json b/package.json index 8ac05db3..c011f0ba 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "@uiw/codemirror-extensions-color": "^4.23.10", "@uiw/codemirror-theme-github": "^4.23.10", "autoprefixer": "^10.4.21", + "client-vector-search": "^0.2.0", "codemirror": "^6.0.1", "color": "^5.0.0", "dompurify": "^3.2.4", diff --git a/src/plugins/built-in/globalSearch/SearchBar.svelte b/src/plugins/built-in/globalSearch/SearchBar.svelte index c4330e9d..a9838714 100644 --- a/src/plugins/built-in/globalSearch/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/SearchBar.svelte @@ -6,7 +6,7 @@ import { type StaticCommandItem } from './commands'; import type { CombinedResult } from './types'; import { createSearchIndexes, performSearch as doSearch } from './searchUtils'; - import { highlightMatch, highlightSnippet } from './highlightUtils'; + import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from './highlightUtils'; import Fuse from 'fuse.js'; import Calculator from './Calculator.svelte'; import { actionMap } from './indexing/actions'; @@ -105,14 +105,14 @@ }; }); - const performSearch = () => { + const performSearch = async () => { isLoading = true; selectedIndex = 0; const term = searchTerm.trim().toLowerCase(); if (commandsFuse && dynamicContentFuse) { - combinedResults = doSearch( + combinedResults = await doSearch( term, commandsFuse, dynamicContentFuse, @@ -288,8 +288,9 @@ onclick={() => executeItemAction(dynamicItem)} >
      +
      {dynamicItem.metadata?.icon || '\ue924'}
      - {@html highlightMatch(dynamicItem.text, searchTerm, result.matches)} + {@html stripHtmlButKeepHighlights(highlightMatch(dynamicItem.text, searchTerm, result.matches))} {dynamicItem.category} @@ -297,7 +298,7 @@
      {#if dynamicItem.content}
      - {@html highlightSnippet(dynamicItem.content, searchTerm, result.matches)} + {@html stripHtmlButKeepHighlights(highlightSnippet(dynamicItem.content, searchTerm, result.matches))}
      {/if} diff --git a/src/plugins/built-in/globalSearch/client-vector-search-docs.md b/src/plugins/built-in/globalSearch/client-vector-search-docs.md new file mode 100644 index 00000000..97fab159 --- /dev/null +++ b/src/plugins/built-in/globalSearch/client-vector-search-docs.md @@ -0,0 +1,597 @@ +# client-vector-search + +A client side vector search library that can embed, search, and cache. Works on the browser and server side. + +It outperforms OpenAI's text-embedding-ada-002 and is way faster than Pinecone and other VectorDBs. + +I'm the founder of [searchbase.app](https://searchbase.app) and we needed this for our product and customers. We'll be using this library in production. You can be sure it'll be maintained and improved. + +- Embed documents using transformers by default: gte-small (~30mb). +- Calculate cosine similarity between embeddings. +- Create an index and search on the client side +- Cache vectors with browser caching support. + +Lots of improvements are coming! + +## Roadmap + +Our goal is to build a super simple, fast vector search that works with couple hundred to thousands vectors. ~1k vectors per user covers 99% of the use cases. + +We'll initially keep things super simple and sub 100ms + +### TODOs +- [ ] add HNSW index that works on node and browser env, don't rely on hnsw binder libs +- [ ] add a proper testing suite and ci/cd for the lib + - [ ] simple health tests + - [ ] mock the @xenova/transformers for jest, it's not happy with it + - [ ] performance tests, recall, memory usage, cpu usage etc. + + +## Installation + +```bash +npm i client-vector-search +``` + + +## Quickstart + +This library provides a plug-and-play solution for embedding and vector search. It's designed to be easy to use, efficient, and versatile. Here's a quick start guide: + + +```ts + import { getEmbedding, EmbeddingIndex } from 'client-vector-search'; + + // getEmbedding is an async function, so you need to use 'await' or '.then()' to get the result + const embedding = await getEmbedding("Apple"); // Returns embedding as number[] + + // Each object should have an 'embedding' property of type number[] + const initialObjects = [ + { id: 1, name: "Apple", embedding: embedding }, + { id: 2, name: "Banana", embedding: await getEmbedding("Banana") }, + { id: 3, name: "Cheddar", embedding: await getEmbedding("Cheddar")}, + { id: 4, name: "Space", embedding: await getEmbedding("Space")}, + { id: 5, name: "database", embedding: await getEmbedding("database")}, + ]; + const index = new EmbeddingIndex(initialObjects); // Creates an index + + // The query should be an embedding of type number[] + const queryEmbedding = await getEmbedding('Fruit'); // Query embedding + const results = await index.search(queryEmbedding, { topK: 5 }); // Returns top similar objects + + // specify the storage type + await index.saveIndex('indexedDB'); + const results = await index.search([1, 2, 3], { + topK: 5, + useStorage: 'indexedDB', + // storageOptions: { // use only if you overrode the defaults + // indexedDBName: 'clientVectorDB', + // indexedDBObjectStoreName: 'ClientEmbeddingStore', + // }, + }); + + console.log(results); + + await index.deleteIndexedDB(); // if you overrode default, specify db name +``` + +## Trouble-shooting + +### NextJS +To use it inside NextJS projects you'll need to update the `next.config.js` file to include the following: + +```js +module.exports = { + // Override the default webpack configuration + webpack: (config) => { + // See https://webpack.js.org/configuration/resolve/#resolvealias + config.resolve.alias = { + ...config.resolve.alias, + sharp$: false, + "onnxruntime-node$": false, + }; + return config; + }, +}; +``` + +#### Model load after page is loaded + +You can initialize the model before using it to generate embeddings. This will ensure that the model is loaded before you use it and provide a better UX. + +```js +import { initializeModel } from "client-vector-search" +... + useEffect(() => { + try { + initializeModel(); + } catch (e) { + console.log(e); + } + }, []); +``` + +## Usage Guide + +This guide provides a step-by-step walkthrough of the library's main features. It covers everything from generating embeddings for a string to performing operations on the index such as adding, updating, and removing objects. It also includes instructions on how to save the index to a database and perform search operations within it. + +Until we have a reference documentation, you can find all the methods and their usage in this guide. Each step is accompanied by a code snippet to illustrate the usage of the method in question. Make sure to follow along and try out the examples in your own environment to get a better understanding of how everything works. + +Let's get started! + +### Step 1: Generate Embeddings for String +Generate embeddings for a given string using the `getEmbedding` method. + +```ts +const embedding = await getEmbedding("Apple"); // Returns embedding as number[] +``` +> **Note**: `getEmbedding` is asynchronous; make sure to use `await`. + +--- + +### Step 2: Calculate Cosine Similarity +Calculate the cosine similarity between two embeddings. + +```ts +const similarity = cosineSimilarity(embedding1, embedding2, 6); +``` +> **Note**: Both embeddings should be of the same length. + +--- + +### Step 3: Create an Index +Create an index with an initial array of objects. Each object must have an 'embedding' property. + +```ts +const initialObjects = [...]; +const index = new EmbeddingIndex(initialObjects); +``` + +--- + +### Step 4: Add to Index +Add an object to the index. + +```ts +const objectToAdd = { id: 6, name: 'Cat', embedding: await getEmbedding('Cat') }; +index.add(objectToAdd); +``` + +--- + +### Step 5: Update Index +Update an existing object in the index. + +```ts +const vectorToUpdate = { id: 6, name: 'Dog', embedding: await getEmbedding('Dog') }; +index.update({ id: 6 }, vectorToUpdate); +``` + +--- + +### Step 6: Remove from Index +Remove an object from the index. + +```ts +index.remove({ id: 6 }); +``` + +--- + +### Step 7: Retrieve from Index +Retrieve an object from the index. + +```ts +const vector = index.get({ id: 1 }); +``` + +--- + +### Step 8: Search the Index +Search the index with a query embedding. + +```ts +const queryEmbedding = await getEmbedding('Fruit'); +const results = await index.search(queryEmbedding, { topK: 5 }); +``` + +--- + +### Step 9: Print the Index +Print the entire index to the console. + +```ts +index.printIndex(); +``` + +--- + +### Step 10: Save Index to IndexedDB (for browser) +Save the index to a persistent IndexedDB database. Note + +```ts +await index.saveIndex("indexedDB", { DBName: "clientVectorDB", objectStoreName:"ClientEmbeddingStore"}) +``` + +--- + +### Important: Search in indexedDB +Perform a search operation in the IndexedDB. + +```ts +const results = await index.search(queryEmbedding, { + topK: 5, + useStorage: "indexedDB", + storageOptions: { // only if you want to override the default options, defaults are below + indexedDBName: 'clientVectorDB', + indexedDBObjectStoreName: 'ClientEmbeddingStore' + } +}); + +--- + +### Delete Database +To delete an entire database. + +```ts +await IndexedDbManager.deleteIndexedDB("clientVectorDB"); +``` + +--- + +### Delete Object Store +To delete an object store from a database. + +```ts +await IndexedDbManager.deleteIndexedDBObjectStore("clientVectorDB", "ClientEmbeddingStore"); +``` + +--- + +### Retrieve All Objects +To retrieve all objects from a specific object store. + +```ts +const allObjects = await IndexedDbManager.getAllObjectsFromIndexedDB("clientVectorDB", "ClientEmbeddingStore"); +``` + + + + +# THE MAIN INDEX.TS FILE THAT YOU ARE IMPORTING FROM +```index.ts +const DEFAULT_TOP_K = 3; + +interface Filter { + [key: string]: any; +} + +import Cache from './cache'; +import { IndexedDbManager } from './indexedDB'; +import { cosineSimilarity } from './utils'; +export { ExperimentalHNSWIndex } from './hnsw'; + +// uncomment if you want to test indexedDB implementation in node env for faster dev cycle +// import { IDBFactory } from 'fake-indexeddb'; +// const indexedDB = new IDBFactory(); + +export interface SearchResult { + similarity: number; + object: any; +} + +type StorageOptions = 'indexedDB' | 'localStorage' | 'none'; + +/** + * Interface for search options in the EmbeddingIndex class. + * topK: The number of top similar items to return. + * filter: An optional filter to apply to the objects before searching. + * useStorage: A flag to indicate whether to use storage options like indexedDB or localStorage. + */ +interface SearchOptions { + topK?: number; + filter?: Filter; + useStorage?: StorageOptions; + storageOptions?: { indexedDBName: string; indexedDBObjectStoreName: string }; // TODO: generalize it to localStorage as well +} + +const cacheInstance = Cache.getInstance(); + +let pipe: any; +let currentModel: string; + +export const initializeModel = async ( + model: string = 'Xenova/gte-small', +): Promise => { + if (model !== currentModel) { + const transformersModule = await import('@xenova/transformers'); + const pipeline = transformersModule.pipeline; + pipe = await pipeline('feature-extraction', model); + currentModel = model; + } +}; + +export const getEmbedding = async ( + text: string, + precision: number = 7, + options = { pooling: 'mean', normalize: false }, + model = 'Xenova/gte-small', +): Promise => { + const cachedEmbedding = cacheInstance.get(text); + if (cachedEmbedding) { + return Promise.resolve(cachedEmbedding); + } + + if (model !== currentModel) { + await initializeModel(model); + } + + const output = await pipe(text, options); + const roundedOutput = Array.from(output.data as number[]).map( + (value: number) => parseFloat(value.toFixed(precision)), + ); + cacheInstance.set(text, roundedOutput); + return Array.from(roundedOutput); +}; + +export class EmbeddingIndex { + private objects: Filter[]; + private keys: string[]; + + constructor(initialObjects?: Filter[]) { + // TODO: add support for options while creating index such as {... indexedDB: true, ...} + this.objects = []; + this.keys = []; + if (initialObjects && initialObjects.length > 0) { + initialObjects.forEach((obj) => this.validateAndAdd(obj)); + if (initialObjects[0]) { + this.keys = Object.keys(initialObjects[0]); + } + } + } + + private findVectorIndex(filter: Filter): number { + return this.objects.findIndex((object) => + Object.keys(filter).every((key) => object[key] === filter[key]), + ); + } + + private validateAndAdd(obj: Filter) { + if (!Array.isArray(obj.embedding) || obj.embedding.some(isNaN)) { + throw new Error( + 'Object must have an embedding property of type number[]', + ); + } + if (this.keys.length === 0) { + this.keys = Object.keys(obj); + } else if (!this.keys.every((key) => key in obj)) { + throw new Error( + 'Object must have the same properties as the initial objects', + ); + } + this.objects.push(obj); + } + + add(obj: Filter) { + this.validateAndAdd(obj); + } + + // Method to update an existing vector in the index + update(filter: Filter, vector: Filter) { + const index = this.findVectorIndex(filter); + if (index === -1) { + throw new Error('Vector not found'); + } + if (vector.hasOwnProperty('embedding')) { + // Validate and add the new vector + this.validateAndAdd(vector); + } + // Replace the old vector with the new one + this.objects[index] = Object.assign(this.objects[index] as Filter, vector); + } + + // Method to remove a vector from the index + remove(filter: Filter) { + const index = this.findVectorIndex(filter); + if (index === -1) { + throw new Error('Vector not found'); + } + // Remove the vector from the index + this.objects.splice(index, 1); + } + + // Method to remove multiple vectors from the index + removeBatch(filters: Filter[]) { + filters.forEach((filter) => { + const index = this.findVectorIndex(filter); + if (index !== -1) { + // Remove the vector from the index + this.objects.splice(index, 1); + } + }); + } + + // Method to retrieve a vector from the index + get(filter: Filter) { + const vector = this.objects[this.findVectorIndex(filter)]; + return vector || null; + } + + size(): number { + // Returns the size of the index + return this.objects.length; + } + + clear() { + this.objects = []; + } + + async search( + queryEmbedding: number[], + options: SearchOptions = { + topK: 3, + useStorage: 'none', + storageOptions: { + indexedDBName: 'clientVectorDB', + indexedDBObjectStoreName: 'ClientEmbeddingStore', + }, + }, + ): Promise { + const topK = options.topK || DEFAULT_TOP_K; + const filter = options.filter || {}; + const useStorage = options.useStorage || 'none'; + + if (useStorage === 'indexedDB') { + const DBname = options.storageOptions?.indexedDBName || 'clientVectorDB'; + const objectStoreName = + options.storageOptions?.indexedDBObjectStoreName || + 'ClientEmbeddingStore'; + + if (typeof indexedDB === 'undefined') { + console.error('IndexedDB is not supported'); + throw new Error('IndexedDB is not supported'); + } + const results = await this.loadAndSearchFromIndexedDB( + DBname, + objectStoreName, + queryEmbedding, + topK, + filter, + ); + return results; + } else { + // Compute similarities + const similarities = this.objects + .filter((object) => + Object.keys(filter).every((key) => object[key] === filter[key]), + ) + .map((obj) => ({ + similarity: cosineSimilarity(queryEmbedding, obj.embedding), + object: obj, + })); + + // Sort by similarity and return topK results + return similarities + .sort((a, b) => b.similarity - a.similarity) + .slice(0, topK); + } + } + + printIndex() { + console.log('Index Content:'); + this.objects.forEach((obj, idx) => { + console.log(`Item ${idx + 1}:`, obj); + }); + } + + async saveIndex( + storageType: string, + options: { DBName: string; objectStoreName: string } = { + DBName: 'clientVectorDB', + objectStoreName: 'ClientEmbeddingStore', + }, + ) { + if (storageType === 'indexedDB') { + await this.saveToIndexedDB(options.DBName, options.objectStoreName); + } else { + throw new Error( + `Unsupported storage type: ${storageType} \n Supported storage types: "indexedDB"`, + ); + } + } + + async saveToIndexedDB( + DBname: string = 'clientVectorDB', + objectStoreName: string = 'ClientEmbeddingStore', + ): Promise { + if (typeof indexedDB === 'undefined') { + console.error('IndexedDB is not defined'); + throw new Error('IndexedDB is not supported'); + } + + if (!this.objects || this.objects.length === 0) { + throw new Error('Index is empty. Nothing to save'); + } + + try { + const db = await IndexedDbManager.create(DBname, objectStoreName); + await db.addToIndexedDB(this.objects); + console.log( + `Index saved to database '${DBname}' object store '${objectStoreName}'`, + ); + } catch (error) { + console.error('Error saving index to database:', error); + throw new Error('Error saving index to database'); + } + } + + async loadAndSearchFromIndexedDB( + DBname: string = 'clientVectorDB', + objectStoreName: string = 'ClientEmbeddingStore', + queryEmbedding: number[], + topK: number, + filter: { [key: string]: any }, + ): Promise { + const db = await IndexedDbManager.create(DBname, objectStoreName); + const generator = db.dbGenerator(); + const results: { similarity: number; object: any }[] = []; + + for await (const record of generator) { + if (Object.keys(filter).every((key) => record[key] === filter[key])) { + const similarity = cosineSimilarity(queryEmbedding, record.embedding); + results.push({ similarity, object: record }); + } + } + results.sort((a, b) => b.similarity - a.similarity); + return results.slice(0, topK); + } + + async deleteIndexedDB(DBname: string = 'clientVectorDB'): Promise { + if (typeof indexedDB === 'undefined') { + console.error('IndexedDB is not defined'); + throw new Error('IndexedDB is not supported'); + } + return new Promise((resolve, reject) => { + const request = indexedDB.deleteDatabase(DBname); + + request.onsuccess = () => { + console.log(`Database '${DBname}' deleted`); + resolve(); + }; + request.onerror = (event) => { + console.error('Failed to delete database', event); + reject(new Error('Failed to delete database')); + }; + }); + } + + async deleteIndexedDBObjectStore( + DBname: string = 'clientVectorDB', + objectStoreName: string = 'ClientEmbeddingStore', + ): Promise { + const db = await IndexedDbManager.create(DBname, objectStoreName); + + try { + await db.deleteIndexedDBObjectStoreFromDB(DBname, objectStoreName); + console.log( + `Object store '${objectStoreName}' deleted from database '${DBname}'`, + ); + } catch (error) { + console.error('Error deleting object store:', error); + throw new Error('Error deleting object store'); + } + } + + async getAllObjectsFromIndexedDB( + DBname: string = 'clientVectorDB', + objectStoreName: string = 'ClientEmbeddingStore', + ): Promise { + const db = await IndexedDbManager.create(DBname, objectStoreName); + const objects: any[] = []; + for await (const record of db.dbGenerator()) { + objects.push(record); + } + return objects; + } +} +``` \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/commands.ts b/src/plugins/built-in/globalSearch/commands.ts index 618dc6a9..f5b6ae75 100644 --- a/src/plugins/built-in/globalSearch/commands.ts +++ b/src/plugins/built-in/globalSearch/commands.ts @@ -28,7 +28,7 @@ const staticCommands: StaticCommandItem[] = [ window.location.hash = '?page=/home'; loadHomePage(); }, - priority: 10 + priority: 4 }, { id: 'messages', @@ -40,7 +40,7 @@ const staticCommands: StaticCommandItem[] = [ action: () => { window.location.hash = '?page=/messages'; }, - priority: 10 + priority: 4 }, { id: 'timetable', @@ -52,7 +52,7 @@ const staticCommands: StaticCommandItem[] = [ action: () => { window.location.hash = '?page=/timetable'; }, - priority: 10 + priority: 4 }, { id: 'assessments', @@ -64,7 +64,7 @@ const staticCommands: StaticCommandItem[] = [ action: () => { window.location.hash = '?page=/assessments'; }, - priority: 10 + priority: 4 }, { id: 'toggle-dark-mode', @@ -72,7 +72,7 @@ const staticCommands: StaticCommandItem[] = [ category: 'action', text: 'Toggle Dark Mode', action: () => settingsState.DarkMode = !settingsState.DarkMode, - priority: 5, + priority: 2, keywords: ['theme', 'appearance'] } ]; diff --git a/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte b/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte index 15643118..1d5dde71 100644 --- a/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte +++ b/src/plugins/built-in/globalSearch/components/AssessmentItem.svelte @@ -1,12 +1,13 @@