diff --git a/.prettierrc b/.prettierrc index 7c28b83a..a5cd8192 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,5 +1,5 @@ { "tabWidth": 2, "useTabs": false, - "semi": false + "semi": true } diff --git a/lib/inlineWorker.ts b/lib/inlineWorker.ts new file mode 100644 index 00000000..e493125d --- /dev/null +++ b/lib/inlineWorker.ts @@ -0,0 +1,37 @@ +// vite-plugin-inline-worker-dev.ts +import { Plugin } from 'vite' +import fs from 'fs/promises' +import { build, transform } from 'esbuild' + +export default function InlineWorkerDevPlugin(): Plugin { + return { + name: 'vite:inline-worker-dev', + async load(id) { + if (id.includes('?inlineWorker')) { + const [cleanPath] = id.split('?') + console.log('cleanPath', cleanPath) + const code = await fs.readFile(cleanPath, 'utf-8') + const result = await build({ + entryPoints: [cleanPath], + bundle: true, + write: false, + platform: 'browser', + format: 'iife', + target: 'esnext', + }) + + const workerCode = result.outputFiles[0].text + + const workerBlobCode = ` + const code = ${JSON.stringify(workerCode)}; + export default function InlineWorker() { + const blob = new Blob([code], { type: 'application/javascript' }); + return new Worker(URL.createObjectURL(blob), { type: 'module' }); + } + ` + return workerBlobCode + } + return null + } + } +} diff --git a/package.json b/package.json index 9d9f39a5..2502d2d3 100644 --- a/package.json +++ b/package.json @@ -75,16 +75,19 @@ "@uiw/codemirror-extensions-color": "^4.23.10", "@uiw/codemirror-theme-github": "^4.23.10", "autoprefixer": "^10.4.21", + "client-vector-search": "../client-vector-search", "codemirror": "^6.0.1", "color": "^5.0.0", "dompurify": "^3.2.4", "embla-carousel-autoplay": "^8.5.2", "embla-carousel-svelte": "^8.5.2", "events": "^3.3.0", + "flexsearch": "^0.8.147", "fuse.js": "^7.1.0", "idb": "^8.0.2", "localforage": "^1.10.0", "lodash": "^4.17.21", + "mathjs": "^14.4.0", "million": "^3.1.11", "motion": "^12.4.12", "postcss": "^8.5.3", diff --git a/src/declarations.d.ts b/src/declarations.d.ts index f89205fc..deb52700 100644 --- a/src/declarations.d.ts +++ b/src/declarations.d.ts @@ -5,6 +5,11 @@ declare module '*.png'; declare module '*.html'; declare module '*.svelte'; +declare module '*?inlineWorker' { + const value: () => Worker; + export default value; +} + declare module "*.png?base64" { const value: string; export default value; diff --git a/src/interface/components/PickerSwatch.svelte b/src/interface/components/PickerSwatch.svelte index 643f5f46..631d954d 100644 --- a/src/interface/components/PickerSwatch.svelte +++ b/src/interface/components/PickerSwatch.svelte @@ -5,6 +5,7 @@ + + \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/components/Calculator.svelte b/src/plugins/built-in/globalSearch/src/components/Calculator.svelte new file mode 100644 index 00000000..4092df71 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/components/Calculator.svelte @@ -0,0 +1,135 @@ + + +{#if result !== null} +
+

Calculator

+
+
+
+ {searchTerm} +
+
+ {inputUnit || 'Question'} +
+
+ +
+
+
+ → +
+
+
+ + {#if !isCalculating} +
+
+ {result} +
+
+ {outputUnit || 'Result'} +
+
+ {:else} +
+ {/if} +
+
+{/if} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte new file mode 100644 index 00000000..c19185c0 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte @@ -0,0 +1,390 @@ + + +{#if commandPalleteOpen} + +{/if} + +{#snippet Shortcut({ text, keybind }: { text: string, keybind: string[] }) } +
+
+ {#each keybind as key} + {key} + {/each} +
+ {text} +
+{/snippet} + + \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/components/unitMap.ts b/src/plugins/built-in/globalSearch/src/components/unitMap.ts new file mode 100644 index 00000000..35e50b31 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/components/unitMap.ts @@ -0,0 +1,193 @@ +export const unitFullNames: Record = { + // --- Length --- + m: "Meters", + km: "Kilometers", + cm: "Centimeters", + mm: "Millimeters", + µm: "Micrometers", + nm: "Nanometers", + pm: "Picometers", + fm: "Femtometers", + am: "Attometers", + zm: "Zeptometers", + ym: "Yoctometers", + mi: "Miles", + yd: "Yards", + ft: "Feet", + in: "Inches", + nmi: "Nautical Miles", + angstrom: "Angstroms", + au: "Astronomical Units", + ly: "Light Years", + pc: "Parsecs", + + // --- Mass --- + kg: "Kilograms", + g: "Grams", + mg: "Milligrams", + µg: "Micrograms", + ng: "Nanograms", + lb: "Pounds", + oz: "Ounces", + ton: "Tons (Imperial)", + tonne: "Tonnes (Metric)", + slug: "Slugs", + stone: "Stones", + + // --- Time --- + s: "Seconds", + ms: "Milliseconds", + µs: "Microseconds", + ns: "Nanoseconds", + ps: "Picoseconds", + min: "Minutes", + h: "Hours", + day: "Days", + week: "Weeks", + month: "Months (30 days)", + year: "Years (365 days)", + fortnight: "Fortnights", + + // --- Temperature --- + K: "Kelvin", + degC: "Degrees Celsius", + degF: "Degrees Fahrenheit", + degR: "Degrees Rankine", + + // --- Volume --- + "m³": "Cubic Meters", + "cm³": "Cubic Centimeters", + "mm³": "Cubic Millimeters", + l: "Liters", + ml: "Milliliters", + gal: "Gallons (US)", + qt: "Quarts (US)", + pt: "Pints (US)", + cup: "Cups (US)", + floz: "Fluid Ounces (US)", + tbsp: "Tablespoons (US)", + tsp: "Teaspoons (US)", + + // --- Area --- + "m²": "Square Meters", + "km²": "Square Kilometers", + "cm²": "Square Centimeters", + "mm²": "Square Millimeters", + ha: "Hectares", + acre: "Acres", + "ft²": "Square Feet", + "in²": "Square Inches", + "mi²": "Square Miles", + + // --- Speed --- + "m/s": "Meters per Second", + "km/h": "Kilometers per Hour", + mph: "Miles per Hour", + knot: "Knots", + + // --- Acceleration --- + "m/s²": "Meters per Second Squared", + + // --- Force --- + N: "Newtons", + lbf: "Pound-Force", + dyn: "Dynes", + + // --- Energy --- + J: "Joules", + kJ: "Kilojoules", + cal: "Calories", + kcal: "Kilocalories", + Wh: "Watt Hours", + kWh: "Kilowatt Hours", + BTU: "British Thermal Units", + erg: "Ergs", + eV: "Electronvolts", + + // --- Power --- + W: "Watts", + kW: "Kilowatts", + MW: "Megawatts", + GW: "Gigawatts", + hp: "Horsepower", + + // --- Pressure --- + Pa: "Pascals", + kPa: "Kilopascals", + bar: "Bar", + atm: "Atmospheres", + psi: "Pounds per Square Inch", + torr: "Torr", + mmHg: "Millimeters of Mercury", + + // --- Frequency --- + Hz: "Hertz", + kHz: "Kilohertz", + MHz: "Megahertz", + GHz: "Gigahertz", + THz: "Terahertz", + + // --- Electric --- + V: "Volts", + A: "Amperes", + C: "Coulombs", + Ω: "Ohms", + F: "Farads", + S: "Siemens", + H: "Henries", + Wb: "Webers", + T: "Teslas", + lx: "Lux", + + // --- Angle & Rotation --- + rad: "Radians", + deg: "Degrees", + grad: "Gradians", + cycle: "Cycles", + turn: "Turns", + rev: "Revolutions", + + // --- Charge & Capacitance --- + e: "Elementary Charges", + + // --- Magnetic & Light --- + lm: "Lumens", + ph: "Photons", + + // --- Miscellaneous / Dimensionless --- + "%": "Percent", + ppm: "Parts per Million", + ppb: "Parts per Billion", + pptr: "Parts per Trillion", + dB: "Decibels", + bit: "Bits", + byte: "Bytes", + + // --- Digital Storage --- + b: "Bits", + B: "Bytes", + kb: "Kilobits", + kB: "Kilobytes", + Mb: "Megabits", + MB: "Megabytes", + Gb: "Gigabits", + GB: "Gigabytes", + Tb: "Terabits", + TB: "Terabytes", + + // --- Currency --- + USD: "US Dollars", + EUR: "Euros", + GBP: "British Pounds", + AUD: "Australian Dollars", + CAD: "Canadian Dollars", + CHF: "Swiss Francs", + JPY: "Japanese Yen", + CNY: "Chinese Yuan", + INR: "Indian Rupees", + NZD: "New Zealand Dollars", + SEK: "Swedish Krona", + NOK: "Norwegian Krone", + SGD: "Singapore Dollars", + HKD: "Hong Kong Dollars", +}; diff --git a/src/plugins/built-in/globalSearch/src/core/commands.ts b/src/plugins/built-in/globalSearch/src/core/commands.ts new file mode 100644 index 00000000..ca7d39c1 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/core/commands.ts @@ -0,0 +1,85 @@ +import { settingsState } from "@/seqta/utils/listeners/SettingsState"; +import { loadHomePage } from "@/seqta/utils/Loaders/LoadHomePage"; + +export interface BaseCommandItem { + id: string; + text: string; + category: string; + icon: string; + action: () => void; + keywords?: string[]; + priority?: number; +} + +export interface StaticCommandItem extends BaseCommandItem { + keybind?: string[]; + keybindLabel?: string[]; +} + +const staticCommands: StaticCommandItem[] = [ + { + id: "home", + icon: "\ueb4c", + category: "navigation", + text: "Home", + keybind: ["alt+h"], + keybindLabel: ["Alt", "H"], + action: () => { + window.location.hash = "?page=/home"; + loadHomePage(); + }, + priority: 4, + }, + { + id: "messages", + icon: "\uebfd", + category: "navigation", + text: "Direct Messages", + keybind: ["alt+m"], + keybindLabel: ["Alt", "M"], + action: () => { + window.location.hash = "?page=/messages"; + }, + priority: 4, + }, + { + id: "timetable", + icon: "\ue9cd", + category: "navigation", + text: "Timetable", + keybind: ["alt+t"], + keybindLabel: ["Alt", "T"], + action: () => { + window.location.hash = "?page=/timetable"; + }, + priority: 4, + }, + { + id: "assessments", + icon: "\ueac3", + category: "navigation", + text: "Assessments", + keybind: ["alt+a"], + keybindLabel: ["Alt", "A"], + action: () => { + window.location.hash = "?page=/assessments"; + }, + priority: 4, + }, + { + id: "toggle-dark-mode", + icon: "\uecfe", + category: "action", + text: "Toggle Dark Mode", + action: () => (settingsState.DarkMode = !settingsState.DarkMode), + priority: 2, + keywords: ["theme", "appearance"], + }, +]; + +/** + * Returns the predefined list of static commands. + */ +export const getStaticCommands = (): StaticCommandItem[] => { + return [...staticCommands]; +}; diff --git a/src/plugins/built-in/globalSearch/src/core/index.ts b/src/plugins/built-in/globalSearch/src/core/index.ts new file mode 100644 index 00000000..6fa61712 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/core/index.ts @@ -0,0 +1,89 @@ +import type { Plugin } from "@/plugins/core/types"; +import { BasePlugin } from "@/plugins/core/settings"; +import { + booleanSetting, + defineSettings, + Setting, + stringSetting, +} from "@/plugins/core/settingsHelpers"; +import styles from "./styles.css?inline"; +import { waitForElm } from "@/seqta/utils/waitForElm"; +import { runIndexing } from "../indexing/indexer"; +import { initVectorSearch } from "../search/vector/vectorSearch"; +import { cleanupSearchBar, mountSearchBar } from "./mountSearchBar"; + +const settings = defineSettings({ + searchHotkey: stringSetting({ + default: "ctrl+k", + title: "Search Hotkey", + description: "Keyboard shortcut to open the search (cmd on Mac)", + }), + showRecentFirst: booleanSetting({ + default: true, + title: "Show Recent First", + description: "Sort dynamic content by most recent first", + }), + transparencyEffects: booleanSetting({ + default: true, + title: "Transparency Effects", + description: "Enable transparency effects for the search bar", + }), + runIndexingOnLoad: booleanSetting({ + default: true, + title: "Index on Page Load", + description: "Run content indexing when SEQTA loads", + }), +}); + +class GlobalSearchPlugin extends BasePlugin { + @Setting(settings.searchHotkey) + searchHotkey!: string; + + @Setting(settings.showRecentFirst) + showRecentFirst!: boolean; + + @Setting(settings.transparencyEffects) + transparencyEffects!: boolean; + + @Setting(settings.runIndexingOnLoad) + runIndexingOnLoad!: boolean; +} + +const settingsInstance = new GlobalSearchPlugin(); + +const globalSearchPlugin: Plugin = { + id: "global-search", + name: "Global Search", + description: "Quick search for everything in SEQTA", + version: "1.0.0", + settings: settingsInstance.settings, + disableToggle: true, + styles: styles, + + run: async (api) => { + const appRef = { current: null }; + + initVectorSearch(); + + if (api.settings.runIndexingOnLoad) { + setTimeout(async () => { + await runIndexing(); + }, 2000); + } + + const title = document.querySelector("#title"); + + if (title) { + mountSearchBar(title, api, appRef); + } else { + await waitForElm("#title", true, 100, 60); + mountSearchBar(document.querySelector("#title") as Element, api, appRef); + } + + return () => { + cleanupSearchBar(appRef); + }; + }, +}; + +export default globalSearchPlugin; diff --git a/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts b/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts new file mode 100644 index 00000000..88e8bbca --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts @@ -0,0 +1,56 @@ +import renderSvelte from "@/interface/main"; +import SearchBar from "../components/SearchBar.svelte"; +import { unmount } from "svelte"; +import { VectorWorkerManager } from "../indexing/worker/vectorWorkerManager"; + +export function mountSearchBar( + titleElement: Element, + api: any, + appRef: { current: any } +) { + if (titleElement.querySelector(".search-trigger")) { + return; + } + + const searchButton = document.createElement("div"); + searchButton.className = "search-trigger"; + searchButton.innerHTML = /* html */ ` + + + + +

Quick search...

+ ⌘K + `; + + titleElement.appendChild(searchButton); + + const searchRoot = document.createElement("div"); + document.body.appendChild(searchRoot); + const searchRootShadow = searchRoot.attachShadow({ mode: "open" }); + + searchButton.addEventListener("click", () => { + // @ts-ignore - Intentionally adding to window + window.setCommandPalleteOpen(true); + }); + + try { + appRef.current = renderSvelte(SearchBar, searchRootShadow, { + transparencyEffects: api.settings.transparencyEffects ? true : false, + showRecentFirst: api.settings.showRecentFirst, + }); + } catch (error) { + console.error("Error rendering Svelte component:", error); + } +} + +export function cleanupSearchBar(appRef: { current: any }) { + const searchButton = document.querySelector(".search-trigger"); + const searchRoot = document.querySelector(".global-search-root"); + if (searchButton) searchButton.remove(); + if (searchRoot) searchRoot.remove(); + + // Clean up workers + VectorWorkerManager.getInstance().terminate(); + unmount(appRef.current); +} diff --git a/src/plugins/built-in/globalSearch/src/core/styles.css b/src/plugins/built-in/globalSearch/src/core/styles.css new file mode 100644 index 00000000..89f902f9 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/core/styles.css @@ -0,0 +1,58 @@ +.search-trigger { + display: flex; + align-items: center; + justify-content: center; + height: 32px; + margin-left: 10px; + border-radius: 8px; + cursor: pointer; + transition: all 0.2s ease; + margin-right: auto !important; + padding: 3px 12px; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.2); + backdrop-filter: blur(4px); + user-select: none; + + svg { + opacity: 0.8; + } + + p { + font-size: 14px; + margin-left: 8px; + margin-right: 48px; + height: 100%; + margin-bottom: 0; + line-height: 32px; + font-weight: 400; + } +} + +/* Light mode styles */ +.search-trigger { + background-color: rgba(248, 250, 252, 0.05) !important; + border: 1px solid rgba(0, 0, 0, 0.1) !important; + color: #555 !important; + + p { + color: #555 !important; + } + + svg { + color: #555; + } +} + +.dark .search-trigger { + background-color: rgba(0, 0, 0, 0.03) !important; + border: 1px solid rgba(255, 255, 255, 0.1) !important; + color: #aaa !important; + + p { + color: #aaa !important; + } + + svg { + color: #aaa; + } +} diff --git a/src/plugins/built-in/globalSearch/src/core/types.ts b/src/plugins/built-in/globalSearch/src/core/types.ts new file mode 100644 index 00000000..ce529485 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/core/types.ts @@ -0,0 +1,28 @@ +import type { StaticCommandItem } from "./commands"; +import type { HydratedIndexItem } from "../indexing/types"; + +export interface MatchIndices { + readonly 0: number; + readonly 1: number; +} + +export interface FuseResultMatch { + key?: string; + value?: string; + indices: readonly MatchIndices[]; +} + +export interface CombinedResult { + id: string; + type: "command" | "dynamic"; + score: number; + item: StaticCommandItem | HydratedIndexItem; + matches?: readonly FuseResultMatch[]; +} + +export interface FuseResult { + item: T; + refIndex: number; + score?: number; + matches?: readonly FuseResultMatch[]; +} diff --git a/src/plugins/built-in/globalSearch/src/indexing/actions.ts b/src/plugins/built-in/globalSearch/src/indexing/actions.ts new file mode 100644 index 00000000..074264a0 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/actions.ts @@ -0,0 +1,40 @@ +import type { IndexItem } from "./types"; + +interface MessageMetadata { + messageId: number; + author: string; + senderId: number; + senderType: string; + timestamp: string; + hasAttachments: boolean; + attachmentCount: number; + read: boolean; +} + +interface AssessmentMetadata { + assessmentId?: number; + messageId?: number; + subject?: string; + term?: string; + programmeId?: number; + metaclassId?: number; + timestamp: string; + isMessageBased?: boolean; + author?: string; +} + +type ActionHandler = (item: IndexItem & { metadata: T }) => void; + +export const actionMap: Record> = { + message: ((item: IndexItem & { metadata: MessageMetadata }) => { + window.location.hash = `#?page=/messages&id=${item.metadata.messageId}`; + }) as ActionHandler, + + assessment: ((item: IndexItem & { metadata: AssessmentMetadata }) => { + if (item.metadata.isMessageBased) { + window.location.hash = `#?page=/messages&id=${item.metadata.messageId}`; + } else { + window.location.hash = `#?page=/assessments&id=${item.metadata.assessmentId}`; + } + }) as ActionHandler, +}; diff --git a/src/plugins/built-in/globalSearch/src/indexing/db.ts b/src/plugins/built-in/globalSearch/src/indexing/db.ts new file mode 100644 index 00000000..6e4a0cc6 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/db.ts @@ -0,0 +1,202 @@ +const DB_NAME = "betterseqta-index"; +const META_STORE = "meta"; +const VERSION_KEY = "betterseqta-index-version"; + +let dbPromise: Promise | null = null; + +// Get the current version from localStorage or start at 1 +function getCurrentVersion(): number { + const storedVersion = localStorage.getItem(VERSION_KEY); + return storedVersion ? parseInt(storedVersion, 10) : 1; +} + +// Update the version in localStorage +function updateVersion(version: number) { + localStorage.setItem(VERSION_KEY, version.toString()); +} + +function openDB(): Promise { + if (dbPromise) return dbPromise; + + const currentVersion = getCurrentVersion(); + + dbPromise = new Promise((resolve, reject) => { + let request: IDBOpenDBRequest; + + try { + request = indexedDB.open(DB_NAME, currentVersion); + } catch (e) { + // If there's a version error, try to delete the database and start fresh + console.warn("Database version conflict, recreating database..."); + indexedDB.deleteDatabase(DB_NAME); + localStorage.removeItem(VERSION_KEY); + request = indexedDB.open(DB_NAME, 1); + updateVersion(1); + } + + request.onupgradeneeded = (event) => { + const db = request.result; + const existingStores = Array.from(db.objectStoreNames); + + // Always ensure META_STORE exists + if (!existingStores.includes(META_STORE)) { + db.createObjectStore(META_STORE); + } + + // Update version in localStorage to match the database + updateVersion(event.newVersion || 1); + }; + + request.onsuccess = () => resolve(request.result); + + request.onerror = () => { + console.error("Error opening database:", request.error); + // If there's an error, try to recover by deleting and recreating + indexedDB.deleteDatabase(DB_NAME); + localStorage.removeItem(VERSION_KEY); + reject(request.error); + }; + }); + + return dbPromise; +} + +async function getStore(store: string, mode: IDBTransactionMode = "readonly") { + const db = await openDB(); + + // Create store dynamically if needed + if (!db.objectStoreNames.contains(store)) { + db.close(); + await upgradeDB(store); + return getStore(store, mode); + } + + const tx = db.transaction(store, mode); + return tx.objectStore(store); +} + +function upgradeDB(newStore: string): Promise { + return new Promise((resolve, reject) => { + const currentVersion = getCurrentVersion(); + const newVersion = currentVersion + 1; + + // Close any existing connections + if (dbPromise) { + dbPromise.then((db) => db.close()); + dbPromise = null; + } + + const request = indexedDB.open(DB_NAME, newVersion); + + request.onupgradeneeded = (event) => { + const db = request.result; + if (!db.objectStoreNames.contains(newStore)) { + db.createObjectStore(newStore); + } + // Update version in localStorage + updateVersion(event.newVersion || newVersion); + }; + + request.onsuccess = () => { + dbPromise = Promise.resolve(request.result); + resolve(); + }; + + request.onerror = () => { + console.error("Error upgrading database:", request.error); + reject(request.error); + }; + }); +} + +export async function getAll(store: string): Promise { + try { + const s = await getStore(store); + return new Promise((resolve, reject) => { + const req = s.getAll(); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in getAll for store ${store}:`, error); + return []; + } +} + +export async function get(store: string, key: string): Promise { + try { + const s = await getStore(store); + return new Promise((resolve, reject) => { + const req = s.get(key); + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in get for store ${store}, key ${key}:`, error); + return null; + } +} + +export async function put( + store: string, + value: any, + key?: string, +): Promise { + try { + const s = await getStore(store, "readwrite"); + return new Promise((resolve, reject) => { + const req = key ? s.put(value, key) : s.put(value); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in put for store ${store}:`, error); + throw error; + } +} + +export async function remove(store: string, key: string): Promise { + try { + const s = await getStore(store, "readwrite"); + return new Promise((resolve, reject) => { + const req = s.delete(key); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in remove for store ${store}, key ${key}:`, error); + throw error; + } +} + +export async function clear(store: string): Promise { + try { + const s = await getStore(store, "readwrite"); + return new Promise((resolve, reject) => { + const req = s.clear(); + req.onsuccess = () => resolve(); + req.onerror = () => reject(req.error); + }); + } catch (error) { + console.error(`Error in clear for store ${store}:`, error); + throw error; + } +} + +// Helper function to reset the database if needed +export async function resetDatabase(): Promise { + if (dbPromise) { + const db = await dbPromise; + db.close(); + dbPromise = null; + } + + return new Promise((resolve, reject) => { + const req = indexedDB.deleteDatabase(DB_NAME); + req.onsuccess = () => { + localStorage.removeItem(VERSION_KEY); + resolve(); + }; + req.onerror = () => reject(req.error); + }); +} diff --git a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts new file mode 100644 index 00000000..ff7d917f --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts @@ -0,0 +1,284 @@ +import { clear, getAll, put, remove } from "./db"; +import { jobs } from "./jobs"; +import { renderComponentMap } from "./renderComponents"; +import type { HydratedIndexItem, IndexItem, Job, JobContext } from "./types"; +import { VectorWorkerManager } from "./worker/vectorWorkerManager"; + +const META_STORE = "meta"; +const LOCK_KEY = "bsq-indexer-lock"; +const HEARTBEAT_INTERVAL = 10000; +const LOCK_TIMEOUT = 20000; + +let heartbeatTimer: ReturnType | null = null; + +function shouldRun(job: Job, lastRun?: number): boolean { + const now = Date.now(); + + if (job.frequency === "pageLoad") return true; + if (!lastRun) return true; + + if (job.frequency.type === "interval") { + return now - lastRun >= job.frequency.ms; + } + + if (job.frequency.type === "expiry") { + return now - lastRun >= job.frequency.afterMs; + } + + return false; +} + +function getLastRunMeta(jobId: string): Promise { + return getAll(META_STORE).then((metaItems) => { + const match = metaItems.find((m: any) => m.jobId === jobId); + return match?.lastRun; + }); +} + +async function updateLastRunMeta(jobId: string): Promise { + await put(META_STORE, { jobId, lastRun: Date.now() }, jobId); +} + +function shouldIndex(): boolean { + const last = parseInt(localStorage.getItem(LOCK_KEY) || "0", 10); + return isNaN(last) || Date.now() - last > LOCK_TIMEOUT; +} + +function startHeartbeat() { + localStorage.setItem(LOCK_KEY, `${Date.now()}`); + heartbeatTimer = setInterval(() => { + localStorage.setItem(LOCK_KEY, `${Date.now()}`); + }, HEARTBEAT_INTERVAL); +} + +function stopHeartbeat() { + if (heartbeatTimer) clearInterval(heartbeatTimer); + localStorage.removeItem(LOCK_KEY); +} + +function dispatchProgress(completed: number, total: number, indexing: boolean, status?: string, detail?: string) { + const event = new CustomEvent("indexing-progress", { + detail: { completed, total, indexing, status, detail }, + }); + window.dispatchEvent(event); +} + +export async function loadAllStoredItems(): Promise { + const all: HydratedIndexItem[] = []; + const jobIds = Object.keys(jobs); + + for (const jobId of jobIds) { + try { + const items = await getAll(jobId) as IndexItem[]; + const job = jobs[jobId]; + const renderComponent = renderComponentMap[job.renderComponentId]; + + if (!renderComponent) { + console.warn(`Render component not found for job ${jobId} (ID: ${job.renderComponentId})`); + } + + for (const item of items) { + // Ensure item has all required fields before pushing + if (item && item.id && item.text && item.category && item.actionId && job.renderComponentId) { + all.push({ + ...item, + renderComponent: renderComponent || undefined, // Assign undefined if not found + }); + } else { + console.warn(`Skipping invalid item from job ${jobId}:`, item); + } + } + } catch (error) { + console.error(`Error loading items for job ${jobId}:`, error); + } + } + console.debug(`[Indexer] Loaded ${all.length} items from non-vector storage.`); + return all; +} + +export async function runIndexing(): Promise { + if (!shouldIndex()) { + console.debug( + "%c[Indexer] Skipping indexing (another tab has the lock)", + "color: gray", + ); + return; + } + + startHeartbeat(); + console.debug("%c[Indexer] Starting indexing...", "color: green"); + + const jobIds = Object.keys(jobs); + let completedJobs = 0; + // Add an extra step for vectorization + const totalSteps = jobIds.length + 1; + dispatchProgress(completedJobs, totalSteps, true, "Starting jobs"); + + const allItemsFromJobs: HydratedIndexItem[] = []; + + // --- Step 1: Run Fetching/Storing Jobs (Main Thread) --- + for (const jobId of jobIds) { + dispatchProgress(completedJobs, totalSteps, true, `Running job: ${jobs[jobId].label}`); + const job = jobs[jobId]; + const lastRun = await getLastRunMeta(jobId); + + if (!shouldRun(job, lastRun)) { + console.debug( + `%c[Indexer] Skipping job "${jobId}" (not due)`, + "color: gray", + ); + completedJobs++; + dispatchProgress(completedJobs, totalSteps, true, `Skipped job: ${job.label}`); + continue; + } + + // These DB operations happen on the main thread (acceptable per request) + const getStoredItems = async () => await getAll(jobId); + const setStoredItems = async (items: IndexItem[]) => { + await clear(jobId); + // Add validation before putting + const validItems = items.filter(i => i && i.id); + if (validItems.length !== items.length) { + console.warn(`[Indexer Job ${jobId}] Filtered out ${items.length - validItems.length} invalid items before storing.`); + } + await Promise.all(validItems.map((i) => put(jobId, i, i.id))); + }; + const addItem = async (item: IndexItem) => { + if (item && item.id) { // Add validation + await put(jobId, item, item.id); + } else { + console.warn(`[Indexer Job ${jobId}] Attempted to add invalid item:`, item); + } + }; + const removeItem = async (id: string) => { + await remove(jobId, id); + }; + + const ctx: JobContext = { + getStoredItems, + setStoredItems, + addItem, + removeItem, + }; + + console.debug(`%c[Indexer] Running job "${jobId}"...`, "color: #4ea1ff"); + + try { + const newItemsRaw = await job.run(ctx); + const stored = await getStoredItems(); + + let merged = mergeItems(stored, newItemsRaw); + if (job.purge) merged = job.purge(merged); + + await setStoredItems(merged); // Store merged non-vector data + await updateLastRunMeta(jobId); + + // Hydrate items for vector processing + const renderComponent = renderComponentMap[job.renderComponentId]; + if (!renderComponent) { + console.warn(`Render component not found for job ${jobId} (ID: ${job.renderComponentId}) during hydration`); + } + const hydratedItems = merged + .filter(item => item && item.id && item.text && item.category && item.actionId && job.renderComponentId) // Filter invalid before hydrating + .map((item) => ({ + ...item, + renderComponent: renderComponent || undefined, // Assign undefined if not found + })); + + if (hydratedItems.length !== merged.length) { + console.warn(`[Indexer Job ${jobId}] Filtered out ${merged.length - hydratedItems.length} invalid items during hydration.`); + } + + allItemsFromJobs.push(...hydratedItems); + + console.debug( + `%c[Indexer] ✅ ${job.label}: ${newItemsRaw.length} new items fetched, ${merged.length} total stored (non-vector).`, + "color: #00c46f", + ); + } catch (err) { + console.debug(`%c[Indexer] ❌ ${job.label} failed:`, "color: red"); + console.error(err); + } + + completedJobs++; + dispatchProgress(completedJobs, totalSteps, true, `Finished job: ${job.label}`); + } + + // --- Step 2: Delegate Vectorization to Worker (Off Main Thread) --- + if (allItemsFromJobs.length > 0) { + console.debug( + `%c[Indexer] Sending ${allItemsFromJobs.length} items to worker for vectorization...`, + "color: #4ea1ff", + ); + dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization"); + + try { + const workerManager = VectorWorkerManager.getInstance(); + // Pass a progress callback to the worker manager + await workerManager.processItems(allItemsFromJobs, (progress) => { + // Update overall progress based on worker feedback + let detailMessage = progress.message || ''; + if (progress.status === 'processing' && progress.total && progress.processed !== undefined) { + detailMessage = `Vectorizing: ${progress.processed} / ${progress.total}`; + // You could potentially update the 'completed' count more granularly here + // For simplicity, we'll just update the detail message + } else if (progress.status === 'complete') { + detailMessage = "Vectorization complete"; + // Mark the vectorization step as complete + dispatchProgress(totalSteps, totalSteps, true, "Vectorization finished"); + } else if (progress.status === 'error') { + detailMessage = `Vectorization error: ${progress.message}`; + dispatchProgress(completedJobs, totalSteps, true, "Vectorization failed", detailMessage); // Show error + } else if (progress.status === 'started') { + detailMessage = `Vectorization started for ${progress.total} items`; + } else if (progress.status === 'cancelled') { + detailMessage = `Vectorization cancelled: ${progress.message}`; + dispatchProgress(completedJobs, totalSteps, true, "Vectorization cancelled", detailMessage); + } + + // Update the status detail + dispatchProgress(completedJobs, totalSteps, true, "Vectorization in progress", detailMessage); + + // When worker signals completion of *its* task, mark the final step complete + if (progress.status === 'complete') { + completedJobs++; // Increment completion count *after* vectorization finishes + dispatchProgress(completedJobs, totalSteps, false, "Indexing finished"); // Set indexing to false + } else if (progress.status === 'error' || progress.status === 'cancelled') { + // Don't increment completed count on failure/cancel, just stop indexing indicator + dispatchProgress(completedJobs, totalSteps, false, "Indexing stopped due to error/cancel"); + } + }); + console.debug("%c[Indexer] Vectorization task sent to worker.", "color: green"); + // Note: runIndexing might return *before* vectorization is complete now. + // The progress updates will signal the true end state. + } catch (error) { + console.error(`%c[Indexer] ❌ Failed to send items to vector worker:`, "color: red", error); + dispatchProgress(completedJobs, totalSteps, false, "Vectorization failed", String(error)); // Stop indexing indicator + } + + } else { + console.debug("%c[Indexer] No items to send for vectorization.", "color: gray"); + // If no vectorization needed, indexing is done here. + completedJobs++; // Count the "skipped" vectorization step + dispatchProgress(completedJobs, totalSteps, false, "Indexing finished (no vectorization needed)"); + } + + + // Stop heartbeat ONLY when all jobs *and* the vectorization dispatch are done. + // The actual *completion* of vectorization is now asynchronous. + stopHeartbeat(); + // Final progress update might be handled by the worker callback now. + // dispatchProgress(completedJobs, totalSteps, false); // This might be premature +} + +function mergeItems(existing: IndexItem[], incoming: IndexItem[]): IndexItem[] { + const map = new Map(); + // Prioritize incoming items if IDs clash + for (const item of existing) { + if (item && item.id) map.set(item.id, item); + } + for (const item of incoming) { + if (item && item.id) map.set(item.id, item); + } + return Array.from(map.values()); +} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts new file mode 100644 index 00000000..a18d614f --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts @@ -0,0 +1,351 @@ +import type { Job } from "./types"; +import type { IndexItem } from "./types"; + +interface MessageNotification { + notificationID: number; + type: "message"; + message: { + subtitle: string; + messageID: number; + title: string; + }; + timestamp: string; +} + +interface AssessmentNotification { + notificationID: number; + type: "coneqtassessments"; + coneqtAssessments: { + programmeID: number; + metaclassID: number; + subtitle: string; + term: string; + title: string; + assessmentID: number; + subjectCode: string; + }; + timestamp: string; +} + +type Notification = MessageNotification | AssessmentNotification; + +interface MessageListResponse { + payload: { + hasMore: boolean; + messages: { + date: string; + attachments: boolean; + attachmentCount: number; + read: number; + sender: string; + sender_id: number; + sender_type: string; + subject: string; + id: number; + participants: Array<{ + name: string; + photo: string; + type: string; + }>; + }[]; + ts: string; + }; + status: string; +} + +interface MessageContentResponse { + payload: { + date: string; + blind: boolean; + read: boolean; + subject: string; + sender_type: string; + sender_id: number; + starred: boolean; + contents: string; + sender: string; + files: any[]; + id: number; + participants: Array<{ + read: number; + name: string; + photo: string; + id: number; + type: string; + }>; + }; + status: string; +} + +// Helper to strip HTML tags from text +function stripHtmlTags(html: string): string { + return html.replace(/<[^>]*>/g, ""); +} + +// Helper to fetch messages with pagination +async function fetchMessages( + offset: number = 0, + limit: number = 100, +): Promise { + const response = await fetch( + `${location.origin}/seqta/student/load/message`, + { + method: "POST", + credentials: "include", + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + body: JSON.stringify({ + searchValue: "", + sortBy: "date", + sortOrder: "desc", + action: "list", + label: "inbox", + offset, + limit, + datetimeUntil: null, + }), + }, + ); + + return await response.json(); +} + +// Helper to fetch message content +async function fetchMessageContent( + messageId: number, +): Promise { + const response = await fetch( + `${location.origin}/seqta/student/load/message`, + { + method: "POST", + credentials: "include", + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + body: JSON.stringify({ + action: "message", + id: messageId, + }), + }, + ); + + return await response.json(); +} + +// Helper to fetch notifications +async function fetchNotifications(): Promise { + const response = await fetch(`${location.origin}/seqta/student/heartbeat?`, { + method: "POST", + headers: { + "Content-Type": "application/json; charset=utf-8", + }, + body: JSON.stringify({ + timestamp: "1970-01-01 00:00:00.0", + hash: "#?page=/notifications", + }), + }); + + const json = await response.json(); + return json.notifications ?? []; +} + +export const jobs: Record = { + messages: { + id: "messages", + label: "Messages", + renderComponentId: "message", + frequency: { type: "expiry", afterMs: 1000 * 60 * 5 }, // every 5 minutes + + run: async (ctx) => { + // Get existing items first + const existing = await ctx.getStoredItems(); + const existingIds = new Set(existing.map((i) => i.id)); + const newItems: IndexItem[] = []; + let offset = 0; + const limit = 100; + let hasMore = true; + let consecutiveExisting = 0; + + // Fetch all messages with pagination + while (hasMore) { + try { + const response = await fetchMessages(offset, limit); + + if (response.status !== "200") { + console.error("Failed to fetch messages:", response); + break; + } + + const messages = response.payload.messages; + hasMore = response.payload.hasMore; + + // Process each message + for (const message of messages) { + const id = message.id.toString(); + + // Skip if we already have this message + if (existingIds.has(id)) { + consecutiveExisting++; + // If we've found 20 consecutive existing messages, assume we've caught up + if (consecutiveExisting >= 20) { + console.debug( + "[Messages Job] Found 20 consecutive existing messages, stopping fetch", + ); + hasMore = false; + break; + } + continue; + } + + // Reset consecutive counter when we find a new message + consecutiveExisting = 0; + + try { + // Fetch message content + const contentResponse = await fetchMessageContent(message.id); + + if (contentResponse.status !== "200") { + console.error( + "Failed to fetch message content:", + contentResponse, + ); + continue; + } + + const content = stripHtmlTags(contentResponse.payload.contents); + + newItems.push({ + id, + text: message.subject, + category: "messages", + content: `From: ${message.sender}\n\n${content}`, + dateAdded: new Date(message.date).getTime(), + metadata: { + messageId: message.id, + author: message.sender, + senderId: message.sender_id, + senderType: message.sender_type, + timestamp: message.date, + hasAttachments: message.attachments, + attachmentCount: message.attachmentCount, + read: message.read === 1, + }, + actionId: "message", + renderComponentId: "message", + }); + + // Add to existingIds as we process to prevent duplicates in the same run + existingIds.add(id); + } catch (error) { + console.error("Error fetching message content:", error); + continue; + } + } + + offset += limit; + } catch (error) { + console.error("Error fetching messages:", error); + break; + } + + // Small delay to avoid overwhelming the server + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + console.debug(`[Messages Job] Found ${newItems.length} new messages`); + return newItems; + }, + + purge: (items) => { + // Keep messages from the last 30 days + const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000; + return items.filter((i) => i.dateAdded >= cutoff); + }, + }, + + assessments: { + id: "assessments", + label: "Assessments", + renderComponentId: "assessment", + frequency: { type: "expiry", afterMs: 1000 * 60 * 15 }, // every 15 minutes + + run: async (ctx) => { + const notifications = await fetchNotifications(); + const assessmentNotifications = notifications.filter( + (n): n is MessageNotification | AssessmentNotification => + n.type === "coneqtassessments" || + (n.type === "message" && + n.message.title.toLowerCase().includes("assessment")), + ); + + const existing = await ctx.getStoredItems(); + const existingIds = new Set(existing.map((i) => i.id)); + const newItems: IndexItem[] = []; + + for (const notification of assessmentNotifications) { + const id = notification.notificationID.toString(); + if (existingIds.has(id)) continue; + + if (notification.type === "coneqtassessments") { + const { coneqtAssessments: assessment } = notification; + newItems.push({ + id, + text: assessment.title, + category: "assessments", + content: assessment.subtitle, + dateAdded: new Date(notification.timestamp).getTime(), + metadata: { + assessmentId: assessment.assessmentID, + subject: assessment.subjectCode, + term: assessment.term, + programmeId: assessment.programmeID, + metaclassId: assessment.metaclassID, + timestamp: notification.timestamp, + }, + actionId: "assessment", + renderComponentId: "assessment", + }); + } else { + // Handle message-based assessments + const { message } = notification; + newItems.push({ + id, + text: message.title, + category: "assessments", + content: `From: ${message.subtitle}`, + dateAdded: new Date(notification.timestamp).getTime(), + metadata: { + messageId: message.messageID, + author: message.subtitle, + timestamp: notification.timestamp, + isMessageBased: true, + }, + actionId: "assessment", + renderComponentId: "assessment", + }); + } + } + + return newItems; + }, + + purge: (items) => { + // Keep assessments from the current year + const date = new Date(); + date.setMonth(0); // January + date.setDate(1); + date.setHours(0); + date.setMinutes(0); + date.setSeconds(0); + const cutoff = date.getTime(); + return items.filter((i) => i.dateAdded >= cutoff); + }, + }, + + // We can add more job types here as needed: + // - notices + // - timetable changes + // - homework + // etc. +}; diff --git a/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts b/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts new file mode 100644 index 00000000..2dbb7b08 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts @@ -0,0 +1,10 @@ +import type { SvelteComponent } from "svelte"; +import AssessmentComponent from "../components/AssessmentItem.svelte"; +// import other components as needed + +export const renderComponentMap: Record = { + assessment: AssessmentComponent as unknown as typeof SvelteComponent, + // messages: MessageComponent, + // subject: SubjectComponent, + // etc... +}; diff --git a/src/plugins/built-in/globalSearch/src/indexing/types.ts b/src/plugins/built-in/globalSearch/src/indexing/types.ts new file mode 100644 index 00000000..094757b4 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/types.ts @@ -0,0 +1,37 @@ +import type { SvelteComponent } from "svelte"; + +export interface IndexItem { + id: string; + text: string; + category: string; + content: string; + dateAdded: number; + metadata: Record; + actionId: string; + renderComponentId: string; +} + +export interface HydratedIndexItem extends IndexItem { + renderComponent: typeof SvelteComponent; +} + +export type Frequency = + | "pageLoad" + | { type: "interval"; ms: number } + | { type: "expiry"; afterMs: number }; + +export interface JobContext { + getStoredItems: () => Promise; + setStoredItems: (items: IndexItem[]) => Promise; + addItem: (item: IndexItem) => Promise; + removeItem: (id: string) => Promise; +} + +export interface Job { + id: string; + label: string; + frequency: Frequency; + renderComponentId: string; + run: (ctx: JobContext) => Promise; + purge?: (items: IndexItem[]) => IndexItem[]; +} diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts new file mode 100644 index 00000000..04c18baa --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts @@ -0,0 +1,419 @@ +import { + EmbeddingIndex, + getEmbedding, + initializeModel, +} from "client-vector-search"; +import type { HydratedIndexItem } from "../types"; + +let vectorIndex: EmbeddingIndex | null = null; +let isInitialized = false; +let currentAbortController: AbortController | null = null; + +async function initWorker() { + if (isInitialized) { + console.debug("Vector worker already initialized."); + return; + } + console.debug("Initializing vector worker..."); + try { + await initializeModel(); + vectorIndex = new EmbeddingIndex([]); + + const stored = await vectorIndex.getAllObjectsFromIndexedDB(); + if (stored.length > 0) { + stored.forEach((item) => vectorIndex!.add(item)); + console.debug( + `Vector index loaded ${stored.length} items from IndexedDB.`, + ); + } else { + console.debug("No existing vector index found in IndexedDB."); + } + isInitialized = true; + console.debug("Vector worker initialized successfully."); + } catch (e) { + console.error("Failed to initialize vector worker:", e); + // Set as initialized even on error to prevent retries, but index will be null + isInitialized = true; + vectorIndex = null; // Ensure index is null on error + } +} + +async function vectorizeItem( + item: HydratedIndexItem, +): Promise<(HydratedIndexItem & { embedding: number[] }) | null> { + // Simplified for brevity - assumes embedding function doesn't need cancellation signal + try { + const textToEmbed = [ + item.text, + item.content, + item.category, + item.metadata?.author, + item.metadata?.subject, + ] + .filter(Boolean) + .join(" "); + + const embedding = await getEmbedding(textToEmbed); + return { ...item, embedding }; + } catch (error) { + console.error(`Error vectorizing item ${item.id}:`, error); + return null; // Return null if vectorization fails for an item + } +} + +async function processItems(items: HydratedIndexItem[], signal: AbortSignal) { + console.debug("Worker received process request."); + if (!vectorIndex) { + console.warn( + "Processing requested but vector index not ready. Attempting init.", + ); + await initWorker(); // Attempt initialization if not ready + if (!vectorIndex) { + // Check again after attempt + self.postMessage({ + type: "progress", + data: { + status: "error", + message: + "Vector index not available for processing after init attempt.", + }, + }); + return; + } + } + + // Find items we haven't processed yet by checking against the index instance + const unprocessedItems = items.filter((item) => { + if (signal.aborted) return false; // Check cancellation during filtering + try { + // Check if the item ID already exists in the index (loaded or added) + return !vectorIndex!.get({ id: item.id }); + } catch (e) { + // If get throws (e.g., item not found), it means it's unprocessed + return true; + } + }); + + if (signal.aborted) { + console.debug("Processing cancelled before starting."); + self.postMessage({ + type: "progress", + data: { + status: "cancelled", + message: "Processing cancelled before start", + }, + }); + return; + } + + if (unprocessedItems.length === 0) { + console.debug("No new items to process."); + self.postMessage({ + type: "progress", + data: { status: "complete", message: "No new items to process" }, + }); + return; + } + + console.debug(`Starting processing of ${unprocessedItems.length} items.`); + self.postMessage({ + type: "progress", + data: { + status: "started", + total: unprocessedItems.length, + processed: 0, + }, + }); + + const BATCH_SIZE = 5; + let processedCount = 0; + for (let i = 0; i < unprocessedItems.length; i += BATCH_SIZE) { + if (signal.aborted) { + console.debug("Processing cancelled during batching."); + self.postMessage({ + type: "progress", + data: { + status: "cancelled", + message: "Processing cancelled during batching", + }, + }); + return; + } + + const batch = unprocessedItems.slice(i, i + BATCH_SIZE); + // Vectorize batch + const vectorizationResults = await Promise.all(batch.map(vectorizeItem)); + const successfullyVectorized = vectorizationResults.filter( + (result) => result !== null, + ) as (HydratedIndexItem & { embedding: number[] })[]; + + if (signal.aborted) { + console.debug("Processing cancelled after vectorization batch."); + self.postMessage({ + type: "progress", + data: { + status: "cancelled", + message: "Processing cancelled after vectorization", + }, + }); + return; + } + + // Add successfully vectorized items to index + if (successfullyVectorized.length > 0) { + try { + successfullyVectorized.forEach((item) => vectorIndex!.add(item)); + } catch (e) { + console.error("Error adding batch to index:", e); + self.postMessage({ + type: "progress", + data: { status: "error", message: `Error adding to index: ${e}` }, + }); + // Decide whether to continue or stop on error + // return; // Example: Stop processing if adding fails + } + } + + if (signal.aborted) { + console.debug("Processing cancelled before saving batch."); + self.postMessage({ + type: "progress", + data: { + status: "cancelled", + message: "Processing cancelled before saving", + }, + }); + return; + } + + // Save index after processing the batch + try { + await vectorIndex!.saveIndex("indexedDB"); + console.debug(`Saved index after processing batch ${i / BATCH_SIZE + 1}`); + } catch (e) { + console.error("Error saving index batch:", e); + self.postMessage({ + type: "progress", + data: { status: "error", message: `Error saving index batch: ${e}` }, + }); + // Continue processing next batch even if saving failed? Or stop? + // return; // Example: Stop if saving fails + } + + processedCount = Math.min(i + BATCH_SIZE, unprocessedItems.length); + + // Report progress + self.postMessage({ + type: "progress", + data: { + status: "processing", + total: unprocessedItems.length, + processed: processedCount, + }, + }); + + // Yield control briefly to allow other messages (like cancellation) to be processed + await new Promise((resolve) => setTimeout(resolve, 0)); + } + + if (!signal.aborted) { + console.debug("Processing completed successfully."); + self.postMessage({ + type: "progress", + data: { status: "complete", message: "All items processed successfully" }, + }); + } else { + console.debug("Processing completed, but was cancelled."); + // No need to send 'cancelled' again if already sent during batching + // self.postMessage({ type: 'progress', data: { status: 'cancelled', message: 'Processing finished but was cancelled' }}); + } +} + +async function search( + query: string, + topK: number, + signal: AbortSignal, + messageId: string, +) { + console.debug( + `Worker received search request (ID: ${messageId}): "${query}"`, + ); + if (!vectorIndex) { + console.warn( + `Search (ID: ${messageId}) requested but vector index not ready. Attempting init.`, + ); + await initWorker(); // Attempt initialization + // Re-check after waiting/init attempt + if (!vectorIndex) { + console.error( + `Search (ID: ${messageId}) failed: Vector index unavailable after init attempt.`, + ); + self.postMessage({ + type: "searchError", + data: { messageId, error: "Vector index not available." }, + }); + return; + } + console.debug( + `Vector index ready after init for search (ID: ${messageId}).`, + ); + } + + if (signal.aborted) { + console.debug(`Search (ID: ${messageId}) cancelled before starting.`); + self.postMessage({ type: "searchCancelled", data: { messageId } }); + return; + } + + try { + console.debug(`Getting embedding for query (ID: ${messageId})...`); + const queryEmbedding = await getEmbedding(query); + + if (signal.aborted) { + console.debug(`Search (ID: ${messageId}) cancelled after embedding.`); + self.postMessage({ type: "searchCancelled", data: { messageId } }); + return; + } + + console.debug(`Performing vector search (ID: ${messageId})...`); + // Await the search and let TypeScript infer the type + const results = await vectorIndex!.search(queryEmbedding, { + topK, + useStorage: "indexedDB", // Ensure we search the stored index + }); + + console.debug( + `Vector search (ID: ${messageId}) completed with ${results.length} results.`, + ); + + if (signal.aborted) { + console.debug( + `Search (ID: ${messageId}) cancelled after search completed, discarding results.`, + ); + self.postMessage({ type: "searchCancelled", data: { messageId } }); + return; + } + + // Post results back to the main thread + self.postMessage({ type: "searchResults", data: { messageId, results } }); + } catch (error) { + console.error(`Vector search error in worker (ID: ${messageId}):`, error); + // Ensure signal isn't checked *after* an error occurred before posting error message + if (!signal.aborted) { + // Only post error if not cancelled + self.postMessage({ + type: "searchError", + data: { + messageId, + error: error instanceof Error ? error.message : String(error), + }, + }); + } else { + console.debug( + `Search (ID: ${messageId}) encountered error but was cancelled, suppressing error message.`, + ); + self.postMessage({ type: "searchCancelled", data: { messageId } }); // Still notify of cancellation + } + } +} + +// Handle messages from the main thread +self.addEventListener("message", async (e) => { + // Make sure data and type exist + if (!e.data || !e.data.type) { + console.warn("Worker received message with no data or type."); + return; + } + + const { type, data, messageId } = e.data; // messageId used for requests needing response/cancellation tracking + + // Cancel previous long-running operation (process or search) if a new one starts + if (type === "process" || type === "search") { + if (currentAbortController) { + console.debug( + `Worker cancelling previous operation due to new '${type}' request.`, + ); + currentAbortController.abort(`New '${type}' operation requested`); + } + currentAbortController = new AbortController(); + console.debug(`Worker starting new '${type}' operation.`); + } + + // Use the signal from the *current* controller for the task being started + const signal = currentAbortController?.signal; + + switch (type) { + case "process": + if (signal && data?.items) { + await processItems(data.items, signal); + } else if (!signal) { + console.error( + "Process message received but no abort signal available.", + ); + } else if (!data?.items) { + console.error("Process message received without 'items' data."); + self.postMessage({ + type: "progress", + data: { + status: "error", + message: "Process command received without items.", + }, + }); + } + break; + + case "search": + if (signal && messageId && typeof data?.query === "string") { + await search(data.query, data.topK ?? 10, signal, messageId); + } else { + const errorReason = !signal + ? "Missing signal" + : !messageId + ? "Missing messageId" + : "Missing or invalid query"; + console.error(`Search message received invalid: ${errorReason}.`, { + data, + messageId, + signalExists: !!signal, + }); + // Send an error back if messageId exists + if (messageId) { + self.postMessage({ + type: "searchError", + data: { messageId, error: `Worker internal error: ${errorReason}` }, + }); + } + } + break; + + case "init": + // Init should not be cancellable in the same way, it's foundational + // Check if already initialized before potentially running it again + if (!isInitialized) { + await initWorker(); + self.postMessage({ type: "ready" }); // Signal ready *after* init attempt + } else { + console.debug("Received init message, but worker already initialized."); + self.postMessage({ type: "ready" }); // Signal ready anyway + } + break; + + // No explicit 'cancel' case needed as new tasks auto-cancel previous ones + + default: + console.warn("Unknown message type received by vector worker:", type); + } +}); + +// Initial check or trigger for initialization when the worker starts +initWorker() + .then(() => { + self.postMessage({ type: "ready" }); + }) + .catch((err) => { + console.error("Initial worker initialization failed:", err); + // Still need to signal readiness, perhaps with an error state? + // Or rely on the first 'process' or 'search' to retry init. + // For now, just signal ready, but the index might be null. + self.postMessage({ type: "ready" }); + }); diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts new file mode 100644 index 00000000..b51cde9b --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts @@ -0,0 +1,221 @@ +import type { HydratedIndexItem } from '../types'; +import vectorWorker from './vectorWorker.ts?inlineWorker'; +import type { SearchResult } from 'client-vector-search'; + +export type ProgressCallback = (data: { + status: 'started' | 'processing' | 'complete' | 'error' | 'cancelled'; + total?: number; + processed?: number; + message?: string; +}) => void; + +export class VectorWorkerManager { + private static instance: VectorWorkerManager; + private worker: Worker | null = null; + private isInitialized = false; + private readyPromise: Promise | null = null; // To await initialization + private progressCallback: ProgressCallback | null = null; + private searchPromises = new Map void, reject: (reason?: any) => void, timer: NodeJS.Timeout }>(); + private debounceTimer: NodeJS.Timeout | null = null; + private lastSearchParams: { query: string; topK: number; resolve: (results: SearchResult[]) => void, reject: (reason?: any) => void } | null = null; + + + private constructor() { + // Start initialization immediately, but allow awaiting it + this.readyPromise = this.initWorker(); + } + + static getInstance(): VectorWorkerManager { + if (!VectorWorkerManager.instance) { + VectorWorkerManager.instance = new VectorWorkerManager(); + } + return VectorWorkerManager.instance; + } + + private async initWorker(): Promise { + // If already initialized or initializing, return the existing promise + if (this.isInitialized) return Promise.resolve(); + if (this.readyPromise) return this.readyPromise; + + return new Promise((resolve, reject) => { + // Create the worker + this.worker = vectorWorker(); + + const timeout = setTimeout(() => { + console.error('Vector worker initialization timed out'); + this.worker?.terminate(); // Clean up worker if it exists + this.worker = null; + this.isInitialized = false; // Ensure state reflects failure + this.readyPromise = null; // Allow retrying init later + reject(new Error('Worker initialization timed out')); + }, 10000); // Increased timeout + + // Set up message handling + this.worker!.addEventListener('message', (e) => { + const { type, data } = e.data; + console.debug("Message from vector worker:", type, data); + + switch (type) { + case 'ready': + this.isInitialized = true; + clearTimeout(timeout); + console.debug('Vector worker initialized and ready.'); + resolve(); // Resolve the init promise + break; + + case 'progress': + if (this.progressCallback) { + this.progressCallback(data); + } + break; + + case 'searchResults': + const searchInfo = this.searchPromises.get(data.messageId); + if (searchInfo) { + clearTimeout(searchInfo.timer); // Clear timeout on success + searchInfo.resolve(data.results); + this.searchPromises.delete(data.messageId); + } else { + console.warn('Received search results for unknown messageId:', data.messageId); + } + break; + + case 'searchError': + const errorInfo = this.searchPromises.get(data.messageId); + if (errorInfo) { + clearTimeout(errorInfo.timer); // Clear timeout on error + errorInfo.reject(new Error(data.error)); + this.searchPromises.delete(data.messageId); + } else { + console.warn('Received search error for unknown messageId:', data.messageId); + } + break; + + case 'searchCancelled': + const cancelledInfo = this.searchPromises.get(data.messageId); + if (cancelledInfo) { + clearTimeout(cancelledInfo.timer); // Clear timeout on cancel + // Reject with a specific cancellation error or resolve with empty? Let's reject. + cancelledInfo.reject(new Error('Search cancelled by worker')); + this.searchPromises.delete(data.messageId); + } else { + console.debug('Received cancellation for unknown messageId:', data.messageId); + } + break; + + default: + console.warn('Unknown message from worker:', type, data); + } + }); + + // Initialize the worker + this.worker!.postMessage({ type: 'init' }); + }); + } + + // Ensures worker is ready before proceeding + private async ensureReady() { + if (!this.readyPromise) { + // If init wasn't called or failed, try again + console.warn("Worker not initialized, attempting init..."); + this.readyPromise = this.initWorker(); + } + await this.readyPromise; + if (!this.isInitialized || !this.worker) { + throw new Error("Vector Worker is not available after initialization attempt."); + } + } + + async processItems(items: HydratedIndexItem[], onProgress?: ProgressCallback) { + await this.ensureReady(); // Wait for worker to be ready + + this.progressCallback = onProgress || null; + + // Cancel any ongoing search when starting processing + this.cancelAllSearches("Processing started"); + + console.debug(`Sending ${items.length} items to worker for processing.`); + this.worker!.postMessage({ + type: 'process', + data: { items } + }); + } + + // Public search method + public async search(query: string, topK: number = 10): Promise { + await this.ensureReady(); + + return new Promise((resolve, reject) => { + this.lastSearchParams = { query, topK, resolve, reject }; + + const messageId = crypto.randomUUID(); + if (this.lastSearchParams && this.worker) { + const currentParams = this.lastSearchParams; // Capture current params + this.lastSearchParams = null; // Clear last params *before* posting + this.debounceTimer = null; + + // Set a timeout for the search operation itself + const searchTimeout = 10000; // e.g., 10 seconds + const searchTimer = setTimeout(() => { + if (this.searchPromises.has(messageId)) { + console.error(`Search timed out for messageId: ${messageId}`); + currentParams.reject(new Error(`Search timed out after ${searchTimeout}ms`)); + this.searchPromises.delete(messageId); + } + }, searchTimeout); + + + this.searchPromises.set(messageId, { resolve: currentParams.resolve, reject: currentParams.reject, timer: searchTimer }); + + console.debug(`Sending search request (ID: ${messageId}) to worker: "${currentParams.query}"`); + this.worker.postMessage({ + type: "search", + data: { query: currentParams.query, topK: currentParams.topK }, + messageId + }); + } else if (this.lastSearchParams) { + // This case might happen if ensureReady failed but didn't throw + console.error("Worker unavailable when trying to send search request."); + this.lastSearchParams.reject(new Error("Worker unavailable for search")); + this.lastSearchParams = null; + this.debounceTimer = null; + } + }); + } + + // Method to cancel all pending/debounced searches + private cancelAllSearches(reason: string = "Cancelled") { + if (this.debounceTimer) { + clearTimeout(this.debounceTimer); + this.debounceTimer = null; + if (this.lastSearchParams) { + this.lastSearchParams.reject(new Error(`Search cancelled: ${reason}`)); + this.lastSearchParams = null; + } + } + // We might also want to tell the worker to cancel its *current* search + // if it supports it, but this requires worker modification. + // For now, just reject pending promises in the manager. + for (const [messageId, promiseInfo] of this.searchPromises.entries()) { + clearTimeout(promiseInfo.timer); + promiseInfo.reject(new Error(`Search cancelled: ${reason}`)); + this.searchPromises.delete(messageId); + } + } + + + terminate() { + console.debug("Terminating Vector Worker Manager..."); + this.cancelAllSearches("Worker terminated"); // Cancel pending searches + + if (this.worker) { + this.worker.terminate(); + this.worker = null; + } + this.isInitialized = false; + this.readyPromise = null; // Reset init promise + this.progressCallback = null; + // Clear the static instance? Or assume app lifecycle handles this? + // VectorWorkerManager.instance = null; // Uncomment if needed + } +} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/search/searchUtils.ts b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts new file mode 100644 index 00000000..ba1311ed --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts @@ -0,0 +1,215 @@ +import Fuse, { type FuseResult } from "fuse.js"; +import { getStaticCommands, type StaticCommandItem } from "../core/commands"; +import { getDynamicItems } from "../utils/dynamicItems"; +import type { CombinedResult } from "../core/types"; +import type { HydratedIndexItem } from "../indexing/types"; +import { searchVectors } from "./vector/vectorSearch"; +import type { VectorSearchResult } from "./vector/vectorTypes"; + +export function createSearchIndexes() { + const commands = getStaticCommands(); + const dynamicItems = getDynamicItems(); + + const commandOptions = { + keys: ["text", "category", "keywords"], + includeScore: true, + includeMatches: true, + threshold: 0.6, + minMatchCharLength: 1, + ignoreLocation: true, + useExtendedSearch: false, + }; + + const dynamicOptions = { + keys: [ + "text", + "content", + "category", + "metadata.author", + "metadata.subject", + ], + includeScore: true, + includeMatches: true, + threshold: 0.6, + minMatchCharLength: 3, + distance: 50, + useExtendedSearch: false, + }; + + return { + commandsFuse: new Fuse(commands, commandOptions) as Fuse, + dynamicContentFuse: new Fuse( + dynamicItems, + dynamicOptions, + ) as Fuse, + commands, + dynamicItems, + }; +} + +export function searchCommands( + commandsFuse: Fuse, + query: string, + commandIdToItemMap: Map, + limit = 10, +): CombinedResult[] { + if (!commandsFuse) return []; + + if (!query.trim()) { + return Array.from(commandIdToItemMap.values()) + .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)) // Sort by priority when no query + .slice(0, limit) // Limit results even when no query + .map((item) => ({ + id: item.id, + type: "command" as const, + score: 100 + (item.priority ?? 0), + item, + })); + } + + const searchResults = commandsFuse.search(query, { limit }); + + return searchResults.map((result: FuseResult) => { + const item = result.item; + const fuseScore = 15 * (1 - (result.score || 0.5)); + const score = fuseScore + (item.priority ?? 0); + + return { + id: item.id, + type: "command" as const, + score, + item, + matches: result.matches, + }; + }); +} + +export function searchDynamicItems( + dynamicContentFuse: Fuse, + query: string, + dynamicIdToItemMap: Map, + limit = 10, + sortByRecent: boolean = true, // Added option to control sorting +): CombinedResult[] { + if (!dynamicContentFuse) return []; + + if (!query.trim()) { + let items = Array.from(dynamicIdToItemMap.values()); + if (sortByRecent) { + items = items.sort((a, b) => b.dateAdded - a.dateAdded); + } + return items.slice(0, limit).map((item) => ({ + id: item.id, + type: "dynamic" as const, + score: 80, // Assign a default score for non-searched items + item, + })); + } + + const now = Date.now(); + const searchResults = dynamicContentFuse.search(query, { limit }); + + return searchResults.map((result: FuseResult) => { + const item = result.item; + const fuseScore = 10 * (1 - (result.score || 0.5)); + const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); + const recencyBoost = sortByRecent ? 1 / (ageInDays + 1) : 0; // Apply boost only if sorting by recent + const score = fuseScore + recencyBoost; + + return { + id: item.id, + type: "dynamic" as const, + score, + item, + matches: result.matches, + }; + }); +} + +export async function performSearch( + query: string, + commandsFuse: Fuse, + dynamicContentFuse: Fuse, + commandIdToItemMap: Map, + dynamicIdToItemMap: Map, + showRecentFirst: boolean, +): Promise { + const startTime = performance.now(); + + // Get all results first + const commandResults = searchCommands( + commandsFuse, + query, + commandIdToItemMap, + ); + const commandEndTime = performance.now(); + const dynamicResults = searchDynamicItems( + dynamicContentFuse, + query, + dynamicIdToItemMap, + 10, + showRecentFirst, + ); + const fuseEndTime = performance.now(); + + // Get vector results in parallel + let vectorResults: VectorSearchResult[] = []; + try { + vectorResults = await searchVectors(query, 10); + } catch (e) {} + const vectorEndTime = performance.now(); + + console.log("Vector results:", vectorResults); + + // Log timings + console.log(`Command search took ${commandEndTime - startTime} milliseconds`); + console.log( + `Dynamic search took ${fuseEndTime - commandEndTime} milliseconds`, + ); + console.log(`Vector search took ${vectorEndTime - fuseEndTime} milliseconds`); + + // Create a map to store our final results, using ID as key to avoid duplicates + const resultMap = new Map(); + + // Add command results first (they keep their original scores) + commandResults.forEach((r) => resultMap.set(r.id, r)); + + // Process dynamic results and vector results together + const seenIds = new Set(); + + // Add dynamic results first + dynamicResults.forEach((r) => { + seenIds.add(r.id); + const vectorMatch = vectorResults.find((v) => v.object.id === r.id); + if (vectorMatch) { + // If we found it in both searches, combine the scores + resultMap.set(r.id, { + ...r, + score: r.score + vectorMatch.similarity * 0.6, // Boost exact matches + }); + } else { + // If only in Fuse results, keep as is + resultMap.set(r.id, r); + } + }); + + // Now add any vector results we haven't seen yet + vectorResults.forEach((v) => { + const id = v.object.id; + if (!seenIds.has(id)) { + // This is a semantic match that Fuse missed - add it with the vector similarity as score + resultMap.set(id, { + id, + type: "dynamic" as const, + score: v.similarity * 0.9, // High base score for semantic matches + item: v.object, + }); + } + }); + + // Convert to array and sort by score + const results = Array.from(resultMap.values()); + results.sort((a, b) => b.score - a.score); + + return results; +} diff --git a/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts new file mode 100644 index 00000000..c3fc6c2b --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts @@ -0,0 +1,33 @@ +import { EmbeddingIndex, getEmbedding, initializeModel } from 'client-vector-search'; +import type { HydratedIndexItem } from '../../indexing/types'; +import type { SearchResult } from 'client-vector-search'; + +let vectorIndex: EmbeddingIndex | null = null; + +export async function initVectorSearch() { + try { + await initializeModel(); + vectorIndex = new EmbeddingIndex([]); + vectorIndex.preloadIndexedDB(); + } catch (e) { + console.error('Error initializing vector search', e); + } +} + +export interface VectorSearchResult extends SearchResult { + object: HydratedIndexItem & { embedding: number[] }; +} + +export async function searchVectors(query: string, topK: number = 10): Promise { + if (!vectorIndex) await initVectorSearch(); + + const queryEmbedding = await getEmbedding(query.slice(0, 100)); + + const results = await vectorIndex!.search(queryEmbedding, { + topK, + useStorage: 'indexedDB', + dedupeEntries: true + }); + + return results as VectorSearchResult[]; +} \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/search/vector/vectorTypes.ts b/src/plugins/built-in/globalSearch/src/search/vector/vectorTypes.ts new file mode 100644 index 00000000..44ce2407 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/search/vector/vectorTypes.ts @@ -0,0 +1,7 @@ +import type { SearchResult } from "client-vector-search"; +import type { HydratedIndexItem } from "../../indexing/types"; + +export interface VectorSearchResult extends SearchResult { + object: HydratedIndexItem & { embedding: number[] }; +} + \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/ui/Shortcut.svelte b/src/plugins/built-in/globalSearch/src/ui/Shortcut.svelte new file mode 100644 index 00000000..e69de29b diff --git a/src/plugins/built-in/globalSearch/src/utils/dynamicItems.ts b/src/plugins/built-in/globalSearch/src/utils/dynamicItems.ts new file mode 100644 index 00000000..5c275267 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/utils/dynamicItems.ts @@ -0,0 +1,30 @@ +import type { SvelteComponent } from "svelte"; +import type { HydratedIndexItem } from "./indexing/types"; + +export interface DynamicContentItem { + id: string; + text: string; + category: string; + content: string; + dateAdded: number; + metadata: Record; + actionId: string; + renderComponentId: string; + renderComponent?: typeof SvelteComponent; +} + +let dynamicItems: HydratedIndexItem[] = []; + +/** + * Loads a new set of dynamic items. + */ +export function loadDynamicItems(items: HydratedIndexItem[]) { + dynamicItems = items; +} + +/** + * Returns all currently loaded dynamic items. + */ +export function getDynamicItems(): HydratedIndexItem[] { + return dynamicItems; +} diff --git a/src/plugins/built-in/globalSearch/src/utils/highlight.ts b/src/plugins/built-in/globalSearch/src/utils/highlight.ts new file mode 100644 index 00000000..46caae0d --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/utils/highlight.ts @@ -0,0 +1,274 @@ +import type { FuseResultMatch, MatchIndices } from "./core/types"; + +/** + * Simple utility to remove HTML tags from a string. + */ +export function stripHtmlTags(html: string): string { + if (!html) return ""; + return html.replace(/<[^>]*>/g, "").replace("\n", " "); +} + +/** + * Removes HTML tags from a string, but preserves tags. + */ +export function stripHtmlButKeepHighlights(html: string): string { + if (!html) return ""; + // Use a placeholder for highlight tags, strip others, then restore placeholders. + const highlightOpenPlaceholder = "__HIGHLIGHT_OPEN__"; + const highlightClosePlaceholder = "__HIGHLIGHT_CLOSE__"; + + let processed = html.replace( + //g, + highlightOpenPlaceholder, + ); + processed = processed.replace(/<\/span>/g, (match, offset, fullString) => { + // Only replace if it likely corresponds to our highlight span + // This is imperfect but helps avoid replacing unrelated spans. + // Look backwards for the nearest opening placeholder. + const lastPlaceholder = fullString.lastIndexOf( + highlightOpenPlaceholder, + offset, + ); + if (lastPlaceholder !== -1) { + // Check if there's another opening tag between the placeholder and the closing span + const interveningContent = fullString.substring( + lastPlaceholder + highlightOpenPlaceholder.length, + offset, + ); + if (!/ if unsure + }); + + // Strip all remaining HTML tags + processed = processed.replace(/<[^>]*>/g, ""); + + // Restore the highlight tags + processed = processed.replace( + new RegExp(highlightOpenPlaceholder, "g"), + '', + ); + processed = processed.replace( + new RegExp(highlightClosePlaceholder, "g"), + "", + ); + + return processed; +} + +export function highlightMatch( + text: string, + term: string, + matches?: readonly FuseResultMatch[], +): string { + if (!term.trim() || !matches || matches.length === 0) return text; + + try { + // Find matches for the text field or allContent that contains the text + const fieldMatches = matches.find( + (match) => + match.key === "text" || + (match.key === "allContent" && match.value?.includes(text)), + ); + + if ( + !fieldMatches || + !fieldMatches.indices || + fieldMatches.indices.length === 0 + ) { + return text; + } + + // Create a map of character positions to mark which ones need highlighting + const highlightMap = new Array(text.length).fill(false); + + fieldMatches.indices.forEach((indices: MatchIndices) => { + const start = indices[0]; + const end = indices[1]; + + if (fieldMatches.key === "allContent") { + // Find where our text appears in the allContent + const allContent = fieldMatches.value; + const textPos = allContent?.indexOf(text) ?? -1; + + // Only highlight if the match overlaps with our text + if (textPos >= 0) { + // Adjust start and end to be relative to our text field + const relStart = start - textPos; + const relEnd = end - textPos; + + // Only highlight if the match actually overlaps with our text field + if (relEnd >= 0 && relStart < text.length) { + // Mark the overlapping characters + for ( + let i = Math.max(0, relStart); + i <= Math.min(text.length - 1, relEnd); + i++ + ) { + highlightMap[i] = true; + } + } + } + } else { + // Regular text field match - ensure indices are within bounds + if (start >= 0 && end < text.length) { + for (let i = start; i <= end; i++) { + highlightMap[i] = true; + } + } + } + }); + + let result = ""; + let inHighlight = false; + + for (let i = 0; i < text.length; i++) { + if (highlightMap[i] && !inHighlight) { + result += ''; + inHighlight = true; + } else if (!highlightMap[i] && inHighlight) { + result += ""; + inHighlight = false; + } + + result += text.charAt(i); + } + + if (inHighlight) { + result += ""; + } + + return result; + } catch (e) { + console.error("Error highlighting match:", e); + return text; + } +} + +// Function to extract and highlight content snippet using Fuse matches +export function highlightSnippet( + content: string, + term: string, + matches?: readonly FuseResultMatch[], +): string { + if (!content || !term.trim() || !matches || matches.length === 0) + return content; + + try { + // Find matches for content field or allContent that contains the content + const contentMatches = matches.find( + (match) => + match.key === "content" || + (match.key === "allContent" && match.value?.includes(content)), + ); + + if ( + !contentMatches || + !contentMatches.indices || + contentMatches.indices.length === 0 + ) { + // No content matches, return plain content + return content.length > 100 ? content.substring(0, 100) + "..." : content; + } + + // Find the match indices + let allIndices: MatchIndices[] = contentMatches.indices as MatchIndices[]; + + // If matching against allContent, adjust indices to be relative to content + if (contentMatches.key === "allContent") { + const allContent = contentMatches.value; + const contentPos = allContent?.indexOf(content) ?? -1; + + if (contentPos >= 0) { + // Adjust indices to be relative to the content field + allIndices = allIndices + .map( + (indices) => + [ + indices[0] - contentPos, + indices[1] - contentPos, + ] as MatchIndices, + ) + .filter((indices) => indices[1] >= 0 && indices[0] < content.length); + } + } + + if (allIndices.length === 0) { + return content.length > 100 ? content.substring(0, 100) + "..." : content; + } + + // Find a good center point for our snippet (average of first match) + const firstMatch = allIndices[0]; + const matchCenter = Math.floor((firstMatch[0] + firstMatch[1]) / 2); + + // Extract a window around the match + const windowSize = 100; + const start = Math.max(0, matchCenter - windowSize / 2); + const end = Math.min(content.length, matchCenter + windowSize / 2); + + // Create the basic snippet + let snippet = content.substring(start, end); + if (start > 0) snippet = "..." + snippet; + if (end < content.length) snippet += "..."; + + // Create a highlighting map for the snippet + const snippetLength = snippet.length; + const highlightMap = new Array(snippetLength).fill(false); + + // Calculate offset for the highlighting + const startOffset = start > 0 ? start - 3 : start; // Account for '...' if present + + // Mark each matched character in the snippet + allIndices.forEach((indices: MatchIndices) => { + const matchStart = indices[0]; + const matchEnd = indices[1]; + + // Skip matches outside our snippet window + if (matchEnd < start || matchStart > end) return; + + // Adjust match indices to be relative to snippet + const snippetMatchStart = Math.max(0, matchStart - startOffset); + const snippetMatchEnd = Math.min( + snippetLength - 1, + matchEnd - startOffset, + ); + + // Mark characters for highlighting + for (let i = snippetMatchStart; i <= snippetMatchEnd; i++) { + if (i >= 0 && i < snippetLength) { + highlightMap[i] = true; + } + } + }); + + // Build the highlighted snippet + let result = ""; + let inHighlight = false; + + for (let i = 0; i < snippetLength; i++) { + // If highlighting state changes, add appropriate tags + if (highlightMap[i] && !inHighlight) { + result += ''; + inHighlight = true; + } else if (!highlightMap[i] && inHighlight) { + result += ""; + inHighlight = false; + } + + // Add the current character + result += snippet.charAt(i); + } + + // Close highlight tag if we're still in one at the end + if (inHighlight) { + result += ""; + } + + return result; + } catch (e) { + console.error("Error highlighting snippet:", e); + return content.length > 100 ? content.substring(0, 100) + "..." : content; + } +} diff --git a/src/plugins/built-in/test/index.ts b/src/plugins/built-in/test/index.ts index 752a43e8..24f7956e 100644 --- a/src/plugins/built-in/test/index.ts +++ b/src/plugins/built-in/test/index.ts @@ -31,8 +31,13 @@ const testPlugin: Plugin = { run: async (api) => { console.log('Test plugin running'); + api.events.on('ping', (data) => { + console.log('Ping received! Page changed to: ', data); + }); + const { unregister } = api.seqta.onPageChange((page) => { - console.log('Page changed to', page); + //console.log('Page changed to', page); + api.events.emit('ping', page); console.log('Current setting value:', api.settings.someSetting); }); diff --git a/src/plugins/index.ts b/src/plugins/index.ts index 5332e966..06db08b5 100644 --- a/src/plugins/index.ts +++ b/src/plugins/index.ts @@ -6,6 +6,9 @@ import notificationCollectorPlugin from './built-in/notificationCollector'; import themesPlugin from './built-in/themes'; import animatedBackgroundPlugin from './built-in/animatedBackground'; import assessmentsAveragePlugin from './built-in/assessmentsAverage'; +import globalSearchPlugin from './built-in/globalSearch/src/core'; +import testPlugin from './built-in/test'; + // Initialize plugin manager const pluginManager = PluginManager.getInstance(); @@ -15,7 +18,8 @@ pluginManager.registerPlugin(animatedBackgroundPlugin); pluginManager.registerPlugin(assessmentsAveragePlugin); pluginManager.registerPlugin(notificationCollectorPlugin); pluginManager.registerPlugin(timetablePlugin); -//pluginManager.registerPlugin(testPlugin); +pluginManager.registerPlugin(globalSearchPlugin); +pluginManager.registerPlugin(testPlugin); export { init as Monofile } from './monofile'; diff --git a/tsconfig.json b/tsconfig.json index eb905118..fc2ff7ba 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,5 +33,5 @@ "node" ] }, - "include": ["src/**/*.ts", "src/**/*.js", "src/**/*.svelte", "src/interface/+layout.sveltes"] + "include": ["src/**/*.ts", "src/**/*.js", "src/**/*.svelte", "src/interface/+layout.svelte", "declarations.d.ts"] } diff --git a/vite.config.ts b/vite.config.ts index 9254e6d8..2cdac04c 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,7 +1,9 @@ import { defineConfig } from 'vite'; -import path, { join, resolve } from 'path'; -import fs from 'fs'; +import { join, resolve } from 'path'; + import { updateManifestPlugin } from './lib/patchPackage'; +import touchGlobalCSSPlugin from './lib/touchGlobalCSS'; +import InlineWorkerPlugin from './lib/inlineWorker'; import { base64Loader } from './lib/base64loader'; import type { BuildTarget } from './lib/types'; import ClosePlugin from './lib/closePlugin'; @@ -19,7 +21,6 @@ import { opera } from './src/manifests/opera'; import { safari } from './src/manifests/safari'; import { crx } from '@crxjs/vite-plugin'; -import touchGlobalCSSPlugin from './lib/touchGlobalCSS'; const targets: BuildTarget[] = [ chrome, brave, edge, firefox, opera, safari ] @@ -30,6 +31,7 @@ const mode = process.env.MODE || 'chrome'; // Check the environment variable to export default defineConfig(({ command }) => ({ plugins: [ base64Loader, + InlineWorkerPlugin(), svelte({ emitCss: false }), @@ -70,6 +72,9 @@ export default defineConfig(({ command }) => ({ legacy: { skipWebSocketTokenCheck: true, }, + worker: { + format: 'es', + }, build: { outDir: resolve(__dirname, 'dist', mode), emptyOutDir: false,