BetterSEQTA/BetterSEQTA-Plus
1
0
Fork 0
mirror of https://github.com/BetterSEQTA/BetterSEQTA-Plus.git synced 2026-06-06 03:34:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add working workers with builds

Browse Source
This commit is contained in:
SethBurkart123
2025-04-11 00:07:29 +10:00
parent 814647e835
commit 886d0a95f1
35 changed files with 1710 additions and 901 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.prettierrc
+1 -1
View File
@@ -1,5 +1,5 @@
{ {
"tabWidth": 2, "tabWidth": 2,
"useTabs": false, "useTabs": false,
"semi": false "semi": true
} }
lib/inlineWorker.ts
+37
View File
@@ -0,0 +1,37 @@
// vite-plugin-inline-worker-dev.ts
import { Plugin } from 'vite'
import fs from 'fs/promises'
import { build, transform } from 'esbuild'
export default function InlineWorkerDevPlugin(): Plugin {
return {
name: 'vite:inline-worker-dev',
async load(id) {
if (id.includes('?inlineWorker')) {
const [cleanPath] = id.split('?')
console.log('cleanPath', cleanPath)
const code = await fs.readFile(cleanPath, 'utf-8')
const result = await build({
entryPoints: [cleanPath],
bundle: true,
write: false,
platform: 'browser',
format: 'iife',
target: 'esnext',
})
const workerCode = result.outputFiles[0].text
const workerBlobCode = `
const code = ${JSON.stringify(workerCode)};
export default function InlineWorker() {
const blob = new Blob([code], { type: 'application/javascript' });
return new Worker(URL.createObjectURL(blob), { type: 'module' });
}
`
return workerBlobCode
}
return null
}
}
}
src/declarations.d.ts
Vendored
+5
View File
@@ -5,6 +5,11 @@ declare module '*.png';
declare module '*.html'; declare module '*.html';
declare module '*.svelte'; declare module '*.svelte';
declare module '*?inlineWorker' {
const value: () => Worker;
export default value;
}
declare module "*.png?base64" { declare module "*.png?base64" {
const value: string; const value: string;
export default value; export default value;
src/manifests/manifest.json
+1 -9
View File
@@ -32,15 +32,7 @@
], ],
"web_accessible_resources": [ "web_accessible_resources": [
{ {
"resources": ["*/*"], "resources": ["*/*", "resources/*", "seqta/utils/migration/migrate.html", "plugins/built-in/globalSearch/*"],
"matches": ["*://*/*"]
},
{
"resources": ["resources/*"],
"matches": ["*://*/*"]
},
{
"resources": ["seqta/utils/migration/migrate.html"],
"matches": ["*://*/*"] "matches": ["*://*/*"]
} }
] ]
src/plugins/built-in/globalSearch/SearchBar.svelte
+3 -3
View File
@@ -3,12 +3,12 @@
import { settingsState } from '@/seqta/utils/listeners/SettingsState' import { settingsState } from '@/seqta/utils/listeners/SettingsState'
import { fade, scale } from 'svelte/transition'; import { fade, scale } from 'svelte/transition';
import { circOut, quintOut } from 'svelte/easing'; import { circOut, quintOut } from 'svelte/easing';
import { type StaticCommandItem } from './commands'; import { type StaticCommandItem } from './core/commands';
import type { CombinedResult } from './types'; import type { CombinedResult } from './core/types';
import { createSearchIndexes, performSearch as doSearch } from './searchUtils'; import { createSearchIndexes, performSearch as doSearch } from './searchUtils';
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from './highlightUtils'; import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from './highlightUtils';
import Fuse from 'fuse.js'; import Fuse from 'fuse.js';
import Calculator from './Calculator.svelte'; import Calculator from './components/Calculator.svelte';
import { actionMap } from './indexing/actions'; import { actionMap } from './indexing/actions';
import type { IndexItem, HydratedIndexItem } from './indexing/types'; import type { IndexItem, HydratedIndexItem } from './indexing/types';
src/plugins/built-in/globalSearch/client-vector-search-docs.md
+63 -34
View File
@@ -20,64 +20,63 @@ Our goal is to build a super simple, fast vector search that works with couple h
We'll initially keep things super simple and sub 100ms We'll initially keep things super simple and sub 100ms
### TODOs ### TODOs
- [ ] add HNSW index that works on node and browser env, don't rely on hnsw binder libs - [ ] add HNSW index that works on node and browser env, don't rely on hnsw binder libs
- [ ] add a proper testing suite and ci/cd for the lib - [ ] add a proper testing suite and ci/cd for the lib
- [ ] simple health tests - [ ] simple health tests
- [ ] mock the @xenova/transformers for jest, it's not happy with it - [ ] mock the @xenova/transformers for jest, it's not happy with it
- [ ] performance tests, recall, memory usage, cpu usage etc. - [ ] performance tests, recall, memory usage, cpu usage etc.
## Installation ## Installation
```bash ```bash
npm i client-vector-search npm i client-vector-search
``` ```
## Quickstart ## Quickstart
This library provides a plug-and-play solution for embedding and vector search. It's designed to be easy to use, efficient, and versatile. Here's a quick start guide: This library provides a plug-and-play solution for embedding and vector search. It's designed to be easy to use, efficient, and versatile. Here's a quick start guide:
```ts ```ts
import { getEmbedding, EmbeddingIndex } from 'client-vector-search'; import { getEmbedding, EmbeddingIndex } from "client-vector-search";
// getEmbedding is an async function, so you need to use 'await' or '.then()' to get the result // getEmbedding is an async function, so you need to use 'await' or '.then()' to get the result
const embedding = await getEmbedding("Apple"); // Returns embedding as number[] const embedding = await getEmbedding("Apple"); // Returns embedding as number[]
// Each object should have an 'embedding' property of type number[] // Each object should have an 'embedding' property of type number[]
const initialObjects = [ const initialObjects = [
{ id: 1, name: "Apple", embedding: embedding }, { id: 1, name: "Apple", embedding: embedding },
{ id: 2, name: "Banana", embedding: await getEmbedding("Banana") }, { id: 2, name: "Banana", embedding: await getEmbedding("Banana") },
{ id: 3, name: "Cheddar", embedding: await getEmbedding("Cheddar")}, { id: 3, name: "Cheddar", embedding: await getEmbedding("Cheddar") },
{ id: 4, name: "Space", embedding: await getEmbedding("Space")}, { id: 4, name: "Space", embedding: await getEmbedding("Space") },
{ id: 5, name: "database", embedding: await getEmbedding("database")}, { id: 5, name: "database", embedding: await getEmbedding("database") },
]; ];
const index = new EmbeddingIndex(initialObjects); // Creates an index const index = new EmbeddingIndex(initialObjects); // Creates an index
// The query should be an embedding of type number[] // The query should be an embedding of type number[]
const queryEmbedding = await getEmbedding('Fruit'); // Query embedding const queryEmbedding = await getEmbedding("Fruit"); // Query embedding
const results = await index.search(queryEmbedding, { topK: 5 }); // Returns top similar objects const results = await index.search(queryEmbedding, { topK: 5 }); // Returns top similar objects
// specify the storage type // specify the storage type
await index.saveIndex('indexedDB'); await index.saveIndex("indexedDB");
const results = await index.search([1, 2, 3], { const results = await index.search([1, 2, 3], {
topK: 5, topK: 5,
useStorage: 'indexedDB', useStorage: "indexedDB",
// storageOptions: { // use only if you overrode the defaults // storageOptions: { // use only if you overrode the defaults
// indexedDBName: 'clientVectorDB', // indexedDBName: 'clientVectorDB',
// indexedDBObjectStoreName: 'ClientEmbeddingStore', // indexedDBObjectStoreName: 'ClientEmbeddingStore',
// }, // },
}); });
console.log(results); console.log(results);
await index.deleteIndexedDB(); // if you overrode default, specify db name await index.deleteIndexedDB(); // if you overrode default, specify db name
``` ```
## Trouble-shooting ## Trouble-shooting
### NextJS ### NextJS
To use it inside NextJS projects you'll need to update the `next.config.js` file to include the following: To use it inside NextJS projects you'll need to update the `next.config.js` file to include the following:
```js ```js
@@ -120,26 +119,31 @@ Until we have a reference documentation, you can find all the methods and their
Let's get started! Let's get started!
### Step 1: Generate Embeddings for String ### Step 1: Generate Embeddings for String
Generate embeddings for a given string using the `getEmbedding` method. Generate embeddings for a given string using the `getEmbedding` method.
```ts ```ts
const embedding = await getEmbedding("Apple"); // Returns embedding as number[] const embedding = await getEmbedding("Apple"); // Returns embedding as number[]
``` ```
> **Note**: `getEmbedding` is asynchronous; make sure to use `await`. > **Note**: `getEmbedding` is asynchronous; make sure to use `await`.
--- ---
### Step 2: Calculate Cosine Similarity ### Step 2: Calculate Cosine Similarity
Calculate the cosine similarity between two embeddings. Calculate the cosine similarity between two embeddings.
```ts ```ts
const similarity = cosineSimilarity(embedding1, embedding2, 6); const similarity = cosineSimilarity(embedding1, embedding2, 6);
``` ```
> **Note**: Both embeddings should be of the same length. > **Note**: Both embeddings should be of the same length.
--- ---
### Step 3: Create an Index ### Step 3: Create an Index
Create an index with an initial array of objects. Each object must have an 'embedding' property. Create an index with an initial array of objects. Each object must have an 'embedding' property.
```ts ```ts
@@ -150,26 +154,37 @@ const index = new EmbeddingIndex(initialObjects);
--- ---
### Step 4: Add to Index ### Step 4: Add to Index
Add an object to the index. Add an object to the index.
```ts ```ts
const objectToAdd = { id: 6, name: 'Cat', embedding: await getEmbedding('Cat') }; const objectToAdd = {
id: 6,
name: "Cat",
embedding: await getEmbedding("Cat"),
};
index.add(objectToAdd); index.add(objectToAdd);
``` ```
--- ---
### Step 5: Update Index ### Step 5: Update Index
Update an existing object in the index. Update an existing object in the index.
```ts ```ts
const vectorToUpdate = { id: 6, name: 'Dog', embedding: await getEmbedding('Dog') }; const vectorToUpdate = {
id: 6,
name: "Dog",
embedding: await getEmbedding("Dog"),
};
index.update({ id: 6 }, vectorToUpdate); index.update({ id: 6 }, vectorToUpdate);
``` ```
--- ---
### Step 6: Remove from Index ### Step 6: Remove from Index
Remove an object from the index. Remove an object from the index.
```ts ```ts
@@ -179,6 +194,7 @@ index.remove({ id: 6 });
--- ---
### Step 7: Retrieve from Index ### Step 7: Retrieve from Index
Retrieve an object from the index. Retrieve an object from the index.
```ts ```ts
@@ -188,16 +204,18 @@ const vector = index.get({ id: 1 });
--- ---
### Step 8: Search the Index ### Step 8: Search the Index
Search the index with a query embedding. Search the index with a query embedding.
```ts ```ts
const queryEmbedding = await getEmbedding('Fruit'); const queryEmbedding = await getEmbedding("Fruit");
const results = await index.search(queryEmbedding, { topK: 5 }); const results = await index.search(queryEmbedding, { topK: 5 });
``` ```
--- ---
### Step 9: Print the Index ### Step 9: Print the Index
Print the entire index to the console. Print the entire index to the console.
```ts ```ts
@@ -207,18 +225,23 @@ index.printIndex();
--- ---
### Step 10: Save Index to IndexedDB (for browser) ### Step 10: Save Index to IndexedDB (for browser)
Save the index to a persistent IndexedDB database. Note Save the index to a persistent IndexedDB database. Note
```ts ```ts
await index.saveIndex("indexedDB", { DBName: "clientVectorDB", objectStoreName:"ClientEmbeddingStore"}) await index.saveIndex("indexedDB", {
DBName: "clientVectorDB",
objectStoreName: "ClientEmbeddingStore",
});
``` ```
--- ---
### Important: Search in indexedDB ### Important: Search in indexedDB
Perform a search operation in the IndexedDB. Perform a search operation in the IndexedDB.
```ts ````ts
const results = await index.search(queryEmbedding, { const results = await index.search(queryEmbedding, {
topK: 5, topK: 5,
useStorage: "indexedDB", useStorage: "indexedDB",
@@ -235,30 +258,36 @@ To delete an entire database.
```ts ```ts
await IndexedDbManager.deleteIndexedDB("clientVectorDB"); await IndexedDbManager.deleteIndexedDB("clientVectorDB");
``` ````
--- ---
### Delete Object Store ### Delete Object Store
To delete an object store from a database. To delete an object store from a database.
```ts ```ts
await IndexedDbManager.deleteIndexedDBObjectStore("clientVectorDB", "ClientEmbeddingStore"); await IndexedDbManager.deleteIndexedDBObjectStore(
"clientVectorDB",
"ClientEmbeddingStore",
);
``` ```
--- ---
### Retrieve All Objects ### Retrieve All Objects
To retrieve all objects from a specific object store. To retrieve all objects from a specific object store.
```ts ```ts
const allObjects = await IndexedDbManager.getAllObjectsFromIndexedDB("clientVectorDB", "ClientEmbeddingStore"); const allObjects = await IndexedDbManager.getAllObjectsFromIndexedDB(
"clientVectorDB",
"ClientEmbeddingStore",
);
``` ```
# THE MAIN INDEX.TS FILE THAT YOU ARE IMPORTING FROM # THE MAIN INDEX.TS FILE THAT YOU ARE IMPORTING FROM
```index.ts ```index.ts
const DEFAULT_TOP_K = 3; const DEFAULT_TOP_K = 3;
src/plugins/built-in/globalSearch/commands.ts
-85
View File
@@ -1,85 +0,0 @@
import { settingsState } from '@/seqta/utils/listeners/SettingsState';
import { loadHomePage } from '@/seqta/utils/Loaders/LoadHomePage';
export interface BaseCommandItem {
id: string;
text: string;
category: string;
icon: string;
action: () => void;
keywords?: string[];
priority?: number;
}
export interface StaticCommandItem extends BaseCommandItem {
keybind?: string[];
keybindLabel?: string[];
}
const staticCommands: StaticCommandItem[] = [
{
id: 'home',
icon: '\ueb4c',
category: 'navigation',
text: 'Home',
keybind: ['alt+h'],
keybindLabel: ['Alt', 'H'],
action: () => {
window.location.hash = '?page=/home';
loadHomePage();
},
priority: 4
},
{
id: 'messages',
icon: '\uebfd',
category: 'navigation',
text: 'Direct Messages',
keybind: ['alt+m'],
keybindLabel: ['Alt', 'M'],
action: () => {
window.location.hash = '?page=/messages';
},
priority: 4
},
{
id: 'timetable',
icon: '\ue9cd',
category: 'navigation',
text: 'Timetable',
keybind: ['alt+t'],
keybindLabel: ['Alt', 'T'],
action: () => {
window.location.hash = '?page=/timetable';
},
priority: 4
},
{
id: 'assessments',
icon: '\ueac3',
category: 'navigation',
text: 'Assessments',
keybind: ['alt+a'],
keybindLabel: ['Alt', 'A'],
action: () => {
window.location.hash = '?page=/assessments';
},
priority: 4
},
{
id: 'toggle-dark-mode',
icon: '\uecfe',
category: 'action',
text: 'Toggle Dark Mode',
action: () => settingsState.DarkMode = !settingsState.DarkMode,
priority: 2,
keywords: ['theme', 'appearance']
}
];
/**
* Returns the predefined list of static commands.
*/
export const getStaticCommands = (): StaticCommandItem[] => {
return [...staticCommands];
};
src/plugins/built-in/globalSearch/components/AssessmentItem.svelte
+1 -1
View File
@@ -1,7 +1,7 @@
<script lang="ts"> <script lang="ts">
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../highlightUtils'; import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../highlightUtils';
import type { DynamicContentItem } from '../dynamicSearch'; import type { DynamicContentItem } from '../dynamicSearch';
import type { FuseResultMatch } from '../types'; import type { FuseResultMatch } from '../core/types';
const { item, isSelected, searchTerm, matches } = $props<{ const { item, isSelected, searchTerm, matches } = $props<{
item: DynamicContentItem; item: DynamicContentItem;
src/plugins/built-in/globalSearch/Calculator.svelte → src/plugins/built-in/globalSearch/components/Calculator.svelte
View File
src/plugins/built-in/globalSearch/components/unitMap.ts
+193
View File
@@ -0,0 +1,193 @@
export const unitFullNames: Record<string, string> = {
// --- Length ---
m: "Meters",
km: "Kilometers",
cm: "Centimeters",
mm: "Millimeters",
µm: "Micrometers",
nm: "Nanometers",
pm: "Picometers",
fm: "Femtometers",
am: "Attometers",
zm: "Zeptometers",
ym: "Yoctometers",
mi: "Miles",
yd: "Yards",
ft: "Feet",
in: "Inches",
nmi: "Nautical Miles",
angstrom: "Angstroms",
au: "Astronomical Units",
ly: "Light Years",
pc: "Parsecs",
// --- Mass ---
kg: "Kilograms",
g: "Grams",
mg: "Milligrams",
µg: "Micrograms",
ng: "Nanograms",
lb: "Pounds",
oz: "Ounces",
ton: "Tons (Imperial)",
tonne: "Tonnes (Metric)",
slug: "Slugs",
stone: "Stones",
// --- Time ---
s: "Seconds",
ms: "Milliseconds",
µs: "Microseconds",
ns: "Nanoseconds",
ps: "Picoseconds",
min: "Minutes",
h: "Hours",
day: "Days",
week: "Weeks",
month: "Months (30 days)",
year: "Years (365 days)",
fortnight: "Fortnights",
// --- Temperature ---
K: "Kelvin",
degC: "Degrees Celsius",
degF: "Degrees Fahrenheit",
degR: "Degrees Rankine",
// --- Volume ---
"m³": "Cubic Meters",
"cm³": "Cubic Centimeters",
"mm³": "Cubic Millimeters",
l: "Liters",
ml: "Milliliters",
gal: "Gallons (US)",
qt: "Quarts (US)",
pt: "Pints (US)",
cup: "Cups (US)",
floz: "Fluid Ounces (US)",
tbsp: "Tablespoons (US)",
tsp: "Teaspoons (US)",
// --- Area ---
"m²": "Square Meters",
"km²": "Square Kilometers",
"cm²": "Square Centimeters",
"mm²": "Square Millimeters",
ha: "Hectares",
acre: "Acres",
"ft²": "Square Feet",
"in²": "Square Inches",
"mi²": "Square Miles",
// --- Speed ---
"m/s": "Meters per Second",
"km/h": "Kilometers per Hour",
mph: "Miles per Hour",
knot: "Knots",
// --- Acceleration ---
"m/s²": "Meters per Second Squared",
// --- Force ---
N: "Newtons",
lbf: "Pound-Force",
dyn: "Dynes",
// --- Energy ---
J: "Joules",
kJ: "Kilojoules",
cal: "Calories",
kcal: "Kilocalories",
Wh: "Watt Hours",
kWh: "Kilowatt Hours",
BTU: "British Thermal Units",
erg: "Ergs",
eV: "Electronvolts",
// --- Power ---
W: "Watts",
kW: "Kilowatts",
MW: "Megawatts",
GW: "Gigawatts",
hp: "Horsepower",
// --- Pressure ---
Pa: "Pascals",
kPa: "Kilopascals",
bar: "Bar",
atm: "Atmospheres",
psi: "Pounds per Square Inch",
torr: "Torr",
mmHg: "Millimeters of Mercury",
// --- Frequency ---
Hz: "Hertz",
kHz: "Kilohertz",
MHz: "Megahertz",
GHz: "Gigahertz",
THz: "Terahertz",
// --- Electric ---
V: "Volts",
A: "Amperes",
C: "Coulombs",
Ω: "Ohms",
F: "Farads",
S: "Siemens",
H: "Henries",
Wb: "Webers",
T: "Teslas",
lx: "Lux",
// --- Angle & Rotation ---
rad: "Radians",
deg: "Degrees",
grad: "Gradians",
cycle: "Cycles",
turn: "Turns",
rev: "Revolutions",
// --- Charge & Capacitance ---
e: "Elementary Charges",
// --- Magnetic & Light ---
lm: "Lumens",
ph: "Photons",
// --- Miscellaneous / Dimensionless ---
"%": "Percent",
ppm: "Parts per Million",
ppb: "Parts per Billion",
pptr: "Parts per Trillion",
dB: "Decibels",
bit: "Bits",
byte: "Bytes",
// --- Digital Storage ---
b: "Bits",
B: "Bytes",
kb: "Kilobits",
kB: "Kilobytes",
Mb: "Megabits",
MB: "Megabytes",
Gb: "Gigabits",
GB: "Gigabytes",
Tb: "Terabits",
TB: "Terabytes",
// --- Currency ---
USD: "US Dollars",
EUR: "Euros",
GBP: "British Pounds",
AUD: "Australian Dollars",
CAD: "Canadian Dollars",
CHF: "Swiss Francs",
JPY: "Japanese Yen",
CNY: "Chinese Yuan",
INR: "Indian Rupees",
NZD: "New Zealand Dollars",
SEK: "Swedish Krona",
NOK: "Norwegian Krone",
SGD: "Singapore Dollars",
HKD: "Hong Kong Dollars",
};
src/plugins/built-in/globalSearch/core/commands.ts
+85
View File
@@ -0,0 +1,85 @@
import { settingsState } from "@/seqta/utils/listeners/SettingsState";
import { loadHomePage } from "@/seqta/utils/Loaders/LoadHomePage";
export interface BaseCommandItem {
id: string;
text: string;
category: string;
icon: string;
action: () => void;
keywords?: string[];
priority?: number;
}
export interface StaticCommandItem extends BaseCommandItem {
keybind?: string[];
keybindLabel?: string[];
}
const staticCommands: StaticCommandItem[] = [
{
id: "home",
icon: "\ueb4c",
category: "navigation",
text: "Home",
keybind: ["alt+h"],
keybindLabel: ["Alt", "H"],
action: () => {
window.location.hash = "?page=/home";
loadHomePage();
},
priority: 4,
},
{
id: "messages",
icon: "\uebfd",
category: "navigation",
text: "Direct Messages",
keybind: ["alt+m"],
keybindLabel: ["Alt", "M"],
action: () => {
window.location.hash = "?page=/messages";
},
priority: 4,
},
{
id: "timetable",
icon: "\ue9cd",
category: "navigation",
text: "Timetable",
keybind: ["alt+t"],
keybindLabel: ["Alt", "T"],
action: () => {
window.location.hash = "?page=/timetable";
},
priority: 4,
},
{
id: "assessments",
icon: "\ueac3",
category: "navigation",
text: "Assessments",
keybind: ["alt+a"],
keybindLabel: ["Alt", "A"],
action: () => {
window.location.hash = "?page=/assessments";
},
priority: 4,
},
{
id: "toggle-dark-mode",
icon: "\uecfe",
category: "action",
text: "Toggle Dark Mode",
action: () => (settingsState.DarkMode = !settingsState.DarkMode),
priority: 2,
keywords: ["theme", "appearance"],
},
];
/**
* Returns the predefined list of static commands.
*/
export const getStaticCommands = (): StaticCommandItem[] => {
return [...staticCommands];
};
src/plugins/built-in/globalSearch/core/index.ts
+176
View File
@@ -0,0 +1,176 @@
import type { Plugin } from "@/plugins/core/types";
import { BasePlugin } from "@/plugins/core/settings";
import {
booleanSetting,
defineSettings,
Setting,
stringSetting,
} from "@/plugins/core/settingsHelpers";
import renderSvelte from "@/interface/main";
import SearchBar from "../SearchBar.svelte";
import styles from "./styles.css?inline";
import { unmount } from "svelte";
import { loadDynamicItems } from "../dynamicSearch";
import { waitForElm } from "@/seqta/utils/waitForElm";
import { runIndexing, loadAllStoredItems } from "../indexing/indexer";
//import { initVectorSearch } from "../search/vector/vectorSearch";
import { VectorWorkerManager } from "../indexing/worker/vectorWorkerManager";
import indexWorker from "./indexWorker?inlineWorker";
const settings = defineSettings({
searchHotkey: stringSetting({
default: "ctrl+k",
title: "Search Hotkey",
description: "Keyboard shortcut to open the search (cmd on Mac)",
}),
showRecentFirst: booleanSetting({
default: true,
title: "Show Recent First",
description: "Sort dynamic content by most recent first",
}),
transparencyEffects: booleanSetting({
default: true,
title: "Transparency Effects",
description: "Enable transparency effects for the search bar",
}),
runIndexingOnLoad: booleanSetting({
default: true,
title: "Index on Page Load",
description: "Run content indexing when SEQTA loads",
}),
});
class GlobalSearchPlugin extends BasePlugin<typeof settings> {
@Setting(settings.searchHotkey)
searchHotkey!: string;
@Setting(settings.showRecentFirst)
showRecentFirst!: boolean;
@Setting(settings.transparencyEffects)
transparencyEffects!: boolean;
@Setting(settings.runIndexingOnLoad)
runIndexingOnLoad!: boolean;
}
const settingsInstance = new GlobalSearchPlugin();
const updateDynamicItemsFromIndex = async () => {
const indexedItems = await loadAllStoredItems();
loadDynamicItems(indexedItems);
console.log(`Loaded ${indexedItems.length} indexed items into search.`);
// Process items through vector search worker
const workerManager = VectorWorkerManager.getInstance();
await workerManager.processItems(indexedItems, (progress) => {
if (progress.status === "started") {
console.debug(`Starting vector processing of ${progress.total} items...`);
} else if (progress.status === "processing") {
console.debug(`Vectorized ${progress.processed}/${progress.total} items`);
} else if (progress.status === "complete") {
console.debug("Vector processing complete:", progress.message);
}
});
window.dispatchEvent(new CustomEvent("dynamic-items-updated"));
};
const globalSearchPlugin: Plugin<typeof settings> = {
id: "global-search",
name: "Global Search",
description: "Quick search for everything in SEQTA",
version: "1.0.0",
settings: settingsInstance.settings,
disableToggle: true,
styles: styles,
run: async (api) => {
let app: any;
console.log("=======================")
/* const worker = new Worker(new URL('./indexWorker.ts', import.meta.url), {
type: 'module',
}); */
//const blob = new Blob([indexWorker], { type: 'application/javascript' });
const worker = indexWorker();
worker.addEventListener("message", (e) => {
console.log(e);
});
worker.postMessage({ type: "ready" });
console.log("=======================")
// Run initial indexing and update dynamic items
if (api.settings.runIndexingOnLoad) {
setTimeout(async () => {
await runIndexing();
await updateDynamicItemsFromIndex();
}, 2000); // Delay initial indexing to let page load
}
const mountSearchBar = (titleElement: Element) => {
if (titleElement.querySelector(".search-trigger")) {
return;
}
const searchButton = document.createElement("div");
searchButton.className = "search-trigger";
searchButton.innerHTML = /* html */ `
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<circle cx="11" cy="11" r="8"></circle>
<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
</svg>
<p>Quick search...</p>
<span style="margin-left: auto; display: flex; align-items: center; color: #777; font-size: 12px;">⌘K</span>
`;
titleElement.appendChild(searchButton);
const searchRoot = document.createElement("div");
document.body.appendChild(searchRoot);
const searchRootShadow = searchRoot.attachShadow({ mode: "open" });
console.log("adding event listener to search button");
searchButton.addEventListener("click", () => {
console.log("search button clicked");
// @ts-ignore - Intentionally adding to window
window.setCommandPalleteOpen(true);
});
try {
app = renderSvelte(SearchBar, searchRootShadow, {
transparencyEffects: api.settings.transparencyEffects ? true : false,
showRecentFirst: api.settings.showRecentFirst,
});
} catch (error) {
console.error("Error rendering Svelte component:", error);
}
};
const title = document.querySelector("#title");
if (title) {
mountSearchBar(title);
} else {
await waitForElm("#title", true, 100, 60);
mountSearchBar(document.querySelector("#title") as Element);
}
return () => {
const searchButton = document.querySelector(".search-trigger");
const searchRoot = document.querySelector(".global-search-root");
if (searchButton) searchButton.remove();
if (searchRoot) searchRoot.remove();
// Clean up vector worker
VectorWorkerManager.getInstance().terminate();
unmount(app);
};
},
};
export default globalSearchPlugin;
src/plugins/built-in/globalSearch/core/indexWorker.ts
+5
View File
@@ -0,0 +1,5 @@
self.addEventListener('message', (e: Event) => {
console.log(e);
});
self.postMessage({ type: 'ready' });
src/plugins/built-in/globalSearch/styles.css → src/plugins/built-in/globalSearch/core/styles.css
View File
src/plugins/built-in/globalSearch/types.ts → src/plugins/built-in/globalSearch/core/types.ts
+3 -3
View File
@@ -1,5 +1,5 @@
import type { StaticCommandItem } from './commands'; import type { StaticCommandItem } from "./commands";
import type { HydratedIndexItem } from './indexing/types'; import type { HydratedIndexItem } from "../indexing/types";
export interface MatchIndices { export interface MatchIndices {
readonly 0: number; readonly 0: number;
@@ -14,7 +14,7 @@ export interface FuseResultMatch {
export interface CombinedResult { export interface CombinedResult {
id: string; id: string;
type: 'command' | 'dynamic'; type: "command" | "dynamic";
score: number; score: number;
item: StaticCommandItem | HydratedIndexItem; item: StaticCommandItem | HydratedIndexItem;
matches?: readonly FuseResultMatch[]; matches?: readonly FuseResultMatch[];
src/plugins/built-in/globalSearch/dynamicSearch.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import type { SvelteComponent } from 'svelte'; import type { SvelteComponent } from "svelte";
import type { HydratedIndexItem } from './indexing/types'; import type { HydratedIndexItem } from "./indexing/types";
export interface DynamicContentItem { export interface DynamicContentItem {
id: string; id: string;
src/plugins/built-in/globalSearch/highlightUtils.ts
+81 -42
View File
@@ -1,31 +1,40 @@
import type { FuseResultMatch, MatchIndices } from './types'; import type { FuseResultMatch, MatchIndices } from "./core/types";
/** /**
* Simple utility to remove HTML tags from a string. * Simple utility to remove HTML tags from a string.
*/ */
export function stripHtmlTags(html: string): string { export function stripHtmlTags(html: string): string {
if (!html) return ''; if (!html) return "";
return html.replace(/<[^>]*>/g, '').replace('\n', ' '); return html.replace(/<[^>]*>/g, "").replace("\n", " ");
} }
/** /**
* Removes HTML tags from a string, but preserves <span class="highlight"> tags. * Removes HTML tags from a string, but preserves <span class="highlight"> tags.
*/ */
export function stripHtmlButKeepHighlights(html: string): string { export function stripHtmlButKeepHighlights(html: string): string {
if (!html) return ''; if (!html) return "";
// Use a placeholder for highlight tags, strip others, then restore placeholders. // Use a placeholder for highlight tags, strip others, then restore placeholders.
const highlightOpenPlaceholder = '__HIGHLIGHT_OPEN__'; const highlightOpenPlaceholder = "__HIGHLIGHT_OPEN__";
const highlightClosePlaceholder = '__HIGHLIGHT_CLOSE__'; const highlightClosePlaceholder = "__HIGHLIGHT_CLOSE__";
let processed = html.replace(/<span class="highlight">/g, highlightOpenPlaceholder); let processed = html.replace(
/<span class="highlight">/g,
highlightOpenPlaceholder,
);
processed = processed.replace(/<\/span>/g, (match, offset, fullString) => { processed = processed.replace(/<\/span>/g, (match, offset, fullString) => {
// Only replace </span> if it likely corresponds to our highlight span // Only replace </span> if it likely corresponds to our highlight span
// This is imperfect but helps avoid replacing unrelated spans. // This is imperfect but helps avoid replacing unrelated spans.
// Look backwards for the nearest opening placeholder. // Look backwards for the nearest opening placeholder.
const lastPlaceholder = fullString.lastIndexOf(highlightOpenPlaceholder, offset); const lastPlaceholder = fullString.lastIndexOf(
highlightOpenPlaceholder,
offset,
);
if (lastPlaceholder !== -1) { if (lastPlaceholder !== -1) {
// Check if there's another opening tag between the placeholder and the closing span // Check if there's another opening tag between the placeholder and the closing span
const interveningContent = fullString.substring(lastPlaceholder + highlightOpenPlaceholder.length, offset); const interveningContent = fullString.substring(
lastPlaceholder + highlightOpenPlaceholder.length,
offset,
);
if (!/<span/i.test(interveningContent)) { if (!/<span/i.test(interveningContent)) {
return highlightClosePlaceholder; return highlightClosePlaceholder;
} }
@@ -34,11 +43,17 @@ export function stripHtmlButKeepHighlights(html: string): string {
}); });
// Strip all remaining HTML tags // Strip all remaining HTML tags
processed = processed.replace(/<[^>]*>/g, ''); processed = processed.replace(/<[^>]*>/g, "");
// Restore the highlight tags // Restore the highlight tags
processed = processed.replace(new RegExp(highlightOpenPlaceholder, 'g'), '<span class="highlight">'); processed = processed.replace(
processed = processed.replace(new RegExp(highlightClosePlaceholder, 'g'), '</span>'); new RegExp(highlightOpenPlaceholder, "g"),
'<span class="highlight">',
);
processed = processed.replace(
new RegExp(highlightClosePlaceholder, "g"),
"</span>",
);
return processed; return processed;
} }
@@ -46,18 +61,23 @@ export function stripHtmlButKeepHighlights(html: string): string {
export function highlightMatch( export function highlightMatch(
text: string, text: string,
term: string, term: string,
matches?: readonly FuseResultMatch[] matches?: readonly FuseResultMatch[],
): string { ): string {
if (!term.trim() || !matches || matches.length === 0) return text; if (!term.trim() || !matches || matches.length === 0) return text;
try { try {
// Find matches for the text field or allContent that contains the text // Find matches for the text field or allContent that contains the text
const fieldMatches = matches.find(match => const fieldMatches = matches.find(
match.key === 'text' || (match) =>
(match.key === 'allContent' && match.value?.includes(text)) match.key === "text" ||
(match.key === "allContent" && match.value?.includes(text)),
); );
if (!fieldMatches || !fieldMatches.indices || fieldMatches.indices.length === 0) { if (
!fieldMatches ||
!fieldMatches.indices ||
fieldMatches.indices.length === 0
) {
return text; return text;
} }
@@ -68,7 +88,7 @@ export function highlightMatch(
const start = indices[0]; const start = indices[0];
const end = indices[1]; const end = indices[1];
if (fieldMatches.key === 'allContent') { if (fieldMatches.key === "allContent") {
// Find where our text appears in the allContent // Find where our text appears in the allContent
const allContent = fieldMatches.value; const allContent = fieldMatches.value;
const textPos = allContent?.indexOf(text) ?? -1; const textPos = allContent?.indexOf(text) ?? -1;
@@ -82,7 +102,11 @@ export function highlightMatch(
// Only highlight if the match actually overlaps with our text field // Only highlight if the match actually overlaps with our text field
if (relEnd >= 0 && relStart < text.length) { if (relEnd >= 0 && relStart < text.length) {
// Mark the overlapping characters // Mark the overlapping characters
for (let i = Math.max(0, relStart); i <= Math.min(text.length - 1, relEnd); i++) { for (
let i = Math.max(0, relStart);
i <= Math.min(text.length - 1, relEnd);
i++
) {
highlightMap[i] = true; highlightMap[i] = true;
} }
} }
@@ -97,7 +121,7 @@ export function highlightMatch(
} }
}); });
let result = ''; let result = "";
let inHighlight = false; let inHighlight = false;
for (let i = 0; i < text.length; i++) { for (let i = 0; i < text.length; i++) {
@@ -105,7 +129,7 @@ export function highlightMatch(
result += '<span class="highlight">'; result += '<span class="highlight">';
inHighlight = true; inHighlight = true;
} else if (!highlightMap[i] && inHighlight) { } else if (!highlightMap[i] && inHighlight) {
result += '</span>'; result += "</span>";
inHighlight = false; inHighlight = false;
} }
@@ -113,12 +137,12 @@ export function highlightMatch(
} }
if (inHighlight) { if (inHighlight) {
result += '</span>'; result += "</span>";
} }
return result; return result;
} catch (e) { } catch (e) {
console.error('Error highlighting match:', e); console.error("Error highlighting match:", e);
return text; return text;
} }
} }
@@ -127,40 +151,52 @@ export function highlightMatch(
export function highlightSnippet( export function highlightSnippet(
content: string, content: string,
term: string, term: string,
matches?: readonly FuseResultMatch[] matches?: readonly FuseResultMatch[],
): string { ): string {
if (!content || !term.trim() || !matches || matches.length === 0) return content; if (!content || !term.trim() || !matches || matches.length === 0)
return content;
try { try {
// Find matches for content field or allContent that contains the content // Find matches for content field or allContent that contains the content
const contentMatches = matches.find(match => const contentMatches = matches.find(
match.key === 'content' || (match) =>
(match.key === 'allContent' && match.value?.includes(content)) match.key === "content" ||
(match.key === "allContent" && match.value?.includes(content)),
); );
if (!contentMatches || !contentMatches.indices || contentMatches.indices.length === 0) { if (
!contentMatches ||
!contentMatches.indices ||
contentMatches.indices.length === 0
) {
// No content matches, return plain content // No content matches, return plain content
return content.length > 100 ? content.substring(0, 100) + '...' : content; return content.length > 100 ? content.substring(0, 100) + "..." : content;
} }
// Find the match indices // Find the match indices
let allIndices: MatchIndices[] = contentMatches.indices as MatchIndices[]; let allIndices: MatchIndices[] = contentMatches.indices as MatchIndices[];
// If matching against allContent, adjust indices to be relative to content // If matching against allContent, adjust indices to be relative to content
if (contentMatches.key === 'allContent') { if (contentMatches.key === "allContent") {
const allContent = contentMatches.value; const allContent = contentMatches.value;
const contentPos = allContent?.indexOf(content) ?? -1; const contentPos = allContent?.indexOf(content) ?? -1;
if (contentPos >= 0) { if (contentPos >= 0) {
// Adjust indices to be relative to the content field // Adjust indices to be relative to the content field
allIndices = allIndices allIndices = allIndices
.map(indices => [indices[0] - contentPos, indices[1] - contentPos] as MatchIndices) .map(
.filter(indices => indices[1] >= 0 && indices[0] < content.length); (indices) =>
[
indices[0] - contentPos,
indices[1] - contentPos,
] as MatchIndices,
)
.filter((indices) => indices[1] >= 0 && indices[0] < content.length);
} }
} }
if (allIndices.length === 0) { if (allIndices.length === 0) {
return content.length > 100 ? content.substring(0, 100) + '...' : content; return content.length > 100 ? content.substring(0, 100) + "..." : content;
} }
// Find a good center point for our snippet (average of first match) // Find a good center point for our snippet (average of first match)
@@ -174,8 +210,8 @@ export function highlightSnippet(
// Create the basic snippet // Create the basic snippet
let snippet = content.substring(start, end); let snippet = content.substring(start, end);
if (start > 0) snippet = '...' + snippet; if (start > 0) snippet = "..." + snippet;
if (end < content.length) snippet += '...'; if (end < content.length) snippet += "...";
// Create a highlighting map for the snippet // Create a highlighting map for the snippet
const snippetLength = snippet.length; const snippetLength = snippet.length;
@@ -194,7 +230,10 @@ export function highlightSnippet(
// Adjust match indices to be relative to snippet // Adjust match indices to be relative to snippet
const snippetMatchStart = Math.max(0, matchStart - startOffset); const snippetMatchStart = Math.max(0, matchStart - startOffset);
const snippetMatchEnd = Math.min(snippetLength - 1, matchEnd - startOffset); const snippetMatchEnd = Math.min(
snippetLength - 1,
matchEnd - startOffset,
);
// Mark characters for highlighting // Mark characters for highlighting
for (let i = snippetMatchStart; i <= snippetMatchEnd; i++) { for (let i = snippetMatchStart; i <= snippetMatchEnd; i++) {
@@ -205,7 +244,7 @@ export function highlightSnippet(
}); });
// Build the highlighted snippet // Build the highlighted snippet
let result = ''; let result = "";
let inHighlight = false; let inHighlight = false;
for (let i = 0; i < snippetLength; i++) { for (let i = 0; i < snippetLength; i++) {
@@ -214,7 +253,7 @@ export function highlightSnippet(
result += '<span class="highlight">'; result += '<span class="highlight">';
inHighlight = true; inHighlight = true;
} else if (!highlightMap[i] && inHighlight) { } else if (!highlightMap[i] && inHighlight) {
result += '</span>'; result += "</span>";
inHighlight = false; inHighlight = false;
} }
@@ -224,12 +263,12 @@ export function highlightSnippet(
// Close highlight tag if we're still in one at the end // Close highlight tag if we're still in one at the end
if (inHighlight) { if (inHighlight) {
result += '</span>'; result += "</span>";
} }
return result; return result;
} catch (e) { } catch (e) {
console.error('Error highlighting snippet:', e); console.error("Error highlighting snippet:", e);
return content.length > 100 ? content.substring(0, 100) + '...' : content; return content.length > 100 ? content.substring(0, 100) + "..." : content;
} }
} }
src/plugins/built-in/globalSearch/index.ts
-189
View File
@@ -1,189 +0,0 @@
import type { Plugin } from '@/plugins/core/types';
import { BasePlugin } from '@/plugins/core/settings';
import { booleanSetting, defineSettings, Setting, stringSetting } from '@/plugins/core/settingsHelpers';
import renderSvelte from '@/interface/main';
import SearchBar from './SearchBar.svelte';
import styles from './styles.css?inline';
import { unmount } from 'svelte';
import { loadDynamicItems } from './dynamicSearch';
import { waitForElm } from '@/seqta/utils/waitForElm';
import { loadAllStoredItems, runIndexing } from './indexing/indexer';
const settings = defineSettings({
searchHotkey: stringSetting({
default: 'ctrl+k',
title: 'Search Hotkey',
description: 'Keyboard shortcut to open the search (cmd on Mac)',
}),
showRecentFirst: booleanSetting({
default: true,
title: 'Show Recent First',
description: 'Sort dynamic content by most recent first',
}),
transparencyEffects: booleanSetting({
default: true,
title: 'Transparency Effects',
description: 'Enable transparency effects for the search bar',
}),
runIndexingOnLoad: booleanSetting({
default: true,
title: 'Index on Page Load',
description: 'Run content indexing when SEQTA loads',
}),
});
class GlobalSearchPlugin extends BasePlugin<typeof settings> {
@Setting(settings.searchHotkey)
searchHotkey!: string;
@Setting(settings.showRecentFirst)
showRecentFirst!: boolean;
@Setting(settings.transparencyEffects)
transparencyEffects!: boolean;
@Setting(settings.runIndexingOnLoad)
runIndexingOnLoad!: boolean;
}
const settingsInstance = new GlobalSearchPlugin();
/* const createSampleDynamicData = (): DynamicContentItem[] => {
const sampleMessages = [
{
id: 'message_1',
text: 'Assignment Discussion',
category: 'messages',
contentType: 'message' as const,
icon: '\uea6e',
content: 'Hey everyone, I was wondering if anyone could help me with the Physics assignment on circular motion. I\'m stuck on question 3 about centripetal force.',
dateAdded: Date.now() - 1000 * 60 * 60 * 2,
action: () => console.log('Open message 1'),
keywords: ['John Smith', 'message', 'chat'],
metadata: { author: 'John Smith'}
},
];
const sampleCourses = [
{
id: 'course_1',
text: 'Physics 101',
category: 'courses',
contentType: 'course' as const,
icon: '\uea67',
content: 'An introduction to mechanics, thermodynamics, and wave phenomena.',
dateAdded: Date.now() - 1000 * 60 * 60 * 24 * 5, // 5 days ago
action: () => console.log('Open Physics course'),
keywords: ['Dr. Richard Feynman', 'course', 'class'],
metadata: { teacher: 'Dr. Richard Feynman' }
},
];
const sampleAssessments = [
{
id: 'assessment_1',
text: 'Physics Lab Report',
category: 'assessments',
contentType: 'assessment' as const,
icon: '\uebb3',
content: 'Complete a lab report on the pendulum experiment.',
dateAdded: Date.now() - 1000 * 60 * 60 * 24, // 1 day ago
action: () => console.log('Open Physics assessment'),
keywords: ['Physics 101', 'assessment', 'homework'],
metadata: { dueDate: Date.now() + 1000 * 60 * 60 * 24 * 3 }
},
];
return [...sampleMessages, ...sampleCourses, ...sampleAssessments];
}; */
// Update dynamic items directly from the indexer without conversion
const updateDynamicItemsFromIndex = async () => {
const indexedItems = await loadAllStoredItems();
loadDynamicItems(indexedItems);
console.log(`Loaded ${indexedItems.length} indexed items into search.`);
window.dispatchEvent(new CustomEvent('dynamic-items-updated'));
};
const globalSearchPlugin: Plugin<typeof settings> = {
id: 'global-search',
name: 'Global Search',
description: 'Quick search for everything in SEQTA',
version: '1.0.0',
settings: settingsInstance.settings,
disableToggle: true,
styles: styles,
run: async (api) => {
let app: any;
// Run initial indexing and update dynamic items
if (api.settings.runIndexingOnLoad) {
setTimeout(async () => {
await runIndexing();
await updateDynamicItemsFromIndex();
}, 2000); // Delay initial indexing to let page load
}
const mountSearchBar = (titleElement: Element) => {
if (titleElement.querySelector('.search-trigger')) {
return;
}
const searchButton = document.createElement('div');
searchButton.className = 'search-trigger';
searchButton.innerHTML = `
<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<circle cx="11" cy="11" r="8"></circle>
<line x1="21" y1="21" x2="16.65" y2="16.65"></line>
</svg>
<p>Quick search...</p>
<span style="margin-left: auto; display: flex; align-items: center; color: #777; font-size: 12px;">⌘K</span>
`;
titleElement.appendChild(searchButton);
const searchRoot = document.createElement('div');
document.body.appendChild(searchRoot);
const searchRootShadow = searchRoot.attachShadow({ mode: 'open' });
console.log('adding event listener to search button');
searchButton.addEventListener('click', () => {
console.log('search button clicked');
// @ts-ignore - Intentionally adding to window
window.setCommandPalleteOpen(true);
});
try {
app = renderSvelte(SearchBar, searchRootShadow, {
transparencyEffects: api.settings.transparencyEffects ? true : false,
showRecentFirst: api.settings.showRecentFirst
});
} catch (error) {
console.error('Error rendering Svelte component:', error);
}
}
const title = document.querySelector('#title');
if (title) {
mountSearchBar(title);
} else {
await waitForElm('#title', true, 100, 60);
mountSearchBar(document.querySelector('#title') as Element);
}
return () => {
const searchButton = document.querySelector('.search-trigger');
const searchRoot = document.querySelector('.global-search-root');
if (searchButton) searchButton.remove();
if (searchRoot) searchRoot.remove();
unmount(app);
};
}
};
export default globalSearchPlugin;
src/plugins/built-in/globalSearch/indexing/actions.ts
+2 -2
View File
@@ -1,4 +1,4 @@
import type { IndexItem } from './types'; import type { IndexItem } from "./types";
interface MessageMetadata { interface MessageMetadata {
messageId: number; messageId: number;
@@ -36,5 +36,5 @@ export const actionMap: Record<string, ActionHandler<any>> = {
} else { } else {
window.location.hash = `#?page=/assessments&id=${item.metadata.assessmentId}`; window.location.hash = `#?page=/assessments&id=${item.metadata.assessmentId}`;
} }
}) as ActionHandler<any> }) as ActionHandler<any>,
}; };
src/plugins/built-in/globalSearch/indexing/db.ts
+16 -12
View File
@@ -1,6 +1,6 @@
const DB_NAME = 'betterseqta-index'; const DB_NAME = "betterseqta-index";
const META_STORE = 'meta'; const META_STORE = "meta";
const VERSION_KEY = 'betterseqta-index-version'; const VERSION_KEY = "betterseqta-index-version";
let dbPromise: Promise<IDBDatabase> | null = null; let dbPromise: Promise<IDBDatabase> | null = null;
@@ -27,7 +27,7 @@ function openDB(): Promise<IDBDatabase> {
request = indexedDB.open(DB_NAME, currentVersion); request = indexedDB.open(DB_NAME, currentVersion);
} catch (e) { } catch (e) {
// If there's a version error, try to delete the database and start fresh // If there's a version error, try to delete the database and start fresh
console.warn('Database version conflict, recreating database...'); console.warn("Database version conflict, recreating database...");
indexedDB.deleteDatabase(DB_NAME); indexedDB.deleteDatabase(DB_NAME);
localStorage.removeItem(VERSION_KEY); localStorage.removeItem(VERSION_KEY);
request = indexedDB.open(DB_NAME, 1); request = indexedDB.open(DB_NAME, 1);
@@ -50,7 +50,7 @@ function openDB(): Promise<IDBDatabase> {
request.onsuccess = () => resolve(request.result); request.onsuccess = () => resolve(request.result);
request.onerror = () => { request.onerror = () => {
console.error('Error opening database:', request.error); console.error("Error opening database:", request.error);
// If there's an error, try to recover by deleting and recreating // If there's an error, try to recover by deleting and recreating
indexedDB.deleteDatabase(DB_NAME); indexedDB.deleteDatabase(DB_NAME);
localStorage.removeItem(VERSION_KEY); localStorage.removeItem(VERSION_KEY);
@@ -61,7 +61,7 @@ function openDB(): Promise<IDBDatabase> {
return dbPromise; return dbPromise;
} }
async function getStore(store: string, mode: IDBTransactionMode = 'readonly') { async function getStore(store: string, mode: IDBTransactionMode = "readonly") {
const db = await openDB(); const db = await openDB();
// Create store dynamically if needed // Create store dynamically if needed
@@ -82,7 +82,7 @@ function upgradeDB(newStore: string): Promise<void> {
// Close any existing connections // Close any existing connections
if (dbPromise) { if (dbPromise) {
dbPromise.then(db => db.close()); dbPromise.then((db) => db.close());
dbPromise = null; dbPromise = null;
} }
@@ -103,7 +103,7 @@ function upgradeDB(newStore: string): Promise<void> {
}; };
request.onerror = () => { request.onerror = () => {
console.error('Error upgrading database:', request.error); console.error("Error upgrading database:", request.error);
reject(request.error); reject(request.error);
}; };
}); });
@@ -137,9 +137,13 @@ export async function get(store: string, key: string): Promise<any> {
} }
} }
export async function put(store: string, value: any, key?: string): Promise<void> { export async function put(
store: string,
value: any,
key?: string,
): Promise<void> {
try { try {
const s = await getStore(store, 'readwrite'); const s = await getStore(store, "readwrite");
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const req = key ? s.put(value, key) : s.put(value); const req = key ? s.put(value, key) : s.put(value);
req.onsuccess = () => resolve(); req.onsuccess = () => resolve();
@@ -153,7 +157,7 @@ export async function put(store: string, value: any, key?: string): Promise<void
export async function remove(store: string, key: string): Promise<void> { export async function remove(store: string, key: string): Promise<void> {
try { try {
const s = await getStore(store, 'readwrite'); const s = await getStore(store, "readwrite");
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const req = s.delete(key); const req = s.delete(key);
req.onsuccess = () => resolve(); req.onsuccess = () => resolve();
@@ -167,7 +171,7 @@ export async function remove(store: string, key: string): Promise<void> {
export async function clear(store: string): Promise<void> { export async function clear(store: string): Promise<void> {
try { try {
const s = await getStore(store, 'readwrite'); const s = await getStore(store, "readwrite");
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const req = s.clear(); const req = s.clear();
req.onsuccess = () => resolve(); req.onsuccess = () => resolve();
src/plugins/built-in/globalSearch/indexing/indexer.ts
+113 -24
View File
@@ -1,27 +1,104 @@
import { clear, getAll, put, remove } from './db'; import { clear, getAll, put, remove } from "./db";
import { jobs } from './jobs'; import { jobs } from "./jobs";
import { renderComponentMap } from './renderComponents'; import { renderComponentMap } from "./renderComponents";
import type { HydratedIndexItem, IndexItem, Job, JobContext } from './types'; import type { HydratedIndexItem, IndexItem, Job, JobContext } from "./types";
import { processItems } from '../vectorSearch'; import { EmbeddingIndex, getEmbedding, initializeModel } from "client-vector-search";
const META_STORE = 'meta'; const META_STORE = "meta";
const LOCK_KEY = 'bsq-indexer-lock'; const LOCK_KEY = "bsq-indexer-lock";
const HEARTBEAT_INTERVAL = 10000; const HEARTBEAT_INTERVAL = 10000;
const LOCK_TIMEOUT = 20000; const LOCK_TIMEOUT = 20000;
let heartbeatTimer: ReturnType<typeof setInterval> | null = null; let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
async function initVectorSearch() {
if (isInitialized) return;
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
// Load existing items from IndexedDB
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach((item) => vectorIndex!.add(item));
console.debug("Vector index loaded from IndexedDB");
}
isInitialized = true;
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
}
}
async function vectorizeItem(
item: HydratedIndexItem,
): Promise<HydratedIndexItem & { embedding: number[] }> {
const textToEmbed = [
item.text,
item.content,
item.category,
item.metadata?.author,
item.metadata?.subject,
]
.filter(Boolean)
.join(" ");
const embedding = await getEmbedding(textToEmbed);
return { ...item, embedding };
}
async function processItems(items: HydratedIndexItem[]) {
if (!vectorIndex) await initVectorSearch();
const unprocessedItems = items.filter((item) => {
try {
return !vectorIndex!.get({ id: item.id });
} catch {
return true;
}
});
if (unprocessedItems.length === 0) {
console.debug("No new items to vectorize");
return;
}
console.debug(`Vectorizing ${unprocessedItems.length} new items...`);
// Process in batches to avoid UI freeze
const BATCH_SIZE = 5;
for (let i = 0; i < unprocessedItems.length; i += BATCH_SIZE) {
const batch = unprocessedItems.slice(i, i + BATCH_SIZE);
const vectorized = await Promise.all(batch.map(vectorizeItem));
for (const item of vectorized) {
vectorIndex!.add(item);
}
// Save periodically to avoid losing progress
await vectorIndex!.saveIndex("indexedDB");
// Log progress
console.debug(
`Vectorized ${Math.min(i + BATCH_SIZE, unprocessedItems.length)}/${unprocessedItems.length} items`,
);
}
}
function shouldRun(job: Job, lastRun?: number): boolean { function shouldRun(job: Job, lastRun?: number): boolean {
const now = Date.now(); const now = Date.now();
if (job.frequency === 'pageLoad') return true; if (job.frequency === "pageLoad") return true;
if (!lastRun) return true; if (!lastRun) return true;
if (job.frequency.type === 'interval') { if (job.frequency.type === "interval") {
return now - lastRun >= job.frequency.ms; return now - lastRun >= job.frequency.ms;
} }
if (job.frequency.type === 'expiry') { if (job.frequency.type === "expiry") {
return now - lastRun >= job.frequency.afterMs; return now - lastRun >= job.frequency.afterMs;
} }
@@ -29,7 +106,7 @@ function shouldRun(job: Job, lastRun?: number): boolean {
} }
function getLastRunMeta(jobId: string): Promise<number | undefined> { function getLastRunMeta(jobId: string): Promise<number | undefined> {
return getAll(META_STORE).then(metaItems => { return getAll(META_STORE).then((metaItems) => {
const match = metaItems.find((m: any) => m.jobId === jobId); const match = metaItems.find((m: any) => m.jobId === jobId);
return match?.lastRun; return match?.lastRun;
}); });
@@ -40,7 +117,7 @@ async function updateLastRunMeta(jobId: string): Promise<void> {
} }
function shouldIndex(): boolean { function shouldIndex(): boolean {
const last = parseInt(localStorage.getItem(LOCK_KEY) || '0', 10); const last = parseInt(localStorage.getItem(LOCK_KEY) || "0", 10);
return isNaN(last) || Date.now() - last > LOCK_TIMEOUT; return isNaN(last) || Date.now() - last > LOCK_TIMEOUT;
} }
@@ -57,8 +134,8 @@ function stopHeartbeat() {
} }
function dispatchProgress(completed: number, total: number, indexing: boolean) { function dispatchProgress(completed: number, total: number, indexing: boolean) {
const event = new CustomEvent('indexing-progress', { const event = new CustomEvent("indexing-progress", {
detail: { completed, total, indexing } detail: { completed, total, indexing },
}); });
window.dispatchEvent(event); window.dispatchEvent(event);
} }
@@ -84,12 +161,15 @@ export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
export async function runIndexing(): Promise<void> { export async function runIndexing(): Promise<void> {
if (!shouldIndex()) { if (!shouldIndex()) {
console.debug('%c[Indexer] Skipping indexing (another tab has the lock)', 'color: gray'); console.debug(
"%c[Indexer] Skipping indexing (another tab has the lock)",
"color: gray",
);
return; return;
} }
startHeartbeat(); startHeartbeat();
console.debug('%c[Indexer] Starting indexing...', 'color: green'); console.debug("%c[Indexer] Starting indexing...", "color: green");
const jobIds = Object.keys(jobs); const jobIds = Object.keys(jobs);
let completedJobs = 0; let completedJobs = 0;
@@ -102,7 +182,10 @@ export async function runIndexing(): Promise<void> {
const lastRun = await getLastRunMeta(jobId); const lastRun = await getLastRunMeta(jobId);
if (!shouldRun(job, lastRun)) { if (!shouldRun(job, lastRun)) {
console.debug(`%c[Indexer] Skipping job "${jobId}" (not due)`, 'color: gray'); console.debug(
`%c[Indexer] Skipping job "${jobId}" (not due)`,
"color: gray",
);
completedJobs++; completedJobs++;
dispatchProgress(completedJobs, jobIds.length, true); dispatchProgress(completedJobs, jobIds.length, true);
continue; continue;
@@ -111,7 +194,7 @@ export async function runIndexing(): Promise<void> {
const getStoredItems = async () => await getAll(jobId); const getStoredItems = async () => await getAll(jobId);
const setStoredItems = async (items: IndexItem[]) => { const setStoredItems = async (items: IndexItem[]) => {
await clear(jobId); await clear(jobId);
await Promise.all(items.map(i => put(jobId, i, i.id))); await Promise.all(items.map((i) => put(jobId, i, i.id)));
}; };
const addItem = async (item: IndexItem) => { const addItem = async (item: IndexItem) => {
await put(jobId, item, item.id); await put(jobId, item, item.id);
@@ -127,7 +210,7 @@ export async function runIndexing(): Promise<void> {
removeItem, removeItem,
}; };
console.debug(`%c[Indexer] Running job "${jobId}"...`, 'color: #4ea1ff'); console.debug(`%c[Indexer] Running job "${jobId}"...`, "color: #4ea1ff");
try { try {
const newItems = await job.run(ctx); const newItems = await job.run(ctx);
@@ -140,15 +223,18 @@ export async function runIndexing(): Promise<void> {
await updateLastRunMeta(jobId); await updateLastRunMeta(jobId);
// Add to our collection of new items for vector processing // Add to our collection of new items for vector processing
const hydratedItems = merged.map(item => ({ const hydratedItems = merged.map((item) => ({
...item, ...item,
renderComponent: renderComponentMap[job.renderComponentId] renderComponent: renderComponentMap[job.renderComponentId],
})); }));
allNewItems.push(...hydratedItems); allNewItems.push(...hydratedItems);
console.debug(`%c[Indexer] ✅ ${job.label}: ${newItems.length} items indexed`, 'color: #00c46f'); console.debug(
`%c[Indexer] ✅ ${job.label}: ${newItems.length} items indexed`,
"color: #00c46f",
);
} catch (err) { } catch (err) {
console.debug(`%c[Indexer] ❌ ${job.label} failed:`, 'color: red'); console.debug(`%c[Indexer] ❌ ${job.label} failed:`, "color: red");
console.error(err); console.error(err);
} }
@@ -158,7 +244,10 @@ export async function runIndexing(): Promise<void> {
// Process all new items through vector search // Process all new items through vector search
if (allNewItems.length > 0) { if (allNewItems.length > 0) {
console.debug(`%c[Indexer] Processing ${allNewItems.length} items for vector search...`, 'color: #4ea1ff'); console.debug(
`%c[Indexer] Processing ${allNewItems.length} items for vector search...`,
"color: #4ea1ff",
);
await processItems(allNewItems); await processItems(allNewItems);
} }
src/plugins/built-in/globalSearch/indexing/jobs.ts
+79 -61
View File
@@ -1,9 +1,9 @@
import type { Job } from './types'; import type { Job } from "./types";
import type { IndexItem } from './types'; import type { IndexItem } from "./types";
interface MessageNotification { interface MessageNotification {
notificationID: number; notificationID: number;
type: 'message'; type: "message";
message: { message: {
subtitle: string; subtitle: string;
messageID: number; messageID: number;
@@ -14,7 +14,7 @@ interface MessageNotification {
interface AssessmentNotification { interface AssessmentNotification {
notificationID: number; notificationID: number;
type: 'coneqtassessments'; type: "coneqtassessments";
coneqtAssessments: { coneqtAssessments: {
programmeID: number; programmeID: number;
metaclassID: number; metaclassID: number;
@@ -79,16 +79,21 @@ interface MessageContentResponse {
// Helper to strip HTML tags from text // Helper to strip HTML tags from text
function stripHtmlTags(html: string): string { function stripHtmlTags(html: string): string {
return html.replace(/<[^>]*>/g, ''); return html.replace(/<[^>]*>/g, "");
} }
// Helper to fetch messages with pagination // Helper to fetch messages with pagination
async function fetchMessages(offset: number = 0, limit: number = 100): Promise<MessageListResponse> { async function fetchMessages(
const response = await fetch(`${location.origin}/seqta/student/load/message`, { offset: number = 0,
method: 'POST', limit: number = 100,
credentials: 'include', ): Promise<MessageListResponse> {
const response = await fetch(
`${location.origin}/seqta/student/load/message`,
{
method: "POST",
credentials: "include",
headers: { headers: {
'Content-Type': 'application/json; charset=utf-8' "Content-Type": "application/json; charset=utf-8",
}, },
body: JSON.stringify({ body: JSON.stringify({
searchValue: "", searchValue: "",
@@ -98,26 +103,32 @@ async function fetchMessages(offset: number = 0, limit: number = 100): Promise<M
label: "inbox", label: "inbox",
offset, offset,
limit, limit,
datetimeUntil: null datetimeUntil: null,
}) }),
}); },
);
return await response.json(); return await response.json();
} }
// Helper to fetch message content // Helper to fetch message content
async function fetchMessageContent(messageId: number): Promise<MessageContentResponse> { async function fetchMessageContent(
const response = await fetch(`${location.origin}/seqta/student/load/message`, { messageId: number,
method: 'POST', ): Promise<MessageContentResponse> {
credentials: 'include', const response = await fetch(
`${location.origin}/seqta/student/load/message`,
{
method: "POST",
credentials: "include",
headers: { headers: {
'Content-Type': 'application/json; charset=utf-8' "Content-Type": "application/json; charset=utf-8",
}, },
body: JSON.stringify({ body: JSON.stringify({
action: "message", action: "message",
id: messageId id: messageId,
}) }),
}); },
);
return await response.json(); return await response.json();
} }
@@ -125,14 +136,14 @@ async function fetchMessageContent(messageId: number): Promise<MessageContentRes
// Helper to fetch notifications // Helper to fetch notifications
async function fetchNotifications(): Promise<Notification[]> { async function fetchNotifications(): Promise<Notification[]> {
const response = await fetch(`${location.origin}/seqta/student/heartbeat?`, { const response = await fetch(`${location.origin}/seqta/student/heartbeat?`, {
method: 'POST', method: "POST",
headers: { headers: {
'Content-Type': 'application/json; charset=utf-8' "Content-Type": "application/json; charset=utf-8",
}, },
body: JSON.stringify({ body: JSON.stringify({
timestamp: "1970-01-01 00:00:00.0", timestamp: "1970-01-01 00:00:00.0",
hash: "#?page=/notifications", hash: "#?page=/notifications",
}) }),
}); });
const json = await response.json(); const json = await response.json();
@@ -141,15 +152,15 @@ async function fetchNotifications(): Promise<Notification[]> {
export const jobs: Record<string, Job> = { export const jobs: Record<string, Job> = {
messages: { messages: {
id: 'messages', id: "messages",
label: 'Messages', label: "Messages",
renderComponentId: 'message', renderComponentId: "message",
frequency: { type: 'expiry', afterMs: 1000 * 60 * 5 }, // every 5 minutes frequency: { type: "expiry", afterMs: 1000 * 60 * 5 }, // every 5 minutes
run: async (ctx) => { run: async (ctx) => {
// Get existing items first // Get existing items first
const existing = await ctx.getStoredItems(); const existing = await ctx.getStoredItems();
const existingIds = new Set(existing.map(i => i.id)); const existingIds = new Set(existing.map((i) => i.id));
const newItems: IndexItem[] = []; const newItems: IndexItem[] = [];
let offset = 0; let offset = 0;
const limit = 100; const limit = 100;
@@ -162,7 +173,7 @@ export const jobs: Record<string, Job> = {
const response = await fetchMessages(offset, limit); const response = await fetchMessages(offset, limit);
if (response.status !== "200") { if (response.status !== "200") {
console.error('Failed to fetch messages:', response); console.error("Failed to fetch messages:", response);
break; break;
} }
@@ -178,7 +189,9 @@ export const jobs: Record<string, Job> = {
consecutiveExisting++; consecutiveExisting++;
// If we've found 20 consecutive existing messages, assume we've caught up // If we've found 20 consecutive existing messages, assume we've caught up
if (consecutiveExisting >= 20) { if (consecutiveExisting >= 20) {
console.debug('[Messages Job] Found 20 consecutive existing messages, stopping fetch'); console.debug(
"[Messages Job] Found 20 consecutive existing messages, stopping fetch",
);
hasMore = false; hasMore = false;
break; break;
} }
@@ -193,7 +206,10 @@ export const jobs: Record<string, Job> = {
const contentResponse = await fetchMessageContent(message.id); const contentResponse = await fetchMessageContent(message.id);
if (contentResponse.status !== "200") { if (contentResponse.status !== "200") {
console.error('Failed to fetch message content:', contentResponse); console.error(
"Failed to fetch message content:",
contentResponse,
);
continue; continue;
} }
@@ -202,7 +218,7 @@ export const jobs: Record<string, Job> = {
newItems.push({ newItems.push({
id, id,
text: message.subject, text: message.subject,
category: 'messages', category: "messages",
content: `From: ${message.sender}\n\n${content}`, content: `From: ${message.sender}\n\n${content}`,
dateAdded: new Date(message.date).getTime(), dateAdded: new Date(message.date).getTime(),
metadata: { metadata: {
@@ -213,28 +229,28 @@ export const jobs: Record<string, Job> = {
timestamp: message.date, timestamp: message.date,
hasAttachments: message.attachments, hasAttachments: message.attachments,
attachmentCount: message.attachmentCount, attachmentCount: message.attachmentCount,
read: message.read === 1 read: message.read === 1,
}, },
actionId: 'message', actionId: "message",
renderComponentId: 'message' renderComponentId: "message",
}); });
// Add to existingIds as we process to prevent duplicates in the same run // Add to existingIds as we process to prevent duplicates in the same run
existingIds.add(id); existingIds.add(id);
} catch (error) { } catch (error) {
console.error('Error fetching message content:', error); console.error("Error fetching message content:", error);
continue; continue;
} }
} }
offset += limit; offset += limit;
} catch (error) { } catch (error) {
console.error('Error fetching messages:', error); console.error("Error fetching messages:", error);
break; break;
} }
// Small delay to avoid overwhelming the server // Small delay to avoid overwhelming the server
await new Promise(resolve => setTimeout(resolve, 100)); await new Promise((resolve) => setTimeout(resolve, 100));
} }
console.debug(`[Messages Job] Found ${newItems.length} new messages`); console.debug(`[Messages Job] Found ${newItems.length} new messages`);
@@ -243,38 +259,40 @@ export const jobs: Record<string, Job> = {
purge: (items) => { purge: (items) => {
// Keep messages from the last 30 days // Keep messages from the last 30 days
const cutoff = Date.now() - (30 * 24 * 60 * 60 * 1000); const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000;
return items.filter(i => i.dateAdded >= cutoff); return items.filter((i) => i.dateAdded >= cutoff);
} },
}, },
assessments: { assessments: {
id: 'assessments', id: "assessments",
label: 'Assessments', label: "Assessments",
renderComponentId: 'assessment', renderComponentId: "assessment",
frequency: { type: 'expiry', afterMs: 1000 * 60 * 15 }, // every 15 minutes frequency: { type: "expiry", afterMs: 1000 * 60 * 15 }, // every 15 minutes
run: async (ctx) => { run: async (ctx) => {
const notifications = await fetchNotifications(); const notifications = await fetchNotifications();
const assessmentNotifications = notifications.filter((n): n is (MessageNotification | AssessmentNotification) => const assessmentNotifications = notifications.filter(
n.type === 'coneqtassessments' || (n): n is MessageNotification | AssessmentNotification =>
(n.type === 'message' && n.message.title.toLowerCase().includes('assessment')) n.type === "coneqtassessments" ||
(n.type === "message" &&
n.message.title.toLowerCase().includes("assessment")),
); );
const existing = await ctx.getStoredItems(); const existing = await ctx.getStoredItems();
const existingIds = new Set(existing.map(i => i.id)); const existingIds = new Set(existing.map((i) => i.id));
const newItems: IndexItem[] = []; const newItems: IndexItem[] = [];
for (const notification of assessmentNotifications) { for (const notification of assessmentNotifications) {
const id = notification.notificationID.toString(); const id = notification.notificationID.toString();
if (existingIds.has(id)) continue; if (existingIds.has(id)) continue;
if (notification.type === 'coneqtassessments') { if (notification.type === "coneqtassessments") {
const { coneqtAssessments: assessment } = notification; const { coneqtAssessments: assessment } = notification;
newItems.push({ newItems.push({
id, id,
text: assessment.title, text: assessment.title,
category: 'assessments', category: "assessments",
content: assessment.subtitle, content: assessment.subtitle,
dateAdded: new Date(notification.timestamp).getTime(), dateAdded: new Date(notification.timestamp).getTime(),
metadata: { metadata: {
@@ -283,10 +301,10 @@ export const jobs: Record<string, Job> = {
term: assessment.term, term: assessment.term,
programmeId: assessment.programmeID, programmeId: assessment.programmeID,
metaclassId: assessment.metaclassID, metaclassId: assessment.metaclassID,
timestamp: notification.timestamp timestamp: notification.timestamp,
}, },
actionId: 'assessment', actionId: "assessment",
renderComponentId: 'assessment' renderComponentId: "assessment",
}); });
} else { } else {
// Handle message-based assessments // Handle message-based assessments
@@ -294,17 +312,17 @@ export const jobs: Record<string, Job> = {
newItems.push({ newItems.push({
id, id,
text: message.title, text: message.title,
category: 'assessments', category: "assessments",
content: `From: ${message.subtitle}`, content: `From: ${message.subtitle}`,
dateAdded: new Date(notification.timestamp).getTime(), dateAdded: new Date(notification.timestamp).getTime(),
metadata: { metadata: {
messageId: message.messageID, messageId: message.messageID,
author: message.subtitle, author: message.subtitle,
timestamp: notification.timestamp, timestamp: notification.timestamp,
isMessageBased: true isMessageBased: true,
}, },
actionId: 'assessment', actionId: "assessment",
renderComponentId: 'assessment' renderComponentId: "assessment",
}); });
} }
} }
@@ -321,8 +339,8 @@ export const jobs: Record<string, Job> = {
date.setMinutes(0); date.setMinutes(0);
date.setSeconds(0); date.setSeconds(0);
const cutoff = date.getTime(); const cutoff = date.getTime();
return items.filter(i => i.dateAdded >= cutoff); return items.filter((i) => i.dateAdded >= cutoff);
} },
}, },
// We can add more job types here as needed: // We can add more job types here as needed:
src/plugins/built-in/globalSearch/indexing/renderComponents.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import type { SvelteComponent } from 'svelte'; import type { SvelteComponent } from "svelte";
import AssessmentComponent from '../components/AssessmentItem.svelte'; import AssessmentComponent from "../components/AssessmentItem.svelte";
// import other components as needed // import other components as needed
export const renderComponentMap: Record<string, typeof SvelteComponent> = { export const renderComponentMap: Record<string, typeof SvelteComponent> = {
src/plugins/built-in/globalSearch/indexing/types.ts
+4 -4
View File
@@ -1,4 +1,4 @@
import type { SvelteComponent } from 'svelte'; import type { SvelteComponent } from "svelte";
export interface IndexItem { export interface IndexItem {
id: string; id: string;
@@ -16,9 +16,9 @@ export interface HydratedIndexItem extends IndexItem {
} }
export type Frequency = export type Frequency =
| 'pageLoad' | "pageLoad"
| { type: 'interval'; ms: number } | { type: "interval"; ms: number }
| { type: 'expiry'; afterMs: number }; | { type: "expiry"; afterMs: number };
export interface JobContext { export interface JobContext {
getStoredItems: () => Promise<IndexItem[]>; getStoredItems: () => Promise<IndexItem[]>;
src/plugins/built-in/globalSearch/indexing/worker/vectorWorker.ts
+420
View File
@@ -0,0 +1,420 @@
import {
getEmbedding,
EmbeddingIndex,
initializeModel,
} from "client-vector-search";
import type { HydratedIndexItem } from "../types";
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
let currentAbortController: AbortController | null = null;
async function initWorker() {
// Avoid re-initialization
if (isInitialized) {
console.debug("Vector worker already initialized.");
return;
}
console.debug("Initializing vector worker...");
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach((item) => vectorIndex!.add(item));
console.debug(
`Vector index loaded ${stored.length} items from IndexedDB.`,
);
} else {
console.debug("No existing vector index found in IndexedDB.");
}
isInitialized = true;
console.debug("Vector worker initialized successfully.");
} catch (e) {
console.error("Failed to initialize vector worker:", e);
// Set as initialized even on error to prevent retries, but index will be null
isInitialized = true;
vectorIndex = null; // Ensure index is null on error
}
}
async function vectorizeItem(
item: HydratedIndexItem,
): Promise<(HydratedIndexItem & { embedding: number[] }) | null> {
// Simplified for brevity - assumes embedding function doesn't need cancellation signal
try {
const textToEmbed = [
item.text,
item.content,
item.category,
item.metadata?.author,
item.metadata?.subject,
]
.filter(Boolean)
.join(" ");
const embedding = await getEmbedding(textToEmbed);
return { ...item, embedding };
} catch (error) {
console.error(`Error vectorizing item ${item.id}:`, error);
return null; // Return null if vectorization fails for an item
}
}
async function processItems(items: HydratedIndexItem[], signal: AbortSignal) {
console.debug("Worker received process request.");
if (!vectorIndex) {
console.warn(
"Processing requested but vector index not ready. Attempting init.",
);
await initWorker(); // Attempt initialization if not ready
if (!vectorIndex) {
// Check again after attempt
self.postMessage({
type: "progress",
data: {
status: "error",
message:
"Vector index not available for processing after init attempt.",
},
});
return;
}
}
// Find items we haven't processed yet by checking against the index instance
const unprocessedItems = items.filter((item) => {
if (signal.aborted) return false; // Check cancellation during filtering
try {
// Check if the item ID already exists in the index (loaded or added)
return !vectorIndex!.get({ id: item.id });
} catch (e) {
// If get throws (e.g., item not found), it means it's unprocessed
return true;
}
});
if (signal.aborted) {
console.debug("Processing cancelled before starting.");
self.postMessage({
type: "progress",
data: {
status: "cancelled",
message: "Processing cancelled before start",
},
});
return;
}
if (unprocessedItems.length === 0) {
console.debug("No new items to process.");
self.postMessage({
type: "progress",
data: { status: "complete", message: "No new items to process" },
});
return;
}
console.debug(`Starting processing of ${unprocessedItems.length} items.`);
self.postMessage({
type: "progress",
data: {
status: "started",
total: unprocessedItems.length,
processed: 0,
},
});
const BATCH_SIZE = 5;
let processedCount = 0;
for (let i = 0; i < unprocessedItems.length; i += BATCH_SIZE) {
if (signal.aborted) {
console.debug("Processing cancelled during batching.");
self.postMessage({
type: "progress",
data: {
status: "cancelled",
message: "Processing cancelled during batching",
},
});
return;
}
const batch = unprocessedItems.slice(i, i + BATCH_SIZE);
// Vectorize batch
const vectorizationResults = await Promise.all(batch.map(vectorizeItem));
const successfullyVectorized = vectorizationResults.filter(
(result) => result !== null,
) as (HydratedIndexItem & { embedding: number[] })[];
if (signal.aborted) {
console.debug("Processing cancelled after vectorization batch.");
self.postMessage({
type: "progress",
data: {
status: "cancelled",
message: "Processing cancelled after vectorization",
},
});
return;
}
// Add successfully vectorized items to index
if (successfullyVectorized.length > 0) {
try {
successfullyVectorized.forEach((item) => vectorIndex!.add(item));
} catch (e) {
console.error("Error adding batch to index:", e);
self.postMessage({
type: "progress",
data: { status: "error", message: `Error adding to index: ${e}` },
});
// Decide whether to continue or stop on error
// return; // Example: Stop processing if adding fails
}
}
if (signal.aborted) {
console.debug("Processing cancelled before saving batch.");
self.postMessage({
type: "progress",
data: {
status: "cancelled",
message: "Processing cancelled before saving",
},
});
return;
}
// Save index after processing the batch
try {
await vectorIndex!.saveIndex("indexedDB");
console.debug(`Saved index after processing batch ${i / BATCH_SIZE + 1}`);
} catch (e) {
console.error("Error saving index batch:", e);
self.postMessage({
type: "progress",
data: { status: "error", message: `Error saving index batch: ${e}` },
});
// Continue processing next batch even if saving failed? Or stop?
// return; // Example: Stop if saving fails
}
processedCount = Math.min(i + BATCH_SIZE, unprocessedItems.length);
// Report progress
self.postMessage({
type: "progress",
data: {
status: "processing",
total: unprocessedItems.length,
processed: processedCount,
},
});
// Yield control briefly to allow other messages (like cancellation) to be processed
await new Promise((resolve) => setTimeout(resolve, 0));
}
if (!signal.aborted) {
console.debug("Processing completed successfully.");
self.postMessage({
type: "progress",
data: { status: "complete", message: "All items processed successfully" },
});
} else {
console.debug("Processing completed, but was cancelled.");
// No need to send 'cancelled' again if already sent during batching
// self.postMessage({ type: 'progress', data: { status: 'cancelled', message: 'Processing finished but was cancelled' }});
}
}
async function search(
query: string,
topK: number,
signal: AbortSignal,
messageId: string,
) {
console.debug(
`Worker received search request (ID: ${messageId}): "${query}"`,
);
if (!vectorIndex) {
console.warn(
`Search (ID: ${messageId}) requested but vector index not ready. Attempting init.`,
);
await initWorker(); // Attempt initialization
// Re-check after waiting/init attempt
if (!vectorIndex) {
console.error(
`Search (ID: ${messageId}) failed: Vector index unavailable after init attempt.`,
);
self.postMessage({
type: "searchError",
data: { messageId, error: "Vector index not available." },
});
return;
}
console.debug(
`Vector index ready after init for search (ID: ${messageId}).`,
);
}
if (signal.aborted) {
console.debug(`Search (ID: ${messageId}) cancelled before starting.`);
self.postMessage({ type: "searchCancelled", data: { messageId } });
return;
}
try {
console.debug(`Getting embedding for query (ID: ${messageId})...`);
const queryEmbedding = await getEmbedding(query);
if (signal.aborted) {
console.debug(`Search (ID: ${messageId}) cancelled after embedding.`);
self.postMessage({ type: "searchCancelled", data: { messageId } });
return;
}
console.debug(`Performing vector search (ID: ${messageId})...`);
// Await the search and let TypeScript infer the type
const results = await vectorIndex!.search(queryEmbedding, {
topK,
useStorage: "indexedDB", // Ensure we search the stored index
});
console.debug(
`Vector search (ID: ${messageId}) completed with ${results.length} results.`,
);
if (signal.aborted) {
console.debug(
`Search (ID: ${messageId}) cancelled after search completed, discarding results.`,
);
self.postMessage({ type: "searchCancelled", data: { messageId } });
return;
}
// Post results back to the main thread
self.postMessage({ type: "searchResults", data: { messageId, results } });
} catch (error) {
console.error(`Vector search error in worker (ID: ${messageId}):`, error);
// Ensure signal isn't checked *after* an error occurred before posting error message
if (!signal.aborted) {
// Only post error if not cancelled
self.postMessage({
type: "searchError",
data: {
messageId,
error: error instanceof Error ? error.message : String(error),
},
});
} else {
console.debug(
`Search (ID: ${messageId}) encountered error but was cancelled, suppressing error message.`,
);
self.postMessage({ type: "searchCancelled", data: { messageId } }); // Still notify of cancellation
}
}
}
// Handle messages from the main thread
self.addEventListener("message", async (e) => {
// Make sure data and type exist
if (!e.data || !e.data.type) {
console.warn("Worker received message with no data or type.");
return;
}
const { type, data, messageId } = e.data; // messageId used for requests needing response/cancellation tracking
// Cancel previous long-running operation (process or search) if a new one starts
if (type === "process" || type === "search") {
if (currentAbortController) {
console.debug(
`Worker cancelling previous operation due to new '${type}' request.`,
);
currentAbortController.abort(`New '${type}' operation requested`);
}
currentAbortController = new AbortController();
console.debug(`Worker starting new '${type}' operation.`);
}
// Use the signal from the *current* controller for the task being started
const signal = currentAbortController?.signal;
switch (type) {
case "process":
if (signal && data?.items) {
await processItems(data.items, signal);
} else if (!signal) {
console.error(
"Process message received but no abort signal available.",
);
} else if (!data?.items) {
console.error("Process message received without 'items' data.");
self.postMessage({
type: "progress",
data: {
status: "error",
message: "Process command received without items.",
},
});
}
break;
case "search":
if (signal && messageId && typeof data?.query === "string") {
await search(data.query, data.topK ?? 10, signal, messageId);
} else {
const errorReason = !signal
? "Missing signal"
: !messageId
? "Missing messageId"
: "Missing or invalid query";
console.error(`Search message received invalid: ${errorReason}.`, {
data,
messageId,
signalExists: !!signal,
});
// Send an error back if messageId exists
if (messageId) {
self.postMessage({
type: "searchError",
data: { messageId, error: `Worker internal error: ${errorReason}` },
});
}
}
break;
case "init":
// Init should not be cancellable in the same way, it's foundational
// Check if already initialized before potentially running it again
if (!isInitialized) {
await initWorker();
self.postMessage({ type: "ready" }); // Signal ready *after* init attempt
} else {
console.debug("Received init message, but worker already initialized.");
self.postMessage({ type: "ready" }); // Signal ready anyway
}
break;
// No explicit 'cancel' case needed as new tasks auto-cancel previous ones
default:
console.warn("Unknown message type received by vector worker:", type);
}
});
// Initial check or trigger for initialization when the worker starts
initWorker()
.then(() => {
self.postMessage({ type: "ready" });
})
.catch((err) => {
console.error("Initial worker initialization failed:", err);
// Still need to signal readiness, perhaps with an error state?
// Or rely on the first 'process' or 'search' to retry init.
// For now, just signal ready, but the index might be null.
self.postMessage({ type: "ready" });
});
src/plugins/built-in/globalSearch/indexing/worker/vectorWorkerManager.ts
+132
View File
@@ -0,0 +1,132 @@
import type { HydratedIndexItem } from '../types';
import vectorWorker from './vectorWorker.ts?inlineWorker';
import type { SearchResult } from 'client-vector-search';
export type ProgressCallback = (data: {
status: 'started' | 'processing' | 'complete';
total?: number;
processed?: number;
message?: string;
}) => void;
export class VectorWorkerManager {
private static instance: VectorWorkerManager;
private worker: Worker | null = null;
private isInitialized = false;
private progressCallback: ProgressCallback | null = null;
private searchPromises = new Map<string, { resolve: (value: SearchResult[]) => void, reject: (reason?: any) => void }>();
private constructor() {}
static getInstance(): VectorWorkerManager {
if (!VectorWorkerManager.instance) {
VectorWorkerManager.instance = new VectorWorkerManager();
}
return VectorWorkerManager.instance;
}
async init() {
if (this.isInitialized) return;
// Create the worker
this.worker = vectorWorker();
// Set up message handling
this.worker.addEventListener('message', (e) => {
const { type, data } = e.data;
console.log(e);
switch (type) {
case 'ready':
this.isInitialized = true;
console.debug('Vector worker initialized');
break;
case 'progress':
if (this.progressCallback) {
this.progressCallback(data);
}
break;
case 'searchResults':
const searchPromise = this.searchPromises.get(data.messageId);
if (searchPromise) {
searchPromise.resolve(data.results);
this.searchPromises.delete(data.messageId);
} else {
console.warn('Received search results for unknown messageId:', data.messageId);
}
break;
case 'searchError':
const errorPromise = this.searchPromises.get(data.messageId);
if (errorPromise) {
errorPromise.reject(new Error(data.error));
this.searchPromises.delete(data.messageId);
} else {
console.warn('Received search error for unknown messageId:', data.messageId);
}
break;
case 'searchCancelled':
const cancelledPromise = this.searchPromises.get(data.messageId);
if (cancelledPromise) {
cancelledPromise.reject(new Error('Search cancelled'));
this.searchPromises.delete(data.messageId);
} else {
console.debug('Received cancellation for unknown messageId:', data.messageId);
}
break;
default:
console.warn('Unknown message from worker:', type, data);
}
});
// Initialize the worker
this.worker.postMessage({ type: 'init' });
// Wait for ready message
await new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker initialization timed out'));
}, 5000);
const checkInit = (e: MessageEvent) => {
if (e.data.type === 'ready') {
this.worker!.removeEventListener('message', checkInit);
clearTimeout(timeout);
resolve();
}
};
this.worker!.addEventListener('message', checkInit);
});
}
async processItems(items: HydratedIndexItem[], onProgress?: ProgressCallback) {
if (!this.isInitialized) {
await this.init();
}
this.progressCallback = onProgress || null;
this.worker!.postMessage({
type: 'process',
data: { items }
});
}
terminate() {
if (this.worker) {
// Clean up any pending promises
for (const [messageId, promise] of this.searchPromises.entries()) {
promise.reject(new Error('Worker terminated'));
this.searchPromises.delete(messageId);
}
this.worker.terminate();
this.worker = null;
this.isInitialized = false;
}
}
}
src/plugins/built-in/globalSearch/search/vector/vectorSearch.ts
+66
View File
@@ -0,0 +1,66 @@
import type { VectorSearchResult } from "./vectorTypes";
//import vectorSearchWorker from "./vectorSearchWorker?worker";
export function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
//return VectorSearchWorkerManager.getInstance().search(query, topK);
return new Promise((resolve) => {
resolve([]);
});
}
/* class VectorSearchWorkerManager {
private static instance: VectorSearchWorkerManager;
private worker: Worker | null = null;
private pendingSearches = new Map<string, (results: VectorSearchResult[]) => void>();
constructor() {
this.initWorker();
}
private initWorker() {
try {
this.worker = new vectorSearchWorker({ name: "vectorSearchWorker" });
this.worker.addEventListener('message', this.messageHandler);
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
}
}
private messageHandler = (e: MessageEvent) => {
console.log("Message received", e.data);
if (e.data.type === 'searchResults') {
const resolve = this.pendingSearches.get(e.data.data.messageId);
if (resolve) {
resolve(e.data.data.results);
this.pendingSearches.delete(e.data.data.messageId);
}
}
};
public static getInstance(): VectorSearchWorkerManager {
if (!VectorSearchWorkerManager.instance) {
VectorSearchWorkerManager.instance = new VectorSearchWorkerManager();
}
return VectorSearchWorkerManager.instance;
}
public async search(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
console.log("Searching vectors", query, topK);
if (!this.worker) {
this.initWorker();
}
const messageId = crypto.randomUUID();
return new Promise((resolve) => {
this.pendingSearches.set(messageId, resolve);
this.worker?.postMessage({
type: "search",
data: { query, topK },
messageId
});
});
}
}
export default VectorSearchWorkerManager; */
src/plugins/built-in/globalSearch/search/vector/vectorSearchWorker.ts
+59
View File
@@ -0,0 +1,59 @@
import { EmbeddingIndex, getEmbedding, initializeModel } from "client-vector-search";
import type { VectorSearchResult } from "./vectorTypes";
console.log("%cVector search worker initialized", "background-color: #000; color: #fff;");
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
async function initVectorSearch() {
if (isInitialized) return;
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
// Load existing items from IndexedDB
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach((item) => vectorIndex!.add(item));
console.debug("Vector index loaded from IndexedDB");
}
isInitialized = true;
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
}
}
async function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
if (!vectorIndex) await initVectorSearch();
const queryEmbedding = await getEmbedding(query);
const results = await vectorIndex!.search(queryEmbedding, {
topK,
useStorage: 'indexedDB'
});
return results as VectorSearchResult[];
}
self.addEventListener('message', async (e) => {
const { type, data, messageId } = e.data;
switch (type) {
case 'search':
console.log("Search request received", data);
const results = await searchVectors(data.query, data.topK);
self.postMessage({ type: 'searchResults', data: { messageId, results } });
break;
default:
console.warn(`Unknown message type: ${type}`);
}
});
initVectorSearch();
export default function test() {
console.log("%cTest!!!", "background-color: #000; color: #fff;");
}
src/plugins/built-in/globalSearch/search/vector/vectorTypes.ts
+7
View File
@@ -0,0 +1,7 @@
import type { SearchResult } from "client-vector-search";
import type { HydratedIndexItem } from "../../indexing/types";
export interface VectorSearchResult extends SearchResult {
object: HydratedIndexItem & { embedding: number[] };
}
src/plugins/built-in/globalSearch/searchUtils.ts
+55 -58
View File
@@ -1,48 +1,32 @@
import Fuse, { type FuseResult } from 'fuse.js'; import Fuse, { type FuseResult } from "fuse.js";
import { getStaticCommands, type StaticCommandItem } from './commands'; import { getStaticCommands, type StaticCommandItem } from "./core/commands";
import { getDynamicItems } from './dynamicSearch'; import { getDynamicItems } from "./dynamicSearch";
import type { CombinedResult } from './types'; import type { CombinedResult } from "./core/types";
import type { HydratedIndexItem } from './indexing/types'; import type { HydratedIndexItem } from "./indexing/types";
import { searchVectors, type VectorSearchResult } from './vectorSearch'; import { searchVectors } from "./search/vector/vectorSearch";
// This function is likely no longer needed as items are pre-processed by the indexer
/* export function prepareDynamicItems(items: DynamicContentItem[]): DynamicContentItem[] {
return items.map(item => {
const preparedItem = { ...item };
// @ts-ignore
preparedItem.allContent = [
item.text,
item.content,
item.category,
].filter(Boolean).join(' ');
return preparedItem;
});
} */
export function createSearchIndexes() { export function createSearchIndexes() {
const commands = getStaticCommands(); const commands = getStaticCommands();
const dynamicItems = getDynamicItems(); // Returns HydratedIndexItem[] const dynamicItems = getDynamicItems(); // Returns HydratedIndexItem[]
const commandOptions = { const commandOptions = {
keys: ['text', 'category', 'keywords'], keys: ["text", "category", "keywords"],
includeScore: true, includeScore: true,
includeMatches: true, includeMatches: true,
threshold: 0.6, threshold: 0.6,
minMatchCharLength: 1, minMatchCharLength: 1,
ignoreLocation: true, ignoreLocation: true,
useExtendedSearch: false useExtendedSearch: false,
}; };
// Keys for dynamic items remain the same structurally // Keys for dynamic items remain the same structurally
const dynamicOptions = { const dynamicOptions = {
keys: [ keys: [
'text', "text",
'content', "content",
'category', "category",
'metadata.author', // Example: Include specific metadata if needed "metadata.author", // Example: Include specific metadata if needed
'metadata.subject', // Example: Include specific metadata if needed "metadata.subject", // Example: Include specific metadata if needed
// 'keywords', // Keywords are not currently part of IndexItem, add if needed // 'keywords', // Keywords are not currently part of IndexItem, add if needed
], ],
includeScore: true, includeScore: true,
@@ -55,9 +39,12 @@ export function createSearchIndexes() {
return { return {
commandsFuse: new Fuse(commands, commandOptions) as Fuse<StaticCommandItem>, commandsFuse: new Fuse(commands, commandOptions) as Fuse<StaticCommandItem>,
dynamicContentFuse: new Fuse(dynamicItems, dynamicOptions) as Fuse<HydratedIndexItem>, dynamicContentFuse: new Fuse(
dynamicItems,
dynamicOptions,
) as Fuse<HydratedIndexItem>,
commands, commands,
dynamicItems dynamicItems,
}; };
} }
@@ -65,7 +52,7 @@ export function searchCommands(
commandsFuse: Fuse<StaticCommandItem>, commandsFuse: Fuse<StaticCommandItem>,
query: string, query: string,
commandIdToItemMap: Map<string, StaticCommandItem>, commandIdToItemMap: Map<string, StaticCommandItem>,
limit = 10 limit = 10,
): CombinedResult[] { ): CombinedResult[] {
if (!commandsFuse) return []; if (!commandsFuse) return [];
@@ -73,11 +60,11 @@ export function searchCommands(
return Array.from(commandIdToItemMap.values()) return Array.from(commandIdToItemMap.values())
.sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)) // Sort by priority when no query .sort((a, b) => (b.priority ?? 0) - (a.priority ?? 0)) // Sort by priority when no query
.slice(0, limit) // Limit results even when no query .slice(0, limit) // Limit results even when no query
.map(item => ({ .map((item) => ({
id: item.id, id: item.id,
type: 'command' as const, type: "command" as const,
score: 100 + (item.priority ?? 0), score: 100 + (item.priority ?? 0),
item item,
})); }));
} }
@@ -90,10 +77,10 @@ export function searchCommands(
return { return {
id: item.id, id: item.id,
type: 'command' as const, type: "command" as const,
score, score,
item, item,
matches: result.matches matches: result.matches,
}; };
}); });
} }
@@ -103,7 +90,7 @@ export function searchDynamicItems(
query: string, query: string,
dynamicIdToItemMap: Map<string, HydratedIndexItem>, dynamicIdToItemMap: Map<string, HydratedIndexItem>,
limit = 10, limit = 10,
sortByRecent: boolean = true // Added option to control sorting sortByRecent: boolean = true, // Added option to control sorting
): CombinedResult[] { ): CombinedResult[] {
if (!dynamicContentFuse) return []; if (!dynamicContentFuse) return [];
@@ -112,13 +99,11 @@ export function searchDynamicItems(
if (sortByRecent) { if (sortByRecent) {
items = items.sort((a, b) => b.dateAdded - a.dateAdded); items = items.sort((a, b) => b.dateAdded - a.dateAdded);
} }
return items return items.slice(0, limit).map((item) => ({
.slice(0, limit)
.map(item => ({
id: item.id, id: item.id,
type: 'dynamic' as const, type: "dynamic" as const,
score: 80, // Assign a default score for non-searched items score: 80, // Assign a default score for non-searched items
item item,
})); }));
} }
@@ -129,15 +114,15 @@ export function searchDynamicItems(
const item = result.item; const item = result.item;
const fuseScore = 10 * (1 - (result.score || 0.5)); const fuseScore = 10 * (1 - (result.score || 0.5));
const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24);
const recencyBoost = sortByRecent ? (1 / (ageInDays + 1)) : 0; // Apply boost only if sorting by recent const recencyBoost = sortByRecent ? 1 / (ageInDays + 1) : 0; // Apply boost only if sorting by recent
const score = fuseScore + recencyBoost; const score = fuseScore + recencyBoost;
return { return {
id: item.id, id: item.id,
type: 'dynamic' as const, type: "dynamic" as const,
score, score,
item, item,
matches: result.matches matches: result.matches,
}; };
}); });
} }
@@ -148,45 +133,57 @@ export async function performSearch(
dynamicContentFuse: Fuse<HydratedIndexItem>, dynamicContentFuse: Fuse<HydratedIndexItem>,
commandIdToItemMap: Map<string, StaticCommandItem>, commandIdToItemMap: Map<string, StaticCommandItem>,
dynamicIdToItemMap: Map<string, HydratedIndexItem>, dynamicIdToItemMap: Map<string, HydratedIndexItem>,
showRecentFirst: boolean showRecentFirst: boolean,
): Promise<CombinedResult[]> { ): Promise<CombinedResult[]> {
const startTime = performance.now(); const startTime = performance.now();
// Get all results first // Get all results first
const commandResults = searchCommands(commandsFuse, query, commandIdToItemMap); const commandResults = searchCommands(
commandsFuse,
query,
commandIdToItemMap,
);
const commandEndTime = performance.now(); const commandEndTime = performance.now();
const dynamicResults = searchDynamicItems(dynamicContentFuse, query, dynamicIdToItemMap, 10, showRecentFirst); const dynamicResults = searchDynamicItems(
dynamicContentFuse,
query,
dynamicIdToItemMap,
10,
showRecentFirst,
);
const fuseEndTime = performance.now(); const fuseEndTime = performance.now();
// Get vector results in parallel // Get vector results in parallel
const vectorResults = await searchVectors(query, 10); const vectorResults = await searchVectors(query, 10);
const vectorEndTime = performance.now(); const vectorEndTime = performance.now();
console.log('Vector results:', vectorResults); console.log("Vector results:", vectorResults);
// Log timings // Log timings
console.log(`Command search took ${commandEndTime - startTime} milliseconds`); console.log(`Command search took ${commandEndTime - startTime} milliseconds`);
console.log(`Dynamic search took ${fuseEndTime - commandEndTime} milliseconds`); console.log(
`Dynamic search took ${fuseEndTime - commandEndTime} milliseconds`,
);
console.log(`Vector search took ${vectorEndTime - fuseEndTime} milliseconds`); console.log(`Vector search took ${vectorEndTime - fuseEndTime} milliseconds`);
// Create a map to store our final results, using ID as key to avoid duplicates // Create a map to store our final results, using ID as key to avoid duplicates
const resultMap = new Map<string, CombinedResult>(); const resultMap = new Map<string, CombinedResult>();
// Add command results first (they keep their original scores) // Add command results first (they keep their original scores)
commandResults.forEach(r => resultMap.set(r.id, r)); commandResults.forEach((r) => resultMap.set(r.id, r));
// Process dynamic results and vector results together // Process dynamic results and vector results together
const seenIds = new Set<string>(); const seenIds = new Set<string>();
// Add dynamic results first // Add dynamic results first
dynamicResults.forEach(r => { dynamicResults.forEach((r) => {
seenIds.add(r.id); seenIds.add(r.id);
const vectorMatch = vectorResults.find(v => v.object.id === r.id); const vectorMatch = vectorResults.find((v) => v.object.id === r.id);
if (vectorMatch) { if (vectorMatch) {
// If we found it in both searches, combine the scores // If we found it in both searches, combine the scores
resultMap.set(r.id, { resultMap.set(r.id, {
...r, ...r,
score: r.score + (vectorMatch.similarity * 0.6) // Boost exact matches score: r.score + vectorMatch.similarity * 0.6, // Boost exact matches
}); });
} else { } else {
// If only in Fuse results, keep as is // If only in Fuse results, keep as is
@@ -195,15 +192,15 @@ export async function performSearch(
}); });
// Now add any vector results we haven't seen yet // Now add any vector results we haven't seen yet
vectorResults.forEach(v => { vectorResults.forEach((v) => {
const id = v.object.id; const id = v.object.id;
if (!seenIds.has(id)) { if (!seenIds.has(id)) {
// This is a semantic match that Fuse missed - add it with the vector similarity as score // This is a semantic match that Fuse missed - add it with the vector similarity as score
resultMap.set(id, { resultMap.set(id, {
id, id,
type: 'dynamic' as const, type: "dynamic" as const,
score: v.similarity * 0.9, // High base score for semantic matches score: v.similarity * 0.9, // High base score for semantic matches
item: v.object item: v.object,
}); });
} }
}); });
src/plugins/built-in/globalSearch/unitMap.ts
-193
View File
@@ -1,193 +0,0 @@
export const unitFullNames: Record<string, string> = {
// --- Length ---
"m": "Meters",
"km": "Kilometers",
"cm": "Centimeters",
"mm": "Millimeters",
"µm": "Micrometers",
"nm": "Nanometers",
"pm": "Picometers",
"fm": "Femtometers",
"am": "Attometers",
"zm": "Zeptometers",
"ym": "Yoctometers",
"mi": "Miles",
"yd": "Yards",
"ft": "Feet",
"in": "Inches",
"nmi": "Nautical Miles",
"angstrom": "Angstroms",
"au": "Astronomical Units",
"ly": "Light Years",
"pc": "Parsecs",
// --- Mass ---
"kg": "Kilograms",
"g": "Grams",
"mg": "Milligrams",
"µg": "Micrograms",
"ng": "Nanograms",
"lb": "Pounds",
"oz": "Ounces",
"ton": "Tons (Imperial)",
"tonne": "Tonnes (Metric)",
"slug": "Slugs",
"stone": "Stones",
// --- Time ---
"s": "Seconds",
"ms": "Milliseconds",
"µs": "Microseconds",
"ns": "Nanoseconds",
"ps": "Picoseconds",
"min": "Minutes",
"h": "Hours",
"day": "Days",
"week": "Weeks",
"month": "Months (30 days)",
"year": "Years (365 days)",
"fortnight": "Fortnights",
// --- Temperature ---
"K": "Kelvin",
"degC": "Degrees Celsius",
"degF": "Degrees Fahrenheit",
"degR": "Degrees Rankine",
// --- Volume ---
"m³": "Cubic Meters",
"cm³": "Cubic Centimeters",
"mm³": "Cubic Millimeters",
"l": "Liters",
"ml": "Milliliters",
"gal": "Gallons (US)",
"qt": "Quarts (US)",
"pt": "Pints (US)",
"cup": "Cups (US)",
"floz": "Fluid Ounces (US)",
"tbsp": "Tablespoons (US)",
"tsp": "Teaspoons (US)",
// --- Area ---
"m²": "Square Meters",
"km²": "Square Kilometers",
"cm²": "Square Centimeters",
"mm²": "Square Millimeters",
"ha": "Hectares",
"acre": "Acres",
"ft²": "Square Feet",
"in²": "Square Inches",
"mi²": "Square Miles",
// --- Speed ---
"m/s": "Meters per Second",
"km/h": "Kilometers per Hour",
"mph": "Miles per Hour",
"knot": "Knots",
// --- Acceleration ---
"m/s²": "Meters per Second Squared",
// --- Force ---
"N": "Newtons",
"lbf": "Pound-Force",
"dyn": "Dynes",
// --- Energy ---
"J": "Joules",
"kJ": "Kilojoules",
"cal": "Calories",
"kcal": "Kilocalories",
"Wh": "Watt Hours",
"kWh": "Kilowatt Hours",
"BTU": "British Thermal Units",
"erg": "Ergs",
"eV": "Electronvolts",
// --- Power ---
"W": "Watts",
"kW": "Kilowatts",
"MW": "Megawatts",
"GW": "Gigawatts",
"hp": "Horsepower",
// --- Pressure ---
"Pa": "Pascals",
"kPa": "Kilopascals",
"bar": "Bar",
"atm": "Atmospheres",
"psi": "Pounds per Square Inch",
"torr": "Torr",
"mmHg": "Millimeters of Mercury",
// --- Frequency ---
"Hz": "Hertz",
"kHz": "Kilohertz",
"MHz": "Megahertz",
"GHz": "Gigahertz",
"THz": "Terahertz",
// --- Electric ---
"V": "Volts",
"A": "Amperes",
"C": "Coulombs",
"Ω": "Ohms",
"F": "Farads",
"S": "Siemens",
"H": "Henries",
"Wb": "Webers",
"T": "Teslas",
"lx": "Lux",
// --- Angle & Rotation ---
"rad": "Radians",
"deg": "Degrees",
"grad": "Gradians",
"cycle": "Cycles",
"turn": "Turns",
"rev": "Revolutions",
// --- Charge & Capacitance ---
"e": "Elementary Charges",
// --- Magnetic & Light ---
"lm": "Lumens",
"ph": "Photons",
// --- Miscellaneous / Dimensionless ---
"%": "Percent",
"ppm": "Parts per Million",
"ppb": "Parts per Billion",
"pptr": "Parts per Trillion",
"dB": "Decibels",
"bit": "Bits",
"byte": "Bytes",
// --- Digital Storage ---
"b": "Bits",
"B": "Bytes",
"kb": "Kilobits",
"kB": "Kilobytes",
"Mb": "Megabits",
"MB": "Megabytes",
"Gb": "Gigabits",
"GB": "Gigabytes",
"Tb": "Terabits",
"TB": "Terabytes",
// --- Currency ---
"USD": "US Dollars",
"EUR": "Euros",
"GBP": "British Pounds",
"AUD": "Australian Dollars",
"CAD": "Canadian Dollars",
"CHF": "Swiss Francs",
"JPY": "Japanese Yen",
"CNY": "Chinese Yuan",
"INR": "Indian Rupees",
"NZD": "New Zealand Dollars",
"SEK": "Swedish Krona",
"NOK": "Norwegian Krone",
"SGD": "Singapore Dollars",
"HKD": "Hong Kong Dollars"
};
src/plugins/built-in/globalSearch/vectorSearch.ts
-86
View File
@@ -1,86 +0,0 @@
import { EmbeddingIndex, getEmbedding, initializeModel } from 'client-vector-search';
import type { HydratedIndexItem } from './indexing/types';
import type { SearchResult } from 'client-vector-search';
let vectorIndex: EmbeddingIndex | null = null;
export async function initVectorSearch() {
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
// Load existing items from IndexedDB
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach(item => vectorIndex!.add(item));
console.debug('Vector index loaded from IndexedDB');
}
} catch (e) {
console.debug('Creating new vector index');
vectorIndex = new EmbeddingIndex([]);
}
}
export async function vectorizeItem(item: HydratedIndexItem): Promise<HydratedIndexItem & { embedding: number[] }> {
const textToEmbed = [
item.text,
item.content,
item.category,
item.metadata?.author,
item.metadata?.subject
].filter(Boolean).join(' ');
const embedding = await getEmbedding(textToEmbed);
return { ...item, embedding };
}
export async function processItems(items: HydratedIndexItem[]) {
if (!vectorIndex) await initVectorSearch();
const unprocessedItems = items.filter(item => {
try {
return !vectorIndex!.get({ id: item.id });
} catch {
return true;
}
});
if (unprocessedItems.length === 0) {
console.debug('No new items to vectorize');
return;
}
console.debug(`Vectorizing ${unprocessedItems.length} new items...`);
// Process in batches to avoid UI freeze
const BATCH_SIZE = 5;
for (let i = 0; i < unprocessedItems.length; i += BATCH_SIZE) {
const batch = unprocessedItems.slice(i, i + BATCH_SIZE);
const vectorized = await Promise.all(batch.map(vectorizeItem));
for (const item of vectorized) {
vectorIndex!.add(item);
}
// Save periodically to avoid losing progress
await vectorIndex!.saveIndex('indexedDB');
// Log progress
console.debug(`Vectorized ${Math.min(i + BATCH_SIZE, unprocessedItems.length)}/${unprocessedItems.length} items`);
}
}
export interface VectorSearchResult extends SearchResult {
object: HydratedIndexItem & { embedding: number[] };
}
export async function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
if (!vectorIndex) await initVectorSearch();
const queryEmbedding = await getEmbedding(query);
const results = await vectorIndex!.search(queryEmbedding, {
topK,
useStorage: 'indexedDB'
});
return results as VectorSearchResult[];
}
src/plugins/index.ts
+1 -1
View File
@@ -6,7 +6,7 @@ import notificationCollectorPlugin from './built-in/notificationCollector';
import themesPlugin from './built-in/themes'; import themesPlugin from './built-in/themes';
import animatedBackgroundPlugin from './built-in/animatedBackground'; import animatedBackgroundPlugin from './built-in/animatedBackground';
import assessmentsAveragePlugin from './built-in/assessmentsAverage'; import assessmentsAveragePlugin from './built-in/assessmentsAverage';
import globalSearchPlugin from './built-in/globalSearch'; import globalSearchPlugin from './built-in/globalSearch/core';
import testPlugin from './built-in/test'; import testPlugin from './built-in/test';
// Initialize plugin manager // Initialize plugin manager
src/plugins/monofile.ts
+4
View File
@@ -330,7 +330,11 @@ async function handleMessages(node: Element): Promise<void> {
const element = document.getElementById("title")!.firstChild as HTMLElement const element = document.getElementById("title")!.firstChild as HTMLElement
element.innerText = "Direct Messages" element.innerText = "Direct Messages"
document.title = "Direct Messages ― SEQTA Learn" document.title = "Direct Messages ― SEQTA Learn"
try {
SortMessagePageItems(node) SortMessagePageItems(node)
} catch (error) {
console.error("Error sorting message page items:", error)
}
if (!settingsState.animations) return if (!settingsState.animations) return
vite.config.ts
+8 -3
View File
@@ -1,7 +1,9 @@
import { defineConfig } from 'vite'; import { defineConfig } from 'vite';
import path, { join, resolve } from 'path'; import { join, resolve } from 'path';
import fs from 'fs';
import { updateManifestPlugin } from './lib/patchPackage'; import { updateManifestPlugin } from './lib/patchPackage';
import touchGlobalCSSPlugin from './lib/touchGlobalCSS';
import InlineWorkerPlugin from './lib/inlineWorker';
import { base64Loader } from './lib/base64loader'; import { base64Loader } from './lib/base64loader';
import type { BuildTarget } from './lib/types'; import type { BuildTarget } from './lib/types';
import ClosePlugin from './lib/closePlugin'; import ClosePlugin from './lib/closePlugin';
@@ -19,7 +21,6 @@ import { opera } from './src/manifests/opera';
import { safari } from './src/manifests/safari'; import { safari } from './src/manifests/safari';
import { crx } from '@crxjs/vite-plugin'; import { crx } from '@crxjs/vite-plugin';
import touchGlobalCSSPlugin from './lib/touchGlobalCSS';
const targets: BuildTarget[] = [ const targets: BuildTarget[] = [
chrome, brave, edge, firefox, opera, safari chrome, brave, edge, firefox, opera, safari
] ]
@@ -30,6 +31,7 @@ const mode = process.env.MODE || 'chrome'; // Check the environment variable to
export default defineConfig(({ command }) => ({ export default defineConfig(({ command }) => ({
plugins: [ plugins: [
base64Loader, base64Loader,
InlineWorkerPlugin(),
svelte({ svelte({
emitCss: false emitCss: false
}), }),
@@ -70,6 +72,9 @@ export default defineConfig(({ command }) => ({
legacy: { legacy: {
skipWebSocketTokenCheck: true, skipWebSocketTokenCheck: true,
}, },
worker: {
format: 'es',
},
build: { build: {
outDir: resolve(__dirname, 'dist', mode), outDir: resolve(__dirname, 'dist', mode),
emptyOutDir: false, emptyOutDir: false,