BetterSEQTA/BetterSEQTA-Plus
1
0
Fork 0
mirror of https://github.com/BetterSEQTA/BetterSEQTA-Plus.git synced 2026-06-06 03:34:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: improve results and performance (syncronous)

Browse Source
This commit is contained in:
SethBurkart123
2025-04-13 10:58:05 +10:00
parent 9562368157
commit 454ab283ab
10 changed files with 369 additions and 377 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+1 -1
View File
@@ -75,7 +75,7 @@
"@uiw/codemirror-extensions-color": "^4.23.10",
"@uiw/codemirror-theme-github": "^4.23.10",
"autoprefixer": "^10.4.21",
"client-vector-search": "^0.2.0",
"client-vector-search": "../client-vector-search",
"codemirror": "^6.0.1",
"color": "^5.0.0",
"dompurify": "^3.2.4",
src/plugins/built-in/globalSearch/SearchBar.svelte
+7 -7
View File
@@ -11,6 +11,7 @@
import Calculator from './components/Calculator.svelte';
import { actionMap } from './indexing/actions';
import type { IndexItem, HydratedIndexItem } from './indexing/types';
import debounce from 'lodash/debounce';
const {
transparencyEffects,
@@ -127,9 +128,15 @@
isLoading = false;
};
const debouncedPerformSearch = debounce(performSearch, 10);
$effect(() => {
if (commandPalleteOpen) {
if (searchTerm === '') {
performSearch();
} else {
debouncedPerformSearch();
}
tick().then(() => searchbar?.focus());
} else {
searchTerm = '';
@@ -139,13 +146,6 @@
}
});
$effect(() => {
if (commandPalleteOpen && searchTerm !== prevSearchTerm) {
prevSearchTerm = searchTerm;
performSearch();
}
});
$effect(() => {
if (combinedResults.length === 0 && calculatorResult && commandPalleteOpen) {
selectedIndex = 0;
src/plugins/built-in/globalSearch/components/Calculator.svelte
+4 -1
View File
@@ -31,7 +31,10 @@
// Process the input with debounce to avoid unnecessary calculations
const processInput = (input: string) => {
try {
if (!input.trim()) {
if (
!input.trim() ||
(input.trim().length <= 2 && !/\d/.test(input))
) {
result = null;
inputUnit = '';
outputUnit = '';
src/plugins/built-in/globalSearch/core/index.ts
+3 -27
View File
@@ -10,12 +10,10 @@ import renderSvelte from "@/interface/main";
import SearchBar from "../SearchBar.svelte";
import styles from "./styles.css?inline";
import { unmount } from "svelte";
import { loadDynamicItems } from "../dynamicSearch";
import { waitForElm } from "@/seqta/utils/waitForElm";
import { loadAllStoredItems, runIndexing } from "../indexing/indexer";
//import { initVectorSearch } from "../search/vector/vectorSearch";
import { runIndexing } from "../indexing/indexer";
import { VectorWorkerManager } from "../indexing/worker/vectorWorkerManager";
import VectorSearchWorkerManager from "../search/vector/vectorSearch";
import { initVectorSearch } from "../search/vector/vectorSearch";
const settings = defineSettings({
searchHotkey: stringSetting({
@@ -56,26 +54,6 @@ class GlobalSearchPlugin extends BasePlugin<typeof settings> {
const settingsInstance = new GlobalSearchPlugin();
const updateDynamicItemsFromIndex = async () => {
const indexedItems = await loadAllStoredItems();
loadDynamicItems(indexedItems);
console.log(`Loaded ${indexedItems.length} indexed items into search.`);
// Process items through vector search worker
const workerManager = VectorWorkerManager.getInstance();
await workerManager.processItems(indexedItems, (progress) => {
if (progress.status === "started") {
console.debug(`Starting vector processing of ${progress.total} items...`);
} else if (progress.status === "processing") {
console.debug(`Vectorized ${progress.processed}/${progress.total} items`);
} else if (progress.status === "complete") {
console.debug("Vector processing complete:", progress.message);
}
});
window.dispatchEvent(new CustomEvent("dynamic-items-updated"));
};
const globalSearchPlugin: Plugin<typeof settings> = {
id: "global-search",
name: "Global Search",
@@ -88,13 +66,12 @@ const globalSearchPlugin: Plugin<typeof settings> = {
run: async (api) => {
let app: any;
VectorSearchWorkerManager.getInstance();
initVectorSearch();
// Run initial indexing and update dynamic items
if (api.settings.runIndexingOnLoad) {
setTimeout(async () => {
await runIndexing();
await updateDynamicItemsFromIndex();
}, 2000); // Delay initial indexing to let page load
}
@@ -156,7 +133,6 @@ const globalSearchPlugin: Plugin<typeof settings> = {
// Clean up workers
VectorWorkerManager.getInstance().terminate();
VectorSearchWorkerManager.getInstance().terminate();
unmount(app);
};
},
src/plugins/built-in/globalSearch/indexing/indexer.ts
+129 -108
View File
@@ -2,7 +2,7 @@ import { clear, getAll, put, remove } from "./db";
import { jobs } from "./jobs";
import { renderComponentMap } from "./renderComponents";
import type { HydratedIndexItem, IndexItem, Job, JobContext } from "./types";
import { EmbeddingIndex, getEmbedding, initializeModel } from "client-vector-search";
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
const META_STORE = "meta";
const LOCK_KEY = "bsq-indexer-lock";
@@ -11,83 +11,6 @@ const LOCK_TIMEOUT = 20000;
let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
async function initVectorSearch() {
if (isInitialized) return;
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
// Load existing items from IndexedDB
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach((item) => vectorIndex!.add(item));
console.debug("Vector index loaded from IndexedDB");
}
isInitialized = true;
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
}
}
async function vectorizeItem(
item: HydratedIndexItem,
): Promise<HydratedIndexItem & { embedding: number[] }> {
const textToEmbed = [
item.text,
item.content,
item.category,
item.metadata?.author,
item.metadata?.subject,
]
.filter(Boolean)
.join(" ");
const embedding = await getEmbedding(textToEmbed);
return { ...item, embedding };
}
async function processItems(items: HydratedIndexItem[]) {
if (!vectorIndex) await initVectorSearch();
const unprocessedItems = items.filter((item) => {
try {
return !vectorIndex!.get({ id: item.id });
} catch {
return true;
}
});
if (unprocessedItems.length === 0) {
console.debug("No new items to vectorize");
return;
}
console.debug(`Vectorizing ${unprocessedItems.length} new items...`);
// Process in batches to avoid UI freeze
const BATCH_SIZE = 5;
for (let i = 0; i < unprocessedItems.length; i += BATCH_SIZE) {
const batch = unprocessedItems.slice(i, i + BATCH_SIZE);
const vectorized = await Promise.all(batch.map(vectorizeItem));
for (const item of vectorized) {
vectorIndex!.add(item);
}
// Save periodically to avoid losing progress
await vectorIndex!.saveIndex("indexedDB");
// Log progress
console.debug(
`Vectorized ${Math.min(i + BATCH_SIZE, unprocessedItems.length)}/${unprocessedItems.length} items`,
);
}
}
function shouldRun(job: Job, lastRun?: number): boolean {
const now = Date.now();
@@ -133,29 +56,43 @@ function stopHeartbeat() {
localStorage.removeItem(LOCK_KEY);
}
function dispatchProgress(completed: number, total: number, indexing: boolean) {
function dispatchProgress(completed: number, total: number, indexing: boolean, status?: string, detail?: string) {
const event = new CustomEvent("indexing-progress", {
detail: { completed, total, indexing },
detail: { completed, total, indexing, status, detail },
});
window.dispatchEvent(event);
}
export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
const all: HydratedIndexItem[] = [];
const jobIds = Object.keys(jobs);
for (const jobId in jobs) {
const items = await getAll(jobId);
for (const jobId of jobIds) {
try {
const items = await getAll(jobId) as IndexItem[];
const job = jobs[jobId];
const renderComponent = renderComponentMap[job.renderComponentId];
for (const item of items) {
all.push({
...item,
renderComponent,
});
}
if (!renderComponent) {
console.warn(`Render component not found for job ${jobId} (ID: ${job.renderComponentId})`);
}
for (const item of items) {
// Ensure item has all required fields before pushing
if (item && item.id && item.text && item.category && item.actionId && job.renderComponentId) {
all.push({
...item,
renderComponent: renderComponent || undefined, // Assign undefined if not found
});
} else {
console.warn(`Skipping invalid item from job ${jobId}:`, item);
}
}
} catch (error) {
console.error(`Error loading items for job ${jobId}:`, error);
}
}
console.debug(`[Indexer] Loaded ${all.length} items from non-vector storage.`);
return all;
}
@@ -173,11 +110,15 @@ export async function runIndexing(): Promise<void> {
const jobIds = Object.keys(jobs);
let completedJobs = 0;
dispatchProgress(completedJobs, jobIds.length, true);
// Add an extra step for vectorization
const totalSteps = jobIds.length + 1;
dispatchProgress(completedJobs, totalSteps, true, "Starting jobs");
const allNewItems: HydratedIndexItem[] = [];
const allItemsFromJobs: HydratedIndexItem[] = [];
// --- Step 1: Run Fetching/Storing Jobs (Main Thread) ---
for (const jobId of jobIds) {
dispatchProgress(completedJobs, totalSteps, true, `Running job: ${jobs[jobId].label}`);
const job = jobs[jobId];
const lastRun = await getLastRunMeta(jobId);
@@ -187,17 +128,27 @@ export async function runIndexing(): Promise<void> {
"color: gray",
);
completedJobs++;
dispatchProgress(completedJobs, jobIds.length, true);
dispatchProgress(completedJobs, totalSteps, true, `Skipped job: ${job.label}`);
continue;
}
// These DB operations happen on the main thread (acceptable per request)
const getStoredItems = async () => await getAll(jobId);
const setStoredItems = async (items: IndexItem[]) => {
await clear(jobId);
await Promise.all(items.map((i) => put(jobId, i, i.id)));
// Add validation before putting
const validItems = items.filter(i => i && i.id);
if (validItems.length !== items.length) {
console.warn(`[Indexer Job ${jobId}] Filtered out ${items.length - validItems.length} invalid items before storing.`);
}
await Promise.all(validItems.map((i) => put(jobId, i, i.id)));
};
const addItem = async (item: IndexItem) => {
if (item && item.id) { // Add validation
await put(jobId, item, item.id);
} else {
console.warn(`[Indexer Job ${jobId}] Attempted to add invalid item:`, item);
}
};
const removeItem = async (id: string) => {
await remove(jobId, id);
@@ -213,24 +164,35 @@ export async function runIndexing(): Promise<void> {
console.debug(`%c[Indexer] Running job "${jobId}"...`, "color: #4ea1ff");
try {
const newItems = await job.run(ctx);
const newItemsRaw = await job.run(ctx);
const stored = await getStoredItems();
let merged = mergeItems(stored, newItems);
let merged = mergeItems(stored, newItemsRaw);
if (job.purge) merged = job.purge(merged);
await setStoredItems(merged);
await setStoredItems(merged); // Store merged non-vector data
await updateLastRunMeta(jobId);
// Add to our collection of new items for vector processing
const hydratedItems = merged.map((item) => ({
// Hydrate items for vector processing
const renderComponent = renderComponentMap[job.renderComponentId];
if (!renderComponent) {
console.warn(`Render component not found for job ${jobId} (ID: ${job.renderComponentId}) during hydration`);
}
const hydratedItems = merged
.filter(item => item && item.id && item.text && item.category && item.actionId && job.renderComponentId) // Filter invalid before hydrating
.map((item) => ({
...item,
renderComponent: renderComponentMap[job.renderComponentId],
renderComponent: renderComponent || undefined, // Assign undefined if not found
}));
allNewItems.push(...hydratedItems);
if (hydratedItems.length !== merged.length) {
console.warn(`[Indexer Job ${jobId}] Filtered out ${merged.length - hydratedItems.length} invalid items during hydration.`);
}
allItemsFromJobs.push(...hydratedItems);
console.debug(
`%c[Indexer] ✅ ${job.label}: ${newItems.length} items indexed`,
`%c[Indexer] ✅ ${job.label}: ${newItemsRaw.length} new items fetched, ${merged.length} total stored (non-vector).`,
"color: #00c46f",
);
} catch (err) {
@@ -239,25 +201,84 @@ export async function runIndexing(): Promise<void> {
}
completedJobs++;
dispatchProgress(completedJobs, jobIds.length, true);
dispatchProgress(completedJobs, totalSteps, true, `Finished job: ${job.label}`);
}
// Process all new items through vector search
if (allNewItems.length > 0) {
// --- Step 2: Delegate Vectorization to Worker (Off Main Thread) ---
if (allItemsFromJobs.length > 0) {
console.debug(
`%c[Indexer] Processing ${allNewItems.length} items for vector search...`,
`%c[Indexer] Sending ${allItemsFromJobs.length} items to worker for vectorization...`,
"color: #4ea1ff",
);
await processItems(allNewItems);
dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization");
try {
const workerManager = VectorWorkerManager.getInstance();
// Pass a progress callback to the worker manager
await workerManager.processItems(allItemsFromJobs, (progress) => {
// Update overall progress based on worker feedback
let detailMessage = progress.message || '';
if (progress.status === 'processing' && progress.total && progress.processed !== undefined) {
detailMessage = `Vectorizing: ${progress.processed} / ${progress.total}`;
// You could potentially update the 'completed' count more granularly here
// For simplicity, we'll just update the detail message
} else if (progress.status === 'complete') {
detailMessage = "Vectorization complete";
// Mark the vectorization step as complete
dispatchProgress(totalSteps, totalSteps, true, "Vectorization finished");
} else if (progress.status === 'error') {
detailMessage = `Vectorization error: ${progress.message}`;
dispatchProgress(completedJobs, totalSteps, true, "Vectorization failed", detailMessage); // Show error
} else if (progress.status === 'started') {
detailMessage = `Vectorization started for ${progress.total} items`;
} else if (progress.status === 'cancelled') {
detailMessage = `Vectorization cancelled: ${progress.message}`;
dispatchProgress(completedJobs, totalSteps, true, "Vectorization cancelled", detailMessage);
}
// Update the status detail
dispatchProgress(completedJobs, totalSteps, true, "Vectorization in progress", detailMessage);
// When worker signals completion of *its* task, mark the final step complete
if (progress.status === 'complete') {
completedJobs++; // Increment completion count *after* vectorization finishes
dispatchProgress(completedJobs, totalSteps, false, "Indexing finished"); // Set indexing to false
} else if (progress.status === 'error' || progress.status === 'cancelled') {
// Don't increment completed count on failure/cancel, just stop indexing indicator
dispatchProgress(completedJobs, totalSteps, false, "Indexing stopped due to error/cancel");
}
});
console.debug("%c[Indexer] Vectorization task sent to worker.", "color: green");
// Note: runIndexing might return *before* vectorization is complete now.
// The progress updates will signal the true end state.
} catch (error) {
console.error(`%c[Indexer] ❌ Failed to send items to vector worker:`, "color: red", error);
dispatchProgress(completedJobs, totalSteps, false, "Vectorization failed", String(error)); // Stop indexing indicator
}
} else {
console.debug("%c[Indexer] No items to send for vectorization.", "color: gray");
// If no vectorization needed, indexing is done here.
completedJobs++; // Count the "skipped" vectorization step
dispatchProgress(completedJobs, totalSteps, false, "Indexing finished (no vectorization needed)");
}
// Stop heartbeat ONLY when all jobs *and* the vectorization dispatch are done.
// The actual *completion* of vectorization is now asynchronous.
stopHeartbeat();
dispatchProgress(completedJobs, jobIds.length, false);
// Final progress update might be handled by the worker callback now.
// dispatchProgress(completedJobs, totalSteps, false); // This might be premature
}
function mergeItems(existing: IndexItem[], incoming: IndexItem[]): IndexItem[] {
const map = new Map<string, IndexItem>();
for (const item of existing) map.set(item.id, item);
for (const item of incoming) map.set(item.id, item);
// Prioritize incoming items if IDs clash
for (const item of existing) {
if (item && item.id) map.set(item.id, item);
}
for (const item of incoming) {
if (item && item.id) map.set(item.id, item);
}
return Array.from(map.values());
}
src/plugins/built-in/globalSearch/indexing/worker/vectorWorkerManager.ts
+131 -42
View File
@@ -3,7 +3,7 @@ import vectorWorker from './vectorWorker.ts?inlineWorker';
import type { SearchResult } from 'client-vector-search';
export type ProgressCallback = (data: {
status: 'started' | 'processing' | 'complete';
status: 'started' | 'processing' | 'complete' | 'error' | 'cancelled';
total?: number;
processed?: number;
message?: string;
@@ -13,10 +13,17 @@ export class VectorWorkerManager {
private static instance: VectorWorkerManager;
private worker: Worker | null = null;
private isInitialized = false;
private readyPromise: Promise<void> | null = null; // To await initialization
private progressCallback: ProgressCallback | null = null;
private searchPromises = new Map<string, { resolve: (value: SearchResult[]) => void, reject: (reason?: any) => void }>();
private searchPromises = new Map<string, { resolve: (value: SearchResult[]) => void, reject: (reason?: any) => void, timer: NodeJS.Timeout }>();
private debounceTimer: NodeJS.Timeout | null = null;
private lastSearchParams: { query: string; topK: number; resolve: (results: SearchResult[]) => void, reject: (reason?: any) => void } | null = null;
private constructor() {}
private constructor() {
// Start initialization immediately, but allow awaiting it
this.readyPromise = this.initWorker();
}
static getInstance(): VectorWorkerManager {
if (!VectorWorkerManager.instance) {
@@ -25,21 +32,35 @@ export class VectorWorkerManager {
return VectorWorkerManager.instance;
}
async init() {
if (this.isInitialized) return;
private async initWorker(): Promise<void> {
// If already initialized or initializing, return the existing promise
if (this.isInitialized) return Promise.resolve();
if (this.readyPromise) return this.readyPromise;
return new Promise<void>((resolve, reject) => {
// Create the worker
this.worker = vectorWorker();
const timeout = setTimeout(() => {
console.error('Vector worker initialization timed out');
this.worker?.terminate(); // Clean up worker if it exists
this.worker = null;
this.isInitialized = false; // Ensure state reflects failure
this.readyPromise = null; // Allow retrying init later
reject(new Error('Worker initialization timed out'));
}, 10000); // Increased timeout
// Set up message handling
this.worker.addEventListener('message', (e) => {
this.worker!.addEventListener('message', (e) => {
const { type, data } = e.data;
console.log(e);
console.debug("Message from vector worker:", type, data);
switch (type) {
case 'ready':
this.isInitialized = true;
console.debug('Vector worker initialized');
clearTimeout(timeout);
console.debug('Vector worker initialized and ready.');
resolve(); // Resolve the init promise
break;
case 'progress':
@@ -49,9 +70,10 @@ export class VectorWorkerManager {
break;
case 'searchResults':
const searchPromise = this.searchPromises.get(data.messageId);
if (searchPromise) {
searchPromise.resolve(data.results);
const searchInfo = this.searchPromises.get(data.messageId);
if (searchInfo) {
clearTimeout(searchInfo.timer); // Clear timeout on success
searchInfo.resolve(data.results);
this.searchPromises.delete(data.messageId);
} else {
console.warn('Received search results for unknown messageId:', data.messageId);
@@ -59,9 +81,10 @@ export class VectorWorkerManager {
break;
case 'searchError':
const errorPromise = this.searchPromises.get(data.messageId);
if (errorPromise) {
errorPromise.reject(new Error(data.error));
const errorInfo = this.searchPromises.get(data.messageId);
if (errorInfo) {
clearTimeout(errorInfo.timer); // Clear timeout on error
errorInfo.reject(new Error(data.error));
this.searchPromises.delete(data.messageId);
} else {
console.warn('Received search error for unknown messageId:', data.messageId);
@@ -69,9 +92,11 @@ export class VectorWorkerManager {
break;
case 'searchCancelled':
const cancelledPromise = this.searchPromises.get(data.messageId);
if (cancelledPromise) {
cancelledPromise.reject(new Error('Search cancelled'));
const cancelledInfo = this.searchPromises.get(data.messageId);
if (cancelledInfo) {
clearTimeout(cancelledInfo.timer); // Clear timeout on cancel
// Reject with a specific cancellation error or resolve with empty? Let's reject.
cancelledInfo.reject(new Error('Search cancelled by worker'));
this.searchPromises.delete(data.messageId);
} else {
console.debug('Received cancellation for unknown messageId:', data.messageId);
@@ -84,49 +109,113 @@ export class VectorWorkerManager {
});
// Initialize the worker
this.worker.postMessage({ type: 'init' });
// Wait for ready message
await new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Worker initialization timed out'));
}, 5000);
const checkInit = (e: MessageEvent) => {
if (e.data.type === 'ready') {
this.worker!.removeEventListener('message', checkInit);
clearTimeout(timeout);
resolve();
}
};
this.worker!.addEventListener('message', checkInit);
this.worker!.postMessage({ type: 'init' });
});
}
async processItems(items: HydratedIndexItem[], onProgress?: ProgressCallback) {
if (!this.isInitialized) {
await this.init();
// Ensures worker is ready before proceeding
private async ensureReady() {
if (!this.readyPromise) {
// If init wasn't called or failed, try again
console.warn("Worker not initialized, attempting init...");
this.readyPromise = this.initWorker();
}
await this.readyPromise;
if (!this.isInitialized || !this.worker) {
throw new Error("Vector Worker is not available after initialization attempt.");
}
}
async processItems(items: HydratedIndexItem[], onProgress?: ProgressCallback) {
await this.ensureReady(); // Wait for worker to be ready
this.progressCallback = onProgress || null;
// Cancel any ongoing search when starting processing
this.cancelAllSearches("Processing started");
console.debug(`Sending ${items.length} items to worker for processing.`);
this.worker!.postMessage({
type: 'process',
data: { items }
});
}
terminate() {
if (this.worker) {
// Clean up any pending promises
for (const [messageId, promise] of this.searchPromises.entries()) {
promise.reject(new Error('Worker terminated'));
// Public search method
public async search(query: string, topK: number = 10): Promise<SearchResult[]> {
await this.ensureReady();
return new Promise((resolve, reject) => {
this.lastSearchParams = { query, topK, resolve, reject };
const messageId = crypto.randomUUID();
if (this.lastSearchParams && this.worker) {
const currentParams = this.lastSearchParams; // Capture current params
this.lastSearchParams = null; // Clear last params *before* posting
this.debounceTimer = null;
// Set a timeout for the search operation itself
const searchTimeout = 10000; // e.g., 10 seconds
const searchTimer = setTimeout(() => {
if (this.searchPromises.has(messageId)) {
console.error(`Search timed out for messageId: ${messageId}`);
currentParams.reject(new Error(`Search timed out after ${searchTimeout}ms`));
this.searchPromises.delete(messageId);
}
}, searchTimeout);
this.searchPromises.set(messageId, { resolve: currentParams.resolve, reject: currentParams.reject, timer: searchTimer });
console.debug(`Sending search request (ID: ${messageId}) to worker: "${currentParams.query}"`);
this.worker.postMessage({
type: "search",
data: { query: currentParams.query, topK: currentParams.topK },
messageId
});
} else if (this.lastSearchParams) {
// This case might happen if ensureReady failed but didn't throw
console.error("Worker unavailable when trying to send search request.");
this.lastSearchParams.reject(new Error("Worker unavailable for search"));
this.lastSearchParams = null;
this.debounceTimer = null;
}
});
}
// Method to cancel all pending/debounced searches
private cancelAllSearches(reason: string = "Cancelled") {
if (this.debounceTimer) {
clearTimeout(this.debounceTimer);
this.debounceTimer = null;
if (this.lastSearchParams) {
this.lastSearchParams.reject(new Error(`Search cancelled: ${reason}`));
this.lastSearchParams = null;
}
}
// We might also want to tell the worker to cancel its *current* search
// if it supports it, but this requires worker modification.
// For now, just reject pending promises in the manager.
for (const [messageId, promiseInfo] of this.searchPromises.entries()) {
clearTimeout(promiseInfo.timer);
promiseInfo.reject(new Error(`Search cancelled: ${reason}`));
this.searchPromises.delete(messageId);
}
}
terminate() {
console.debug("Terminating Vector Worker Manager...");
this.cancelAllSearches("Worker terminated"); // Cancel pending searches
if (this.worker) {
this.worker.terminate();
this.worker = null;
}
this.isInitialized = false;
}
this.readyPromise = null; // Reset init promise
this.progressCallback = null;
// Clear the static instance? Or assume app lifecycle handles this?
// VectorWorkerManager.instance = null; // Uncomment if needed
}
}
src/plugins/built-in/globalSearch/search/vector/vectorSearch.ts
+25 -70
View File
@@ -1,87 +1,42 @@
import type { VectorSearchResult } from "./vectorTypes";
import vectorSearchWorker from "./vectorSearchWorker?inlineWorker";
/* import type { VectorSearchResult } from "./vectorTypes";
import { VectorWorkerManager } from '../../indexing/worker/vectorWorkerManager';
export function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
return VectorSearchWorkerManager.getInstance().search(query, topK);
/* return new Promise((resolve) => {
resolve([]);
}); */
// Use the single instance of the VectorWorkerManager (from indexing) to perform the search
return VectorWorkerManager.getInstance().search(query, topK);
}
*/
class VectorSearchWorkerManager {
private static instance: VectorSearchWorkerManager;
private worker: Worker | null = null;
private pendingSearches = new Map<string, (results: VectorSearchResult[]) => void>();
private debounceTimer: NodeJS.Timeout | null = null;
private lastSearchParams: { query: string; topK: number; resolve: (results: VectorSearchResult[]) => void } | null = null;
import { getEmbedding, EmbeddingIndex, initializeModel } from 'client-vector-search';
import type { HydratedIndexItem } from '../../indexing/types';
import type { SearchResult } from 'client-vector-search';
constructor() {
this.initWorker();
}
let vectorIndex: EmbeddingIndex | null = null;
private initWorker() {
export async function initVectorSearch() {
try {
this.worker = vectorSearchWorker();
this.worker.addEventListener('message', this.messageHandler);
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
vectorIndex.preloadIndexedDB();
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
console.error('Error initializing vector search', e);
}
}
private messageHandler = (e: MessageEvent) => {
console.log("Message received", e.data);
if (e.data.type === 'searchResults') {
const resolve = this.pendingSearches.get(e.data.data.messageId);
if (resolve) {
resolve(e.data.data.results);
this.pendingSearches.delete(e.data.data.messageId);
}
}
};
public static getInstance(): VectorSearchWorkerManager {
if (!VectorSearchWorkerManager.instance) {
VectorSearchWorkerManager.instance = new VectorSearchWorkerManager();
}
return VectorSearchWorkerManager.instance;
export interface VectorSearchResult extends SearchResult {
object: HydratedIndexItem & { embedding: number[] };
}
public async search(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
if (!this.worker) {
this.initWorker();
}
export async function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
if (!vectorIndex) await initVectorSearch();
return new Promise((resolve) => {
this.lastSearchParams = { query, topK, resolve };
if (this.debounceTimer) clearTimeout(this.debounceTimer);
this.debounceTimer = setTimeout(() => {
const messageId = crypto.randomUUID();
if (this.lastSearchParams) {
this.pendingSearches.set(messageId, this.lastSearchParams.resolve);
this.worker?.postMessage({
type: "search",
data: { query: this.lastSearchParams.query, topK: this.lastSearchParams.topK },
messageId
const queryEmbedding = await getEmbedding(query.slice(0, 100));
const results = await vectorIndex!.search(queryEmbedding, {
topK,
useStorage: 'indexedDB',
dedupeEntries: true
});
this.lastSearchParams = null;
}
this.debounceTimer = null;
}, query !== '' ? 300 : 0);
});
}
public terminate() {
if (this.worker) {
for (const [messageId, resolve] of this.pendingSearches.entries()) {
resolve([]);
this.pendingSearches.delete(messageId);
return results as VectorSearchResult[];
}
this.worker.terminate();
this.worker = null;
}
}
}
export default VectorSearchWorkerManager;
src/plugins/built-in/globalSearch/search/vector/vectorSearchWorker.ts
-56
View File
@@ -1,56 +0,0 @@
import { EmbeddingIndex, getEmbedding, initializeModel } from "client-vector-search";
import type { VectorSearchResult } from "./vectorTypes";
console.log("%cVector search worker initialized", "background-color: #000; color: #fff;");
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
async function initVectorSearch() {
if (isInitialized) return;
try {
await initializeModel();
vectorIndex = new EmbeddingIndex([]);
// Load existing items from IndexedDB
const stored = await vectorIndex.getAllObjectsFromIndexedDB();
if (stored.length > 0) {
stored.forEach((item) => vectorIndex!.add(item));
console.debug("Vector index loaded from IndexedDB");
}
isInitialized = true;
} catch (e) {
console.error("Failed to initialize vector search:", e);
throw e;
}
}
async function searchVectors(query: string, topK: number = 10): Promise<VectorSearchResult[]> {
if (!vectorIndex) await initVectorSearch();
const queryEmbedding = await getEmbedding(query);
const results = await vectorIndex!.search(queryEmbedding, {
topK,
useStorage: 'indexedDB'
});
return results as VectorSearchResult[];
}
self.addEventListener('message', async (e) => {
const { type, data, messageId } = e.data;
switch (type) {
case 'search':
console.log("Search request received", data);
searchVectors(data.query, data.topK).then((results) => {
self.postMessage({ type: 'searchResults', data: { messageId, results } });
});
break;
default:
console.warn(`Unknown message type: ${type}`);
}
});
initVectorSearch();
src/plugins/built-in/globalSearch/searchUtils.ts
+5 -1
View File
@@ -4,6 +4,7 @@ import { getDynamicItems } from "./dynamicSearch";
import type { CombinedResult } from "./core/types";
import type { HydratedIndexItem } from "./indexing/types";
import { searchVectors } from "./search/vector/vectorSearch";
import type { VectorSearchResult } from "./search/vector/vectorTypes";
export function createSearchIndexes() {
const commands = getStaticCommands();
@@ -154,7 +155,10 @@ export async function performSearch(
const fuseEndTime = performance.now();
// Get vector results in parallel
const vectorResults = await searchVectors(query, 10);
let vectorResults: VectorSearchResult[] = [];
try {
vectorResults = await searchVectors(query, 10);
} catch (e) {}
const vectorEndTime = performance.now();
console.log("Vector results:", vectorResults);
tsconfig.json
+1 -1
View File
@@ -33,5 +33,5 @@
"node"
]
},
"include": ["src/**/*.ts", "src/**/*.js", "src/**/*.svelte", "src/interface/+layout.sveltes"]
"include": ["src/**/*.ts", "src/**/*.js", "src/**/*.svelte", "src/interface/+layout.svelte", "declarations.d.ts"]
}