diff --git a/src/plugins/built-in/globalSearch/src/core/index.ts b/src/plugins/built-in/globalSearch/src/core/index.ts index 35ae03ba..cf6889ce 100644 --- a/src/plugins/built-in/globalSearch/src/core/index.ts +++ b/src/plugins/built-in/globalSearch/src/core/index.ts @@ -129,11 +129,7 @@ const globalSearchPlugin: Plugin = { // Warm up vector worker in background to improve initial response time setTimeout(async () => { try { - const workerManager = VectorWorkerManager.getInstance(); - console.debug("[Global Search] Warming up vector worker..."); - // Just ensure the worker is ready, don't process anything yet - await workerManager.processItems([], () => {}); - console.debug("[Global Search] Vector worker warmed up successfully"); + VectorWorkerManager.getInstance(); } catch (error) { console.warn("[Global Search] Vector worker warm-up failed:", error); } diff --git a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts index 93873bfd..de00fff7 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts @@ -4,6 +4,7 @@ import { renderComponentMap } from "./renderComponents"; import type { IndexItem, Job, JobContext } from "./types"; import { VectorWorkerManager } from "./worker/vectorWorkerManager"; import { loadDynamicItems } from "../utils/dynamicItems"; +import { getVectorizedItemIds } from "./utils"; const META_STORE = "meta"; const LOCK_KEY = "bsq-indexer-lock"; @@ -280,14 +281,24 @@ export async function runIndexing(): Promise { if (allItemsInPrimaryStores.length > 0) { console.debug( - `%c[Indexer] Sending ${allItemsInPrimaryStores.length} items from primary stores to worker for vectorization check...`, + `%c[Indexer] Checking ${allItemsInPrimaryStores.length} items for vectorization...`, "color: #4ea1ff", ); - dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization of stored items"); + + // Pre-filter items to avoid initializing worker if nothing new + const vectorizedItemIds = await getVectorizedItemIds(); + const newItemsToVectorize = allItemsInPrimaryStores.filter(item => !vectorizedItemIds.has(item.id)); + + if (newItemsToVectorize.length > 0) { + console.debug( + `%c[Indexer] Sending ${newItemsToVectorize.length} new items to worker for vectorization (${allItemsInPrimaryStores.length - newItemsToVectorize.length} already vectorized)`, + "color: #4ea1ff", + ); + dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization of new items"); - try { - const workerManager = VectorWorkerManager.getInstance(); - await workerManager.processItems(allItemsInPrimaryStores, (progress) => { + try { + const workerManager = VectorWorkerManager.getInstance(); + await workerManager.processItems(newItemsToVectorize, (progress) => { let detailMessage = progress.message || ""; if ( progress.status === "processing" && @@ -355,6 +366,19 @@ export async function runIndexing(): Promise { String(error), ); } + } else { + console.debug( + `%c[Indexer] All ${allItemsInPrimaryStores.length} items are already vectorized, skipping worker initialization.`, + "color: gray", + ); + completedJobs++; + dispatchProgress( + completedJobs, + totalSteps, + false, + "Indexing finished (all items already vectorized)", + ); + } } else { console.debug( "%c[Indexer] No items found in primary stores to send for vectorization.", diff --git a/src/plugins/built-in/globalSearch/src/indexing/utils.ts b/src/plugins/built-in/globalSearch/src/indexing/utils.ts index 6faba2fe..a5365dfa 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/utils.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/utils.ts @@ -1,3 +1,58 @@ +/** + * Check which items are already vectorized in embeddia's IndexedDB + * Returns a Set of item IDs that are already indexed + */ +export async function getVectorizedItemIds(): Promise> { + return new Promise((resolve) => { + const request = indexedDB.open("embeddiaDB"); + + request.onerror = () => { + console.debug("Could not open embeddiaDB, assuming no items are vectorized"); + resolve(new Set()); + }; + + request.onsuccess = (event) => { + const db = (event.target as IDBOpenDBRequest).result; + + if (!db.objectStoreNames.contains("embeddiaObjectStore")) { + console.debug("embeddiaObjectStore not found, assuming no items are vectorized"); + db.close(); + resolve(new Set()); + return; + } + + try { + const transaction = db.transaction(["embeddiaObjectStore"], "readonly"); + const store = transaction.objectStore("embeddiaObjectStore"); + const getAllRequest = store.getAllKeys(); + + getAllRequest.onsuccess = () => { + const vectorizedIds = new Set(); + getAllRequest.result.forEach(key => { + if (typeof key === 'string') { + vectorizedIds.add(key); + } + }); + + console.debug(`Found ${vectorizedIds.size} already vectorized items in embeddia DB`); + db.close(); + resolve(vectorizedIds); + }; + + getAllRequest.onerror = () => { + console.warn("Error reading vectorized item keys, assuming no items are vectorized"); + db.close(); + resolve(new Set()); + }; + } catch (error) { + console.warn("Error accessing embeddia store, assuming no items are vectorized:", error); + db.close(); + resolve(new Set()); + } + }; + }); +} + export function htmlToPlainText(rawHtml: string): string { const parser = new DOMParser(); const doc = parser.parseFromString(rawHtml, "text/html"); diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts index 71f2f252..b99320f6 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts @@ -18,6 +18,7 @@ export class VectorWorkerManager { private initializationMutex = false; private idleTimer: NodeJS.Timeout | null = null; private lastActivityTime = 0; + private unloadTimer: NodeJS.Timeout | null = null; private streamingSession: { isActive: boolean; @@ -104,6 +105,10 @@ export class VectorWorkerManager { }), ); } + + if (data.status === "complete" || data.status === "cancelled" || data.status === "error") { + this.scheduleUnload(); + } } break; @@ -139,6 +144,7 @@ export class VectorWorkerManager { this.progressCallback = null; this.initializationMutex = false; this.clearIdleTimer(); + this.clearUnloadTimer(); if (this.streamingSession?.isActive) { this.endStreamingSession(); } @@ -161,8 +167,26 @@ export class VectorWorkerManager { } } + private clearUnloadTimer() { + if (this.unloadTimer) { + clearTimeout(this.unloadTimer); + this.unloadTimer = null; + } + } + + private scheduleUnload(delay: number = 10000) { + this.clearUnloadTimer(); + this.unloadTimer = setTimeout(() => { + if (!this.streamingSession?.isActive && this.isInitialized) { + console.debug("[VectorWorker] Auto-unloading after processing complete"); + this.resetWorkerState(); + } + }, delay); + } + private updateActivity() { this.lastActivityTime = Date.now(); + this.clearUnloadTimer(); this.startIdleTimer(); } @@ -449,6 +473,7 @@ export class VectorWorkerManager { } this.streamingSession = null; + this.scheduleUnload(); } async streamItem(item: IndexItem): Promise {