From 955213d5777a3756e9361c1a791bd3dfe836287e Mon Sep 17 00:00:00 2001 From: SethBurkart123 Date: Sun, 4 May 2025 12:01:03 +1000 Subject: [PATCH] fix: indexer not saving vectorized items properly --- package.json | 2 +- src/plugins/built-in/globalSearch/src/core/index.ts | 7 +++++++ .../built-in/globalSearch/src/indexing/indexer.ts | 9 ++++++--- src/plugins/built-in/globalSearch/src/indexing/jobs.ts | 5 ++--- .../globalSearch/src/indexing/renderComponents.ts | 2 +- .../globalSearch/src/indexing/worker/vectorWorker.ts | 1 - .../src/indexing/worker/vectorWorkerManager.ts | 8 +++++++- 7 files changed, 24 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 1f41338d..1f5e4dec 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "codemirror": "^6.0.1", "color": "^5.0.0", "dompurify": "^3.2.4", - "embeddia": "^1.1.0", + "embeddia": "^1.1.3", "embla-carousel-autoplay": "^8.5.2", "embla-carousel-svelte": "^8.5.2", "events": "^3.3.0", diff --git a/src/plugins/built-in/globalSearch/src/core/index.ts b/src/plugins/built-in/globalSearch/src/core/index.ts index a27d0b1e..fc0e16d1 100644 --- a/src/plugins/built-in/globalSearch/src/core/index.ts +++ b/src/plugins/built-in/globalSearch/src/core/index.ts @@ -11,6 +11,7 @@ import { waitForElm } from "@/seqta/utils/waitForElm"; import { runIndexing } from "../indexing/indexer"; import { initVectorSearch } from "../search/vector/vectorSearch"; import { cleanupSearchBar, mountSearchBar } from "./mountSearchBar"; +import { IndexedDbManager } from 'embeddia'; const settings = defineSettings({ searchHotkey: stringSetting({ @@ -64,6 +65,12 @@ const globalSearchPlugin: Plugin = { run: async (api) => { const appRef = { current: null }; + await IndexedDbManager.create( + 'embeddiaDB', + 'embeddiaObjectStore', + { primaryKey: 'id', autoIncrement: false } + ); + initVectorSearch(); if (api.settings.runIndexingOnLoad) { diff --git a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts index ff7d917f..7d12084c 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts @@ -170,7 +170,10 @@ export async function runIndexing(): Promise { let merged = mergeItems(stored, newItemsRaw); if (job.purge) merged = job.purge(merged); - await setStoredItems(merged); // Store merged non-vector data + console.log(merged); + console.log(merged.length); + + await setStoredItems(merged); await updateLastRunMeta(jobId); // Hydrate items for vector processing @@ -192,11 +195,11 @@ export async function runIndexing(): Promise { allItemsFromJobs.push(...hydratedItems); console.debug( - `%c[Indexer] ✅ ${job.label}: ${newItemsRaw.length} new items fetched, ${merged.length} total stored (non-vector).`, + `%c[Indexer] ${job.label}: ${newItemsRaw.length} new items fetched, ${merged.length} total stored (non-vector).`, "color: #00c46f", ); } catch (err) { - console.debug(`%c[Indexer] ❌ ${job.label} failed:`, "color: red"); + console.debug(`%c[Indexer] ${job.label} failed:`, "color: red"); console.error(err); } diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts index a18d614f..1ecc1675 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts @@ -155,7 +155,7 @@ export const jobs: Record = { id: "messages", label: "Messages", renderComponentId: "message", - frequency: { type: "expiry", afterMs: 1000 * 60 * 5 }, // every 5 minutes + frequency: { type: "expiry", afterMs: 1000 }, // every 5 minutes run: async (ctx) => { // Get existing items first @@ -258,8 +258,7 @@ export const jobs: Record = { }, purge: (items) => { - // Keep messages from the last 30 days - const cutoff = Date.now() - 30 * 24 * 60 * 60 * 1000; + const cutoff = Date.now() - 4 * 12 * 30 * 24 * 60 * 60 * 1000; return items.filter((i) => i.dateAdded >= cutoff); }, }, diff --git a/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts b/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts index 2dbb7b08..e0c45651 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/renderComponents.ts @@ -4,7 +4,7 @@ import AssessmentComponent from "../components/AssessmentItem.svelte"; export const renderComponentMap: Record = { assessment: AssessmentComponent as unknown as typeof SvelteComponent, - // messages: MessageComponent, + message: AssessmentComponent as unknown as typeof SvelteComponent, // subject: SubjectComponent, // etc... }; diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts index 51748800..fa3742a7 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts @@ -86,7 +86,6 @@ async function processItems(items: HydratedIndexItem[], signal: AbortSignal) { const unprocessedItems = items.filter((item) => { if (signal.aborted) return false; // Check cancellation during filtering try { - // Check if the item ID already exists in the index (loaded or added) return !vectorIndex!.get({ id: item.id }); } catch (e) { // If get throws (e.g., item not found), it means it's unprocessed diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts index ee3d7960..708b3f33 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts @@ -42,6 +42,8 @@ export class VectorWorkerManager { // Create the worker this.worker = vectorWorker(); + console.log('Worker initialized', this.worker); + const timeout = setTimeout(() => { console.error('Vector worker initialization timed out'); this.worker?.terminate(); // Clean up worker if it exists @@ -140,9 +142,12 @@ export class VectorWorkerManager { this.cancelAllSearches("Processing started"); console.debug(`Sending ${items.length} items to worker for processing.`); + + const serialisableItems = items.map(({ renderComponent, ...rest }) => rest); + this.worker!.postMessage({ type: 'process', - data: { items } + data: { items: serialisableItems } }); } @@ -173,6 +178,7 @@ export class VectorWorkerManager { this.searchPromises.set(messageId, { resolve: currentParams.resolve, reject: currentParams.reject, timer: searchTimer }); console.debug(`Sending search request (ID: ${messageId}) to worker: "${currentParams.query}"`); + console.log(this.worker); this.worker.postMessage({ type: "search", data: { query: currentParams.query, topK: currentParams.topK },