mirror of
https://github.com/BetterSEQTA/BetterSEQTA-Plus.git
synced 2026-06-06 03:34:40 +00:00
perf: only load vectorWorker when required
This commit is contained in:
@@ -129,11 +129,7 @@ const globalSearchPlugin: Plugin<typeof settings> = {
|
|||||||
// Warm up vector worker in background to improve initial response time
|
// Warm up vector worker in background to improve initial response time
|
||||||
setTimeout(async () => {
|
setTimeout(async () => {
|
||||||
try {
|
try {
|
||||||
const workerManager = VectorWorkerManager.getInstance();
|
VectorWorkerManager.getInstance();
|
||||||
console.debug("[Global Search] Warming up vector worker...");
|
|
||||||
// Just ensure the worker is ready, don't process anything yet
|
|
||||||
await workerManager.processItems([], () => {});
|
|
||||||
console.debug("[Global Search] Vector worker warmed up successfully");
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("[Global Search] Vector worker warm-up failed:", error);
|
console.warn("[Global Search] Vector worker warm-up failed:", error);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { renderComponentMap } from "./renderComponents";
|
|||||||
import type { IndexItem, Job, JobContext } from "./types";
|
import type { IndexItem, Job, JobContext } from "./types";
|
||||||
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
|
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
|
||||||
import { loadDynamicItems } from "../utils/dynamicItems";
|
import { loadDynamicItems } from "../utils/dynamicItems";
|
||||||
|
import { getVectorizedItemIds } from "./utils";
|
||||||
|
|
||||||
const META_STORE = "meta";
|
const META_STORE = "meta";
|
||||||
const LOCK_KEY = "bsq-indexer-lock";
|
const LOCK_KEY = "bsq-indexer-lock";
|
||||||
@@ -280,14 +281,24 @@ export async function runIndexing(): Promise<void> {
|
|||||||
|
|
||||||
if (allItemsInPrimaryStores.length > 0) {
|
if (allItemsInPrimaryStores.length > 0) {
|
||||||
console.debug(
|
console.debug(
|
||||||
`%c[Indexer] Sending ${allItemsInPrimaryStores.length} items from primary stores to worker for vectorization check...`,
|
`%c[Indexer] Checking ${allItemsInPrimaryStores.length} items for vectorization...`,
|
||||||
"color: #4ea1ff",
|
"color: #4ea1ff",
|
||||||
);
|
);
|
||||||
dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization of stored items");
|
|
||||||
|
// Pre-filter items to avoid initializing worker if nothing new
|
||||||
|
const vectorizedItemIds = await getVectorizedItemIds();
|
||||||
|
const newItemsToVectorize = allItemsInPrimaryStores.filter(item => !vectorizedItemIds.has(item.id));
|
||||||
|
|
||||||
|
if (newItemsToVectorize.length > 0) {
|
||||||
|
console.debug(
|
||||||
|
`%c[Indexer] Sending ${newItemsToVectorize.length} new items to worker for vectorization (${allItemsInPrimaryStores.length - newItemsToVectorize.length} already vectorized)`,
|
||||||
|
"color: #4ea1ff",
|
||||||
|
);
|
||||||
|
dispatchProgress(completedJobs, totalSteps, true, "Starting vectorization of new items");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const workerManager = VectorWorkerManager.getInstance();
|
const workerManager = VectorWorkerManager.getInstance();
|
||||||
await workerManager.processItems(allItemsInPrimaryStores, (progress) => {
|
await workerManager.processItems(newItemsToVectorize, (progress) => {
|
||||||
let detailMessage = progress.message || "";
|
let detailMessage = progress.message || "";
|
||||||
if (
|
if (
|
||||||
progress.status === "processing" &&
|
progress.status === "processing" &&
|
||||||
@@ -355,6 +366,19 @@ export async function runIndexing(): Promise<void> {
|
|||||||
String(error),
|
String(error),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
console.debug(
|
||||||
|
`%c[Indexer] All ${allItemsInPrimaryStores.length} items are already vectorized, skipping worker initialization.`,
|
||||||
|
"color: gray",
|
||||||
|
);
|
||||||
|
completedJobs++;
|
||||||
|
dispatchProgress(
|
||||||
|
completedJobs,
|
||||||
|
totalSteps,
|
||||||
|
false,
|
||||||
|
"Indexing finished (all items already vectorized)",
|
||||||
|
);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
console.debug(
|
console.debug(
|
||||||
"%c[Indexer] No items found in primary stores to send for vectorization.",
|
"%c[Indexer] No items found in primary stores to send for vectorization.",
|
||||||
|
|||||||
@@ -1,3 +1,58 @@
|
|||||||
|
/**
|
||||||
|
* Check which items are already vectorized in embeddia's IndexedDB
|
||||||
|
* Returns a Set of item IDs that are already indexed
|
||||||
|
*/
|
||||||
|
export async function getVectorizedItemIds(): Promise<Set<string>> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const request = indexedDB.open("embeddiaDB");
|
||||||
|
|
||||||
|
request.onerror = () => {
|
||||||
|
console.debug("Could not open embeddiaDB, assuming no items are vectorized");
|
||||||
|
resolve(new Set());
|
||||||
|
};
|
||||||
|
|
||||||
|
request.onsuccess = (event) => {
|
||||||
|
const db = (event.target as IDBOpenDBRequest).result;
|
||||||
|
|
||||||
|
if (!db.objectStoreNames.contains("embeddiaObjectStore")) {
|
||||||
|
console.debug("embeddiaObjectStore not found, assuming no items are vectorized");
|
||||||
|
db.close();
|
||||||
|
resolve(new Set());
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const transaction = db.transaction(["embeddiaObjectStore"], "readonly");
|
||||||
|
const store = transaction.objectStore("embeddiaObjectStore");
|
||||||
|
const getAllRequest = store.getAllKeys();
|
||||||
|
|
||||||
|
getAllRequest.onsuccess = () => {
|
||||||
|
const vectorizedIds = new Set<string>();
|
||||||
|
getAllRequest.result.forEach(key => {
|
||||||
|
if (typeof key === 'string') {
|
||||||
|
vectorizedIds.add(key);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
console.debug(`Found ${vectorizedIds.size} already vectorized items in embeddia DB`);
|
||||||
|
db.close();
|
||||||
|
resolve(vectorizedIds);
|
||||||
|
};
|
||||||
|
|
||||||
|
getAllRequest.onerror = () => {
|
||||||
|
console.warn("Error reading vectorized item keys, assuming no items are vectorized");
|
||||||
|
db.close();
|
||||||
|
resolve(new Set());
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("Error accessing embeddia store, assuming no items are vectorized:", error);
|
||||||
|
db.close();
|
||||||
|
resolve(new Set());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function htmlToPlainText(rawHtml: string): string {
|
export function htmlToPlainText(rawHtml: string): string {
|
||||||
const parser = new DOMParser();
|
const parser = new DOMParser();
|
||||||
const doc = parser.parseFromString(rawHtml, "text/html");
|
const doc = parser.parseFromString(rawHtml, "text/html");
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ export class VectorWorkerManager {
|
|||||||
private initializationMutex = false;
|
private initializationMutex = false;
|
||||||
private idleTimer: NodeJS.Timeout | null = null;
|
private idleTimer: NodeJS.Timeout | null = null;
|
||||||
private lastActivityTime = 0;
|
private lastActivityTime = 0;
|
||||||
|
private unloadTimer: NodeJS.Timeout | null = null;
|
||||||
|
|
||||||
private streamingSession: {
|
private streamingSession: {
|
||||||
isActive: boolean;
|
isActive: boolean;
|
||||||
@@ -104,6 +105,10 @@ export class VectorWorkerManager {
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (data.status === "complete" || data.status === "cancelled" || data.status === "error") {
|
||||||
|
this.scheduleUnload();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
@@ -139,6 +144,7 @@ export class VectorWorkerManager {
|
|||||||
this.progressCallback = null;
|
this.progressCallback = null;
|
||||||
this.initializationMutex = false;
|
this.initializationMutex = false;
|
||||||
this.clearIdleTimer();
|
this.clearIdleTimer();
|
||||||
|
this.clearUnloadTimer();
|
||||||
if (this.streamingSession?.isActive) {
|
if (this.streamingSession?.isActive) {
|
||||||
this.endStreamingSession();
|
this.endStreamingSession();
|
||||||
}
|
}
|
||||||
@@ -161,8 +167,26 @@ export class VectorWorkerManager {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private clearUnloadTimer() {
|
||||||
|
if (this.unloadTimer) {
|
||||||
|
clearTimeout(this.unloadTimer);
|
||||||
|
this.unloadTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private scheduleUnload(delay: number = 10000) {
|
||||||
|
this.clearUnloadTimer();
|
||||||
|
this.unloadTimer = setTimeout(() => {
|
||||||
|
if (!this.streamingSession?.isActive && this.isInitialized) {
|
||||||
|
console.debug("[VectorWorker] Auto-unloading after processing complete");
|
||||||
|
this.resetWorkerState();
|
||||||
|
}
|
||||||
|
}, delay);
|
||||||
|
}
|
||||||
|
|
||||||
private updateActivity() {
|
private updateActivity() {
|
||||||
this.lastActivityTime = Date.now();
|
this.lastActivityTime = Date.now();
|
||||||
|
this.clearUnloadTimer();
|
||||||
this.startIdleTimer();
|
this.startIdleTimer();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -449,6 +473,7 @@ export class VectorWorkerManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
this.streamingSession = null;
|
this.streamingSession = null;
|
||||||
|
this.scheduleUnload();
|
||||||
}
|
}
|
||||||
|
|
||||||
async streamItem(item: IndexItem): Promise<void> {
|
async streamItem(item: IndexItem): Promise<void> {
|
||||||
|
|||||||
Reference in New Issue
Block a user