diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..8c1d73ae --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +legacy-peer-deps=true + diff --git a/package.json b/package.json index 9632d079..fb0bf251 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,10 @@ "description": "Enhance SEQTA Learn's usability and aesthetics! A fork of BetterSEQTA to continue development add add heaps more features!", "browserslist": "> 0.5%, last 2 versions, not dead", "scripts": { + "autoaudit": "npm audit && npm audit fix && npm run build", "dev": "cross-env MODE=chrome vite dev", "dev:firefox": "cross-env MODE=firefox vite build --watch", + "compile": "npm i && npm run build", "build": "cross-env MODE=chrome vite build && cross-env MODE=firefox vite build", "build:chrome": "cross-env MODE=chrome vite build", "build:firefox": "cross-env MODE=firefox vite build", diff --git a/src/css/injected.scss b/src/css/injected.scss index f0c00395..18dae27d 100644 --- a/src/css/injected.scss +++ b/src/css/injected.scss @@ -9,7 +9,7 @@ background: var(--better-main) !important; --navy: #1a1a1a !important; --auto-background: var(--better-pale, var(--background-secondary)) !important; - font-family: Rubik, sans-serif !important; + font-family: Rubik, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; } ::view-transition-old(root), @@ -36,7 +36,7 @@ body, .legacy-root option, .legacy-root .input, html { - font-family: Rubik, sans-serif !important; + font-family: Rubik, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; } select option { @@ -56,7 +56,7 @@ select { background: var(--auto-background) !important; } :root * { - font-family: Rubik, sans-serif !important; + font-family: Rubik, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; --theme-fg-parts: white; } .extension-editor { @@ -302,7 +302,7 @@ select { .material-icons { font-size: 0px !important; - font-family: Rubik, sans-serif !important; + font-family: Rubik, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; &::before { font-size: 18px !important; content: "Search" !important; @@ -413,7 +413,7 @@ ul.magicDelete > li.deleting { background: var(--better-main) !important; color: var(--text-color); border-right: none; - font-family: Rubik, sans-serif !important; + font-family: Rubik, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif !important; } #menu li > label > svg, #menu section > label > svg { diff --git a/src/plugins/built-in/globalSearch/lazy.ts b/src/plugins/built-in/globalSearch/lazy.ts index 793d42fc..7e50f7d5 100644 --- a/src/plugins/built-in/globalSearch/lazy.ts +++ b/src/plugins/built-in/globalSearch/lazy.ts @@ -42,32 +42,69 @@ const settings = defineSettings({ if (confirmed) { try { - // Dynamically import the worker manager to avoid loading heavy dependencies + // Dynamically import modules to avoid loading heavy dependencies const { VectorWorkerManager } = await import("./src/indexing/worker/vectorWorkerManager"); - const workerManager = VectorWorkerManager.getInstance(); - await workerManager.resetWorker(); - console.log("Vector worker reset successfully"); - } catch (e) { - console.warn("Failed to reset vector worker:", e); - } + const { resetDatabase } = await import("./src/indexing/db"); + + // Reset vector worker first + try { + const workerManager = VectorWorkerManager.getInstance(); + await workerManager.resetWorker(); + console.log("Vector worker reset successfully"); + } catch (e) { + console.warn("Failed to reset vector worker:", e); + } - // Delete both 'embeddiaDB' and 'betterseqta-index' using native IndexedDB APIs - const deleteDb = (dbName: string) => { - return new Promise((resolve, reject) => { - const req = indexedDB.deleteDatabase(dbName); - req.onsuccess = () => resolve(); - req.onerror = () => reject(req.error); - req.onblocked = () => { - reject(new Error(`One database is open, failed to remove: ${dbName}`)); - }; - }); - }; - try { - await deleteDb("embeddiaDB"); - await deleteDb("betterseqta-index"); - alert("Search index and storage have been reset."); + // Close all database connections properly before deletion + try { + await resetDatabase(); + console.log("betterseqta-index database closed and reset"); + } catch (e) { + console.warn("Failed to reset betterseqta-index database:", e); + } + + // Wait a bit for connections to fully close + await new Promise(resolve => setTimeout(resolve, 100)); + + // Delete embeddiaDB (vector search database) + const deleteDb = (dbName: string) => { + return new Promise((resolve, reject) => { + const req = indexedDB.deleteDatabase(dbName); + req.onsuccess = () => { + console.log(`Successfully deleted database: ${dbName}`); + resolve(); + }; + req.onerror = () => { + console.error(`Error deleting database ${dbName}:`, req.error); + reject(req.error); + }; + req.onblocked = () => { + console.warn(`Database ${dbName} deletion blocked - connections still open`); + // Wait and retry once + setTimeout(() => { + const retryReq = indexedDB.deleteDatabase(dbName); + retryReq.onsuccess = () => { + console.log(`Successfully deleted database on retry: ${dbName}`); + resolve(); + }; + retryReq.onerror = () => reject(retryReq.error); + retryReq.onblocked = () => { + reject(new Error(`One database is open, failed to remove: ${dbName}. Please close other tabs and try again.`)); + }; + }, 500); + }; + }); + }; + + try { + await deleteDb("embeddiaDB"); + await deleteDb("betterseqta-index"); + alert("Search index and storage have been reset successfully."); + } catch (e) { + alert("Failed to reset one or more databases: " + String(e) + "\n\nTry closing other browser tabs and try again."); + } } catch (e) { - alert("Failed to reset one or more databases: " + String(e)); + alert("Failed to reset index: " + String(e)); } } }, diff --git a/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte index f02d91c8..912622f0 100644 --- a/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte @@ -35,6 +35,8 @@ let isIndexing = $state(false); let completedJobs = $state(0); let totalJobs = $state(0); + let indexingStatus = $state(null); + let indexingDetail = $state(null); let commandPalleteOpen = $state(false); let searchTerm = $state(''); @@ -110,10 +112,12 @@ onMount(() => { const progressHandler = (event: CustomEvent) => { - const { completed, total, indexing } = event.detail; + const { completed, total, indexing, status, detail } = event.detail; completedJobs = completed; totalJobs = total; isIndexing = indexing; + indexingStatus = status || null; + indexingDetail = detail || null; }; window.addEventListener('indexing-progress', progressHandler as EventListener); @@ -167,7 +171,10 @@ combinedResults = await doSearch( term, commandsFuse, - commandIdToItemMap, + commandIdToItemMap, + dynamicContentFuse, + dynamicIdToItemMap, + true, // sortByRecent ); } else { combinedResults = []; @@ -176,13 +183,19 @@ isLoading = false; }; - const debouncedPerformSearch = debounce(performSearch, 20); + // Optimized debounce: shorter delay for better responsiveness + const debouncedPerformSearch = debounce(performSearch, 50); $effect(() => { if (commandPalleteOpen) { if (searchTerm === '') { + // Immediate search for empty query (shows recent items) + performSearch(); + } else if (searchTerm.length <= 2) { + // Immediate search for very short queries performSearch(); } else { + // Debounced search for longer queries debouncedPerformSearch(); } tick().then(() => searchbar?.focus()); @@ -389,19 +402,6 @@ {@render Shortcut({ text: 'Select', keybind: ['↵']})} {/if} - {#if isIndexing} -
-
- Indexing -
-
-
-
-
- {/if} {/if} diff --git a/src/plugins/built-in/globalSearch/src/core/index.ts b/src/plugins/built-in/globalSearch/src/core/index.ts index f7859808..5890cd9c 100644 --- a/src/plugins/built-in/globalSearch/src/core/index.ts +++ b/src/plugins/built-in/globalSearch/src/core/index.ts @@ -14,6 +14,7 @@ import { initVectorSearch } from "../search/vector/vectorSearch"; import { cleanupSearchBar, mountSearchBar } from "./mountSearchBar"; import { IndexedDbManager } from "embeddia"; import { VectorWorkerManager } from "../indexing/worker/vectorWorkerManager"; +import { checkAndHandleUpdate } from "../utils/versionCheck"; // Platform-aware default hotkey const getDefaultHotkey = () => { @@ -50,31 +51,67 @@ const settings = defineSettings({ if (confirmed) { try { + // Import resetDatabase function to properly close connections + const { resetDatabase } = await import("../indexing/db"); + // Reset the vector worker first - const workerManager = VectorWorkerManager.getInstance(); - await workerManager.resetWorker(); - console.log("Vector worker reset successfully"); - } catch (e) { - console.warn("Failed to reset vector worker:", e); - } + try { + const workerManager = VectorWorkerManager.getInstance(); + await workerManager.resetWorker(); + console.log("Vector worker reset successfully"); + } catch (e) { + console.warn("Failed to reset vector worker:", e); + } - // Delete both 'embeddiaDB' and 'betterseqta-index' using native IndexedDB APIs - const deleteDb = (dbName: string) => { - return new Promise((resolve, reject) => { - const req = indexedDB.deleteDatabase(dbName); - req.onsuccess = () => resolve(); - req.onerror = () => reject(req.error); - req.onblocked = () => { - reject(new Error(`One database is open, failed to remove: ${dbName}`)); - }; - }); - }; - try { - await deleteDb("embeddiaDB"); - await deleteDb("betterseqta-index"); - alert("Search index and storage have been reset."); + // Close all database connections properly before deletion + try { + await resetDatabase(); + } catch (e) { + console.warn("Failed to reset betterseqta-index database:", e); + } + + // Wait a bit for connections to fully close + await new Promise(resolve => setTimeout(resolve, 100)); + + // Delete embeddiaDB (vector search database) + const deleteDb = (dbName: string) => { + return new Promise((resolve, reject) => { + const req = indexedDB.deleteDatabase(dbName); + req.onsuccess = () => { + console.log(`Successfully deleted database: ${dbName}`); + resolve(); + }; + req.onerror = () => { + console.error(`Error deleting database ${dbName}:`, req.error); + reject(req.error); + }; + req.onblocked = () => { + console.warn(`Database ${dbName} deletion blocked - connections still open`); + // Wait and retry once + setTimeout(() => { + const retryReq = indexedDB.deleteDatabase(dbName); + retryReq.onsuccess = () => { + console.log(`Successfully deleted database on retry: ${dbName}`); + resolve(); + }; + retryReq.onerror = () => reject(retryReq.error); + retryReq.onblocked = () => { + reject(new Error(`One database is open, failed to remove: ${dbName}. Please close other tabs and try again.`)); + }; + }, 500); + }; + }); + }; + + try { + await deleteDb("embeddiaDB"); + await deleteDb("betterseqta-index"); + alert("Search index and storage have been reset successfully."); + } catch (e) { + alert("Failed to reset one or more databases: " + String(e) + "\n\nTry closing other browser tabs and try again."); + } } catch (e) { - alert("Failed to reset one or more databases: " + String(e)); + alert("Failed to reset index: " + String(e)); } } }, @@ -114,6 +151,27 @@ const globalSearchPlugin: Plugin = { run: async (api) => { const appRef = { current: null }; + // Check for extension updates and clear caches if needed + // Use a timeout to avoid blocking initialization + setTimeout(async () => { + try { + const wasUpdated = await checkAndHandleUpdate(); + if (wasUpdated) { + console.log("[Global Search] Extension updated - caches cleared"); + } + } catch (error: any) { + // Handle CSS preload errors and other failures gracefully + // These can happen in Firefox or when assets aren't available + if (error?.message?.includes("preload CSS") || + error?.message?.includes("MIME type") || + error?.message?.includes("NS_ERROR_CORRUPTED_CONTENT")) { + console.debug("[Global Search] Version check skipped due to asset loading restrictions:", error.message); + } else { + console.warn("[Global Search] Failed to check for updates:", error); + } + } + }, 100); + try { await IndexedDbManager.create("embeddiaDB", "embeddiaObjectStore", { primaryKey: "id", @@ -126,10 +184,16 @@ const globalSearchPlugin: Plugin = { initVectorSearch(); - // Warm up vector worker in background to improve initial response time + // Warm up vector worker in background to improve initial response time (skip in Firefox) setTimeout(async () => { try { - VectorWorkerManager.getInstance(); + // Only initialize worker if vector search is supported + const { isVectorSearchSupported } = await import("../utils/browserDetection"); + if (isVectorSearchSupported()) { + VectorWorkerManager.getInstance(); + } else { + console.debug("[Global Search] Skipping vector worker warm-up (Firefox detected - using text search only)"); + } } catch (error) { console.warn("[Global Search] Vector worker warm-up failed:", error); } diff --git a/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts b/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts index 39e4ac5e..8a20a1a9 100644 --- a/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts +++ b/src/plugins/built-in/globalSearch/src/core/mountSearchBar.ts @@ -8,7 +8,7 @@ import browser from "webextension-polyfill"; export function mountSearchBar( titleElement: Element, api: any, - appRef: { current: any; storageChangeHandler?: any }, + appRef: { current: any; storageChangeHandler?: any; progressHandler?: any }, ) { if (titleElement.querySelector(".search-trigger")) { return; @@ -21,6 +21,72 @@ export function mountSearchBar( const searchButton = document.createElement("div"); searchButton.className = "search-trigger"; + // Create progress indicator container + const progressContainer = document.createElement("div"); + progressContainer.className = "search-progress-container"; + progressContainer.style.cssText = "display: flex; align-items: center; gap: 8px; margin-left: 8px; min-width: 120px;"; + + // Create progress bar + const progressBarWrapper = document.createElement("div"); + progressBarWrapper.className = "search-progress-bar-wrapper"; + progressBarWrapper.style.cssText = "flex: 1; height: 4px; background: rgba(0, 0, 0, 0.1); border-radius: 2px; overflow: hidden; display: none;"; + + const progressBar = document.createElement("div"); + progressBar.className = "search-progress-bar"; + progressBar.style.cssText = "height: 100%; background: linear-gradient(90deg, #3b82f6, #2563eb, #3b82f6); transition: width 0.3s ease-out; width: 0%; position: relative;"; + + // Add shimmer effect + const shimmer = document.createElement("div"); + shimmer.style.cssText = "position: absolute; inset: 0; background: linear-gradient(90deg, transparent, rgba(255,255,255,0.3), transparent); animation: shimmer 2s infinite;"; + progressBar.appendChild(shimmer); + progressBarWrapper.appendChild(progressBar); + + // Create progress text + const progressText = document.createElement("span"); + progressText.className = "search-progress-text"; + progressText.style.cssText = "font-size: 11px; color: #666; white-space: nowrap; display: none;"; + + progressContainer.appendChild(progressBarWrapper); + progressContainer.appendChild(progressText); + + // Indexing state + let isIndexing = false; + let completedJobs = 0; + let totalJobs = 0; + let indexingStatus: string | null = null; + + const updateProgressDisplay = () => { + if (isIndexing && totalJobs > 0) { + const percentage = Math.round((completedJobs / totalJobs) * 100); + progressBar.style.width = `${Math.max(2, percentage)}%`; + progressBarWrapper.style.display = "block"; + + if (indexingStatus) { + progressText.textContent = indexingStatus.length > 20 ? indexingStatus.substring(0, 20) + "..." : indexingStatus; + progressText.style.display = "block"; + } else { + progressText.textContent = `${completedJobs}/${totalJobs} (${percentage}%)`; + progressText.style.display = "block"; + } + } else { + progressBarWrapper.style.display = "none"; + progressText.style.display = "none"; + } + }; + + // Listen for indexing progress events + const progressHandler = (event: CustomEvent) => { + const { completed, total, indexing, status } = event.detail; + completedJobs = completed || 0; + totalJobs = total || 0; + isIndexing = indexing || false; + indexingStatus = status || null; + updateProgressDisplay(); + }; + + window.addEventListener('indexing-progress', progressHandler as EventListener); + appRef.progressHandler = progressHandler; + const updateSearchButtonDisplay = () => { searchButton.innerHTML = /* html */ ` @@ -34,6 +100,7 @@ export function mountSearchBar( updateSearchButtonDisplay(); titleElement.appendChild(searchButton); + titleElement.appendChild(progressContainer); // Listen for hotkey setting changes const handleStorageChange = (changes: any, area: string) => { @@ -72,7 +139,7 @@ export function mountSearchBar( } } -export function cleanupSearchBar(appRef: { current: any; storageChangeHandler?: any }) { +export function cleanupSearchBar(appRef: { current: any; storageChangeHandler?: any; progressHandler?: any }) { if (appRef.current) { try { unmount(appRef.current); @@ -82,11 +149,23 @@ export function cleanupSearchBar(appRef: { current: any; storageChangeHandler?: } } + // Remove progress event listener + if (appRef.progressHandler) { + window.removeEventListener('indexing-progress', appRef.progressHandler as EventListener); + appRef.progressHandler = null; + } + // Remove search trigger button const searchTrigger = document.querySelector(".search-trigger"); if (searchTrigger) { searchTrigger.remove(); } + + // Remove progress container + const progressContainer = document.querySelector(".search-progress-container"); + if (progressContainer) { + progressContainer.remove(); + } // Remove search root const searchRoot = document.querySelector("div[data-search-root]"); diff --git a/src/plugins/built-in/globalSearch/src/core/styles.css b/src/plugins/built-in/globalSearch/src/core/styles.css index 5ea89f6b..1c50394e 100644 --- a/src/plugins/built-in/globalSearch/src/core/styles.css +++ b/src/plugins/built-in/globalSearch/src/core/styles.css @@ -68,4 +68,72 @@ .dark .highlight { background-color: rgba(255, 230, 100, 0.4); +} + +@keyframes shimmer { + 0% { + transform: translateX(-100%); + } + 100% { + transform: translateX(100%); + } +} + +.animate-shimmer { + animation: shimmer 2s infinite; +} + +/* Progress indicator next to search trigger */ +.search-progress-container { + display: flex; + align-items: center; + gap: 8px; + margin-left: 8px; + min-width: 120px; + max-width: 200px; + height: 32px; +} + +.search-progress-bar-wrapper { + flex: 1; + height: 4px; + background: rgba(0, 0, 0, 0.1); + border-radius: 2px; + overflow: hidden; + display: none; + min-width: 60px; +} + +.dark .search-progress-bar-wrapper { + background: rgba(255, 255, 255, 0.1); +} + +.search-progress-bar { + height: 100%; + background: linear-gradient(90deg, #3b82f6, #2563eb, #3b82f6); + transition: width 0.3s ease-out; + width: 0%; + position: relative; + border-radius: 2px; +} + +.search-progress-bar::after { + content: ''; + position: absolute; + inset: 0; + background: linear-gradient(90deg, transparent, rgba(255,255,255,0.3), transparent); + animation: shimmer 2s infinite; + border-radius: 2px; +} + +.search-progress-text { + font-size: 11px; + color: #666; + white-space: nowrap; + display: none; + font-weight: 500; +} + +.dark .search-progress-text { + color: #999; } \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/indexing/actions.ts b/src/plugins/built-in/globalSearch/src/indexing/actions.ts index ddf28387..a862249c 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/actions.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/actions.ts @@ -59,17 +59,132 @@ export const actionMap: Record> = { }) as ActionHandler, assessment: (async (item: IndexItem & { metadata: AssessmentMetadata }) => { - if (item.metadata.isMessageBased) { + // Deep clone the entire item to avoid Firefox XrayWrapper issues + // Firefox XrayWrapper prevents direct access to nested properties + let itemClone: IndexItem & { metadata: AssessmentMetadata }; + let metadata: AssessmentMetadata; + + try { + // First try to clone the entire item + itemClone = JSON.parse(JSON.stringify(item)); + metadata = itemClone.metadata || {}; + } catch (e) { + console.warn("[Assessment Action] Failed to clone item, trying to clone metadata separately:", e); + try { + // If full clone fails, try cloning just metadata + metadata = JSON.parse(JSON.stringify(item.metadata || {})); + itemClone = { ...item, metadata }; + } catch (e2) { + console.warn("[Assessment Action] Failed to clone metadata, using direct access:", e2); + itemClone = item; + metadata = item.metadata || {} as AssessmentMetadata; + } + } + + // Try to extract metadata values using multiple methods to handle XrayWrapper + const getMetadataValue = (key: string, altKey?: string): any => { + try { + // Try direct access first + const value = metadata[key]; + if (value !== undefined && value !== null) { + return value; + } + if (altKey) { + const altValue = metadata[altKey]; + if (altValue !== undefined && altValue !== null) { + return altValue; + } + } + // Try accessing via Object.keys iteration (works around XrayWrapper) + try { + const keys = Object.keys(metadata); + for (const k of keys) { + if (k === key || k === altKey) { + const val = metadata[k]; + if (val !== undefined && val !== null) { + return val; + } + } + } + } catch (e) { + // Object.keys might fail on XrayWrapper, that's okay + } + return undefined; + } catch (e) { + console.warn(`[Assessment Action] Failed to access metadata.${key}:`, e); + return undefined; + } + }; + + if (getMetadataValue('isMessageBased')) { window.location.hash = `#?page=/messages`; await waitForElm('[class*="Viewer__Viewer___"] > div', true, 20); // Select the specific direct message ReactFiber.find('[class*="Viewer__Viewer___"] > div').setState({ - selected: new Set([item.metadata.messageId]), + selected: new Set([getMetadataValue('messageId')]), }); } else { - window.location.hash = `#?page=/assessments&id=${item.metadata.assessmentId}`; + // Extract values - check both camelCase and PascalCase, and try multiple access methods + let programmeId = getMetadataValue('programmeId', 'programmeID'); + let metaclassId = getMetadataValue('metaclassId', 'metaclassID'); + let assessmentId = getMetadataValue('assessmentId', 'assessmentID'); + + // Fallback: try to extract assessmentId from item ID if metadata is missing + if ((assessmentId === undefined || assessmentId === null) && itemClone.id && itemClone.id.startsWith('assignment-')) { + const extractedId = itemClone.id.replace('assignment-', ''); + assessmentId = Number(extractedId) || extractedId; + console.log("[Assessment Action] Extracted assessmentId from item ID:", assessmentId); + } + + // Convert to numbers, but preserve 0 as valid + if (programmeId !== undefined && programmeId !== null && programmeId !== '') { + const num = Number(programmeId); + programmeId = isNaN(num) ? programmeId : num; + } + if (metaclassId !== undefined && metaclassId !== null && metaclassId !== '') { + const num = Number(metaclassId); + metaclassId = isNaN(num) ? metaclassId : num; + } + if (assessmentId !== undefined && assessmentId !== null && assessmentId !== '') { + const num = Number(assessmentId); + assessmentId = isNaN(num) ? assessmentId : num; + } + + // Check if values exist (including 0, which is a valid ID) + // Use typeof check to properly handle 0 + const hasProgrammeId = programmeId !== undefined && programmeId !== null && programmeId !== '' && typeof programmeId === 'number'; + const hasMetaclassId = metaclassId !== undefined && metaclassId !== null && metaclassId !== '' && typeof metaclassId === 'number'; + const hasAssessmentId = assessmentId !== undefined && assessmentId !== null && assessmentId !== '' && typeof assessmentId === 'number'; + + + + if (hasProgrammeId && hasMetaclassId && hasAssessmentId) { + const url = `#?page=/assessments/${programmeId}:${metaclassId}&item=${assessmentId}`; + console.log("[Assessment Action] ✅ Navigating to:", url); + window.location.hash = url; + } else { + // Fallback: try to navigate to assessments page if metadata is incomplete + console.error("[Assessment Action] ❌ Missing required metadata:", { + programmeId, + metaclassId, + assessmentId, + hasProgrammeId, + hasMetaclassId, + hasAssessmentId, + metadataKeys: Object.keys(metadata), + metadataString: JSON.stringify(metadata), + itemId: itemClone.id, + }); + // If we at least have an assessmentId, try to navigate to the general assessments page + if (hasAssessmentId) { + window.location.hash = `#?page=/assessments/upcoming&item=${assessmentId}`; + } else { + console.warn("[Assessment Action] No valid assessment ID, redirecting to upcoming"); + window.location.hash = `#?page=/assessments/upcoming`; + } + } } }) as ActionHandler, diff --git a/src/plugins/built-in/globalSearch/src/indexing/db.ts b/src/plugins/built-in/globalSearch/src/indexing/db.ts index 3c3046eb..831cde92 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/db.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/db.ts @@ -213,25 +213,54 @@ export async function clear(store: string): Promise { } export async function resetDatabase(): Promise { + // Close cached database connection if (cachedDb) { - cachedDb.close(); + try { + cachedDb.close(); + } catch (e) { + console.warn("[DB] Error closing cached database:", e); + } cachedDb = null; } + // Close pending database promise if (dbPromise) { try { const db = await dbPromise; db.close(); - } catch (e) {} + } catch (e) { + // Database might not be open yet, that's okay + } dbPromise = null; } + // Wait a bit for connections to fully close + await new Promise(resolve => setTimeout(resolve, 100)); + return new Promise((resolve, reject) => { const req = indexedDB.deleteDatabase(DB_NAME); req.onsuccess = () => { localStorage.removeItem(VERSION_KEY); resolve(); }; - req.onerror = () => reject(req.error); + req.onerror = () => { + console.error("[DB] Error deleting database:", req.error); + reject(req.error); + }; + req.onblocked = () => { + console.warn("[DB] Database deletion blocked - waiting for connections to close"); + // Wait a bit longer and try again + setTimeout(() => { + const retryReq = indexedDB.deleteDatabase(DB_NAME); + retryReq.onsuccess = () => { + localStorage.removeItem(VERSION_KEY); + resolve(); + }; + retryReq.onerror = () => reject(retryReq.error); + retryReq.onblocked = () => { + reject(new Error(`Database is still open. Please close other tabs/windows and try again.`)); + }; + }, 500); + }; }); } diff --git a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts index 04e09b2a..c14ee9bc 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/indexer.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/indexer.ts @@ -396,18 +396,34 @@ export async function runIndexing(): Promise { stopHeartbeat(); allItemsInPrimaryStores = await loadAllStoredItems(); - allItemsInPrimaryStores.forEach(item => { - const jobDef = jobs[item.category] || Object.values(jobs).find(j => j.id === item.category) || jobs[item.renderComponentId]; - if (jobDef) { - const renderComponent = renderComponentMap[jobDef.renderComponentId]; - if (renderComponent) { - item.renderComponent = renderComponent; - } - } else if (renderComponentMap[item.renderComponentId]) { - item.renderComponent = renderComponentMap[item.renderComponentId]; + // Create new objects to avoid XrayWrapper issues in Firefox + const itemsWithComponents = allItemsInPrimaryStores.map(item => { + try { + const jobDef = jobs[item.category] || Object.values(jobs).find(j => j.id === item.category) || jobs[item.renderComponentId]; + let renderComponent = item.renderComponent; + if (jobDef) { + renderComponent = renderComponentMap[jobDef.renderComponentId] || renderComponent; + } else if (renderComponentMap[item.renderComponentId]) { + renderComponent = renderComponentMap[item.renderComponentId]; + } + // Deep clone to avoid Firefox XrayWrapper issues with nested objects like metadata + // Use JSON serialization to ensure all nested properties are accessible + try { + const cloned = JSON.parse(JSON.stringify(item)); + cloned.renderComponent = renderComponent; + return cloned; + } catch (e) { + // Fallback to shallow copy if deep clone fails + console.warn("[Indexer] Failed to deep clone item, using shallow copy:", e); + return { ...item, renderComponent }; + } + } catch (error) { + // Fallback: return item as-is if modification fails (Firefox XrayWrapper) + console.warn("[Indexer] Failed to add render component to item (Firefox XrayWrapper):", error); + return item; } }); - loadDynamicItems(allItemsInPrimaryStores); + loadDynamicItems(itemsWithComponents); window.dispatchEvent(new Event("dynamic-items-updated")); } diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts index 20cb36e2..659d2bc0 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts @@ -3,10 +3,12 @@ import { messagesJob } from "./jobs/messages"; import { notificationsJob } from "./jobs/notifications"; import { forumsJob } from "./jobs/forums"; import { subjectsJob } from "./jobs/subjects"; +import { assignmentsJob } from "./jobs/assignments"; export const jobs: Record = { messages: messagesJob, notifications: notificationsJob, forums: forumsJob, subjects: subjectsJob, + assignments: assignmentsJob, }; diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts new file mode 100644 index 00000000..596cbe42 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts @@ -0,0 +1,369 @@ +import type { IndexItem, Job } from "../types"; + +const fetchJSON = async (url: string, body: any) => { + const res = await fetch(`${location.origin}${url}`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json; charset=utf-8" }, + body: JSON.stringify(body), + }); + return res.json(); +}; + +const fetchUpcomingAssessments = async (student: number = 69) => { + try { + const res = await fetchJSON("/seqta/student/assessment/list/upcoming?", { + student, + }); + // Match analytics.rs: payload is an array, return empty array if not found + return Array.isArray(res.payload) ? res.payload : []; + } catch (e) { + console.error("[Assignments job] Failed to fetch upcoming assessments:", e); + return []; + } +}; + +const fetchSubjects = async () => { + try { + const res = await fetchJSON("/seqta/student/load/subjects?", {}); + return res.payload + ?.filter((s: any) => s.active === 1) + ?.flatMap((s: any) => s.subjects) || []; + } catch (e) { + console.error("[Assignments job] Failed to fetch subjects:", e); + return []; + } +}; + +const fetchPastAssessments = async (student: number = 69, subjects: any[]) => { + const map: Record = {}; + + // Fetch past assessments for all subjects in parallel (like assessmentsOverview does) + // This is much faster than sequential fetching + await Promise.all( + subjects.map(async (subject) => { + try { + // Match analytics.rs exactly: parameter order is programme, metaclass, student + const res = await fetchJSON("/seqta/student/assessment/list/past?", { + programme: subject.programme, + metaclass: subject.metaclass, + student, + }); + + // Past assessments API can return data in payload.tasks OR payload.pending (or both) + // Based on analytics.rs fetch_past_assessments, we need to check both arrays + const processAssessment = (assessment: any) => { + if (assessment && assessment.id) { + // Ensure programme and metaclass are included from the subject + // Use the assessment's IDs if available, otherwise fall back to subject's + map[assessment.id] = { + ...assessment, + programme: assessment.programme || assessment.programmeID || subject.programme, + programmeID: assessment.programmeID || assessment.programme || subject.programme, + metaclass: assessment.metaclass || assessment.metaclassID || subject.metaclass, + metaclassID: assessment.metaclassID || assessment.metaclass || subject.metaclass, + }; + } + }; + + // Match analytics.rs: Check both pending and tasks arrays + // Check for pending array first (matching Rust code order) + if (res.payload?.pending && Array.isArray(res.payload.pending)) { + res.payload.pending.forEach(processAssessment); + } + + // Check for tasks array + if (res.payload?.tasks && Array.isArray(res.payload.tasks)) { + res.payload.tasks.forEach(processAssessment); + } + } catch (e) { + console.warn(`[Assignments job] Failed to fetch past assessments for subject ${subject.code || subject.subject || 'unknown'}:`, e); + } + }) + ); + + return Object.values(map); +}; + +export const assignmentsJob: Job = { + id: "assignments", + label: "Assignments", + renderComponentId: "assessment", + frequency: { type: "expiry", afterMs: 1000 * 60 * 60 * 24 }, // Daily + + boostCriteria: (item, searchTerm) => { + if (searchTerm === "") { + return -100; + } + + let score = 0; + + // Boost upcoming assignments + if (item.metadata.dueDate) { + const dueDate = new Date(item.metadata.dueDate).getTime(); + const now = Date.now(); + const daysUntilDue = (dueDate - now) / (1000 * 60 * 60 * 24); + + if (daysUntilDue >= 0 && daysUntilDue <= 7) { + score += 0.05; // Boost assignments due within a week + } + if (daysUntilDue < 0) { + score -= 0.1; // Penalty for overdue assignments + } + } + + // Boost if submitted + if (item.metadata.submitted) { + score += 0.02; + } + + return score; + }, + + run: async (ctx) => { + // Don't filter by existing IDs - we want to process ALL assessments (both new and old) + // to ensure metadata is up-to-date and all past assignments are indexed + const existingItems = await ctx.getStoredItems("assignments"); + const existingIds = new Set(existingItems.map((i) => i.id)); + + const student = 69; // TODO: Get from context if available + + console.debug("[Assignments job] Starting indexing - fetching all assessments (upcoming and past)..."); + + // Fetch data in parallel + const [upcoming, subjects] = await Promise.all([ + fetchUpcomingAssessments(student), + fetchSubjects(), + ]); + + console.debug(`[Assignments job] Fetched ${upcoming.length} upcoming assessments and ${subjects.length} subjects`); + + // Fetch past assessments for ALL subjects to ensure we get all historical assignments + const past = await fetchPastAssessments(student, subjects); + + console.debug(`[Assignments job] Fetched ${past.length} past assessments`); + + // Create a lookup map from subject code to programme/metaclass + const subjectLookup = new Map(); + subjects.forEach((s: any) => { + if (s.code && s.programme && s.metaclass) { + subjectLookup.set(s.code, { programme: s.programme, metaclass: s.metaclass }); + } + }); + + // Combine and deduplicate + const allAssessments = new Map(); + + upcoming.forEach((a: any) => { + if (a && a.id) { + // Prioritize capital ID fields (programmeID, metaclassID) as that's what the API returns + let programme = a.programmeID || a.programme; + let metaclass = a.metaclassID || a.metaclass; + + // If missing, try to get from subject lookup + if ((!programme || !metaclass) && a.code) { + const subjectInfo = subjectLookup.get(a.code); + if (subjectInfo) { + programme = programme || subjectInfo.programme; + metaclass = metaclass || subjectInfo.metaclass; + } + } + + allAssessments.set(a.id, { + ...a, + programme, + metaclass, + programmeID: programme, // Ensure both formats are available + metaclassID: metaclass, + isUpcoming: true, + }); + } + }); + + past.forEach((a: any) => { + if (a && a.id) { + // Prioritize capital ID fields (programmeID, metaclassID) as that's what the API returns + let programme = a.programmeID || a.programme; + let metaclass = a.metaclassID || a.metaclass; + + const existing = allAssessments.get(a.id); + if (existing) { + // Merge past assessment data, ensuring programme/metaclass are preserved + // Use existing values if new ones are missing + programme = programme || existing.programme || existing.programmeID; + metaclass = metaclass || existing.metaclass || existing.metaclassID; + + Object.assign(existing, { + ...a, + programme, + metaclass, + programmeID: programme, + metaclassID: metaclass, + }); + } else { + allAssessments.set(a.id, { + ...a, + programme, + metaclass, + programmeID: programme, + metaclassID: metaclass, + isUpcoming: false + }); + } + } + }); + + const items: IndexItem[] = []; + const processedIds = new Set(); + + // Process assessments in batches to avoid overwhelming the API + const assessmentArray = Array.from(allAssessments.values()); + const pastCount = assessmentArray.filter(a => !a.isUpcoming).length; + const upcomingCount = assessmentArray.filter(a => a.isUpcoming).length; + console.debug(`[Assignments job] Processing ${assessmentArray.length} total assessments (${upcomingCount} upcoming, ${pastCount} past)`); + const batchSize = 15; // Increased batch size for better performance + + // Skip fetching assessment details - the API endpoint doesn't exist or returns 404 + // Details are optional and not critical for search functionality + + // Process ALL assessments (both upcoming and past) to ensure everything is indexed + for (let i = 0; i < assessmentArray.length; i += batchSize) { + const batch = assessmentArray.slice(i, i + batchSize); + + const batchItems = await Promise.all( + batch.map(async (assessment) => { + const id = `assignment-${assessment.id}`; + + // Skip if already processed in this batch + if (processedIds.has(id)) { + return null; + } + + processedIds.add(id); + + // Process ALL assessments (both new and existing, upcoming and past) + // This ensures all historical assignments are indexed and metadata is up-to-date + + // Skip fetching details - API endpoint doesn't exist + const description = ""; + + const subjectName = assessment.subject || assessment.code || "Unknown Subject"; + const dueDate = assessment.due ? new Date(assessment.due).getTime() : null; + + // Prioritize capital ID fields (programmeID, metaclassID) as that's what the API returns + const programmeId = assessment.programmeID || assessment.programme; + const metaclassId = assessment.metaclassID || assessment.metaclass; + + // Validate that we have the required IDs for navigation + if (!programmeId || !metaclassId || !assessment.id) { + console.warn(`[Assignments job] Skipping assignment ${assessment.id} - missing required IDs:`, { + programmeId, + metaclassId, + assessmentId: assessment.id, + programmeID: assessment.programmeID, + metaclassID: assessment.metaclassID, + programme: assessment.programme, + metaclass: assessment.metaclass, + assessment, + }); + return null; + } + + // Convert to numbers, preserving 0 as valid + let finalProgrammeId: number | undefined; + let finalMetaclassId: number | undefined; + + if (programmeId !== undefined && programmeId !== null && programmeId !== '') { + const num = Number(programmeId); + finalProgrammeId = isNaN(num) ? undefined : num; + } + + if (metaclassId !== undefined && metaclassId !== null && metaclassId !== '') { + const num = Number(metaclassId); + finalMetaclassId = isNaN(num) ? undefined : num; + } + + // Final validation - check for actual numbers (including 0) + if (finalProgrammeId === undefined || finalMetaclassId === undefined || !assessment.id) { + console.error(`[Assignments job] ❌ Skipping assignment ${assessment.id} - invalid IDs after conversion:`, { + programmeId: finalProgrammeId, + metaclassId: finalMetaclassId, + assessmentId: assessment.id, + rawProgrammeId: programmeId, + rawMetaclassId: metaclassId, + assessment, + }); + return null; + } + + const item: IndexItem = { + id, + text: assessment.title || assessment.name || "Untitled Assignment", + category: "assignments", + content: `${description}\nSubject: ${subjectName}\nDue: ${assessment.due || "No due date"}`.trim(), + dateAdded: dueDate || Date.now(), + metadata: { + assessmentId: assessment.id, + assessmentID: assessment.id, // Store both variants for compatibility + subject: subjectName, + subjectCode: assessment.code, + dueDate: assessment.due, + programmeId: finalProgrammeId, + programmeID: finalProgrammeId, // Store both variants for compatibility + metaclassId: finalMetaclassId, + metaclassID: finalMetaclassId, // Store both variants for compatibility + submitted: assessment.submitted || false, + isUpcoming: assessment.isUpcoming || false, + term: assessment.term, + timestamp: assessment.due || new Date().toISOString(), // Required by AssessmentMetadata interface + }, + actionId: "assessment", + renderComponentId: "assessment", + }; + + console.debug(`[Assignments job] ✅ Created item for assignment ${assessment.id}:`, { + id: item.id, + programmeId: item.metadata.programmeId, + programmeID: item.metadata.programmeID, + metaclassId: item.metadata.metaclassId, + metaclassID: item.metadata.metaclassID, + assessmentId: item.metadata.assessmentId, + assessmentID: item.metadata.assessmentID, + }); + + return item; + }) + ); + + // Filter out nulls and add to items + batchItems.forEach(item => { + if (item) { + items.push(item); + } + }); + + // Small delay between batches to avoid rate limiting + if (i + batchSize < assessmentArray.length) { + await new Promise(resolve => setTimeout(resolve, 50)); // Reduced delay + } + } + + const newItemsCount = items.filter(item => !existingIds.has(item.id)).length; + const updatedItemsCount = items.length - newItemsCount; + console.debug(`[Assignments job] Indexed ${items.length} assignment items (${newItemsCount} new, ${updatedItemsCount} updated)`); + return items; + }, + + purge: (items) => { + // Keep ALL assignments - don't purge old ones as users may want to search for them + // Only remove items that are truly invalid (missing required metadata) + return items.filter((i) => { + // Keep all items that have valid metadata + return i.metadata && + i.metadata.assessmentId && + i.metadata.programmeId !== undefined && + i.metadata.metaclassId !== undefined; + }); + }, +}; + diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs/messages.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs/messages.ts index c0072d98..617b3cad 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs/messages.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs/messages.ts @@ -604,22 +604,34 @@ export const messagesJob: Job = { if (processedItems.length > 0) { try { const currentItems = await loadAllStoredItems(); - currentItems.forEach((item) => { - const jobDef = - jobs[item.category] || - Object.values(jobs).find((j) => j.id === item.category) || - jobs[item.renderComponentId]; - if (jobDef) { - const renderComponent = - renderComponentMap[jobDef.renderComponentId]; - if (renderComponent) { - item.renderComponent = renderComponent; + // Create new objects to avoid XrayWrapper issues in Firefox + const itemsWithComponents = currentItems.map((item) => { + try { + const jobDef = + jobs[item.category] || + Object.values(jobs).find((j) => j.id === item.category) || + jobs[item.renderComponentId]; + let renderComponent = item.renderComponent; + if (jobDef) { + renderComponent = renderComponentMap[jobDef.renderComponentId] || renderComponent; + } else if (renderComponentMap[item.renderComponentId]) { + renderComponent = renderComponentMap[item.renderComponentId]; } - } else if (renderComponentMap[item.renderComponentId]) { - item.renderComponent = renderComponentMap[item.renderComponentId]; + // Deep clone to avoid Firefox XrayWrapper issues with nested objects like metadata + try { + const cloned = JSON.parse(JSON.stringify(item)); + cloned.renderComponent = renderComponent; + return cloned; + } catch (e) { + // Fallback to shallow copy if deep clone fails + return { ...item, renderComponent }; + } + } catch (error) { + // Fallback: return item as-is if modification fails (Firefox XrayWrapper) + return item; } }); - loadDynamicItems(currentItems); + loadDynamicItems(itemsWithComponents); window.dispatchEvent( new CustomEvent("dynamic-items-updated", { detail: { diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs/notifications.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs/notifications.ts index 26f46180..f0474bbc 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs/notifications.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs/notifications.ts @@ -372,23 +372,34 @@ export const notificationsJob: Job = { if (items.length > 0) { try { const currentItems = await loadAllStoredItems(); - currentItems.forEach((item) => { - const jobDef = - jobs[item.category] || - Object.values(jobs).find((j) => j.id === item.category) || - jobs[item.renderComponentId]; - if (jobDef) { - const renderComponent = - renderComponentMap[jobDef.renderComponentId]; - if (renderComponent) { - item.renderComponent = renderComponent; + // Create new objects to avoid XrayWrapper issues in Firefox + const itemsWithComponents = currentItems.map((item) => { + try { + const jobDef = + jobs[item.category] || + Object.values(jobs).find((j) => j.id === item.category) || + jobs[item.renderComponentId]; + let renderComponent = item.renderComponent; + if (jobDef) { + renderComponent = renderComponentMap[jobDef.renderComponentId] || renderComponent; + } else if (renderComponentMap[item.renderComponentId]) { + renderComponent = renderComponentMap[item.renderComponentId]; } - } else if (renderComponentMap[item.renderComponentId]) { - item.renderComponent = - renderComponentMap[item.renderComponentId]; + // Deep clone to avoid Firefox XrayWrapper issues with nested objects like metadata + try { + const cloned = JSON.parse(JSON.stringify(item)); + cloned.renderComponent = renderComponent; + return cloned; + } catch (e) { + // Fallback to shallow copy if deep clone fails + return { ...item, renderComponent }; + } + } catch (error) { + // Fallback: return item as-is if modification fails (Firefox XrayWrapper) + return item; } }); - loadDynamicItems(currentItems); + loadDynamicItems(itemsWithComponents); window.dispatchEvent( new CustomEvent("dynamic-items-updated", { detail: { diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs/subjects.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs/subjects.ts index 5f41272f..342afd46 100755 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs/subjects.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs/subjects.ts @@ -1,140 +1,140 @@ -import type { IndexItem, Job } from "../types"; - -const fetchSubjects = async () => { - const res = await fetch(`${location.origin}/seqta/student/load/subjects`, { - method: "POST", - credentials: "include", - headers: { "Content-Type": "application/json; charset=utf-8" }, - body: JSON.stringify({ mode: "list" }), - }); - - const data = await res.json(); - return data; -}; - -export const subjectsJob: Job = { - id: "subjects", - label: "Subjects", - renderComponentId: "subject", - frequency: { - type: "expiry", - afterMs: 1000 * 60 * 60 * 24 * 30, - }, - boostCriteria: (item, searchTerm) => { - if (searchTerm == "") { - return -100; - } - - let score = 0; - if (item.metadata.isActive) { - score += 0.01; // Boost for active subjects - } else { - score -= 50; // Penalty for inactive subjects - } - - return score; - }, - - run: async (ctx) => { - const existingIds = new Set( - (await ctx.getStoredItems("subjects")).map((i) => i.id), - ); - - let list; - try { - list = await fetchSubjects(); - } catch (e) { - console.error("[Subjects job] list fetch failed:", e); - return []; - } - - if (list.status !== "200") { - console.error("[Subjects job] API returned non-200 status:", list.status); - return []; - } - - // Check if we have the expected data structure - if (!list.payload || !Array.isArray(list.payload)) { - console.error("[Subjects job] Unexpected API response structure:", list); - return []; - } - - const items: IndexItem[] = []; - - // Process each semester - for (const semester of list.payload) { - if (!semester.subjects || !Array.isArray(semester.subjects)) { - console.warn("[Subjects job] Skipping invalid semester:", semester); - continue; - } - - // Process each subject in the semester - for (const subject of semester.subjects) { - // Skip if subject doesn't have required fields - if (!subject || !subject.code || !subject.title) { - console.warn("[Subjects job] Skipping invalid subject:", subject); - continue; - } - - const id = `${semester.code}-${subject.code}-${subject.metaclass}`; - if (existingIds.has(id)) continue; - - const isActive = semester.active === 1; - - // Create two items for each subject - one for assessments and one for course - const assessmentsItem = { - id: `${id}-assessments`, - text: `${subject.title} Assessments`, - category: "subjects", - content: `View assessments for ${subject.title} (${semester.description})`, - dateAdded: Date.now(), - metadata: { - subjectId: subject.metaclass, - subjectName: subject.title, - subjectCode: subject.code, - programme: subject.programme, - semesterCode: semester.code, - semesterDescription: semester.description, - type: "assessments", - isActive - }, - actionId: "subjectassessment", - renderComponentId: "subject", - }; - - const courseItem = { - id: `${id}-course`, - text: `${subject.title}`, - category: "subjects", - content: `View course content for ${subject.title} (${semester.description})`, - dateAdded: Date.now(), - metadata: { - subjectId: subject.metaclass, - subjectName: subject.title, - subjectCode: subject.code, - programme: subject.programme, - semesterCode: semester.code, - semesterDescription: semester.description, - type: "course", - isActive - }, - actionId: "subjectcourse", - renderComponentId: "subject", - }; - - items.push( - assessmentsItem, - courseItem - ); - } - } - - console.debug(`[Subjects job] Indexed ${items.length} subject items`); - return items; - }, - - purge: (items) => { - // Keep all subjects as they are relatively static - return items; - }, +import type { IndexItem, Job } from "../types"; + +const fetchSubjects = async () => { + const res = await fetch(`${location.origin}/seqta/student/load/subjects`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json; charset=utf-8" }, + body: JSON.stringify({ mode: "list" }), + }); + + const data = await res.json(); + return data; +}; + +export const subjectsJob: Job = { + id: "subjects", + label: "Subjects", + renderComponentId: "subject", + frequency: { + type: "expiry", + afterMs: 1000 * 60 * 60 * 24 * 30, + }, + boostCriteria: (item, searchTerm) => { + if (searchTerm == "") { + return -100; + } + + let score = 0; + if (item.metadata.isActive) { + score += 0.01; // Boost for active subjects + } else { + score -= 50; // Penalty for inactive subjects + } + + return score; + }, + + run: async (ctx) => { + const existingIds = new Set( + (await ctx.getStoredItems("subjects")).map((i) => i.id), + ); + + let list; + try { + list = await fetchSubjects(); + } catch (e) { + console.error("[Subjects job] list fetch failed:", e); + return []; + } + + if (list.status !== "200") { + console.error("[Subjects job] API returned non-200 status:", list.status); + return []; + } + + // Check if we have the expected data structure + if (!list.payload || !Array.isArray(list.payload)) { + console.error("[Subjects job] Unexpected API response structure:", list); + return []; + } + + const items: IndexItem[] = []; + + // Process each semester + for (const semester of list.payload) { + if (!semester.subjects || !Array.isArray(semester.subjects)) { + console.warn("[Subjects job] Skipping invalid semester:", semester); + continue; + } + + // Process each subject in the semester + for (const subject of semester.subjects) { + // Skip if subject doesn't have required fields + if (!subject || !subject.code || !subject.title) { + console.warn("[Subjects job] Skipping invalid subject:", subject); + continue; + } + + const id = `${semester.code}-${subject.code}-${subject.metaclass}`; + if (existingIds.has(id)) continue; + + const isActive = semester.active === 1; + + // Create two items for each subject - one for assessments and one for course + const assessmentsItem = { + id: `${id}-assessments`, + text: `${subject.title} Assessments`, + category: "subjects", + content: `View assessments for ${subject.title} (${semester.description})`, + dateAdded: Date.now(), + metadata: { + subjectId: subject.metaclass, + subjectName: subject.title, + subjectCode: subject.code, + programme: subject.programme, + semesterCode: semester.code, + semesterDescription: semester.description, + type: "assessments", + isActive + }, + actionId: "subjectassessment", + renderComponentId: "subject", + }; + + const courseItem = { + id: `${id}-course`, + text: `${subject.title}`, + category: "subjects", + content: `View course content for ${subject.title} (${semester.description})`, + dateAdded: Date.now(), + metadata: { + subjectId: subject.metaclass, + subjectName: subject.title, + subjectCode: subject.code, + programme: subject.programme, + semesterCode: semester.code, + semesterDescription: semester.description, + type: "course", + isActive + }, + actionId: "subjectcourse", + renderComponentId: "subject", + }; + + items.push( + assessmentsItem, + courseItem + ); + } + } + + console.debug(`[Subjects job] Indexed ${items.length} subject items`); + return items; + }, + + purge: (items) => { + // Keep all subjects as they are relatively static + return items; + }, }; \ No newline at end of file diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts index 4f4c3cb2..64f71f0d 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorker.ts @@ -3,9 +3,24 @@ import type { IndexItem } from "../types"; let vectorIndex: EmbeddingIndex | null = null; let isInitialized = false; +let initializationFailed = false; let currentAbortController: AbortController | null = null; let loadedItemIds = new Set(); +// Detect Firefox in worker context +function isFirefoxWorker(): boolean { + try { + // Check for Firefox-specific APIs or user agent + if (typeof navigator !== "undefined") { + return navigator.userAgent.toLowerCase().includes("firefox"); + } + // In worker context, check for Firefox-specific behavior + return false; + } catch { + return false; + } +} + let streamingSession: { isActive: boolean; totalExpected: number; @@ -21,6 +36,16 @@ async function initWorker() { console.debug("Vector worker already initialized."); return; } + + // Skip initialization in Firefox + if (isFirefoxWorker()) { + console.debug("[Vector Worker] Vector search not supported in Firefox - skipping initialization"); + isInitialized = true; + initializationFailed = true; + vectorIndex = null; + return; + } + console.debug("Initializing vector worker..."); try { await initializeModel(); @@ -48,8 +73,9 @@ async function initWorker() { isInitialized = true; console.debug("Vector worker initialized successfully."); } catch (e) { - console.error("Failed to initialize vector worker:", e); + console.warn("[Vector Worker] Failed to initialize vector worker (will use text search only):", e); isInitialized = true; + initializationFailed = true; vectorIndex = null; } } @@ -80,18 +106,29 @@ async function startStreamingSession( totalExpected: number, batchSize: number = 5, ) { + if (initializationFailed || isFirefoxWorker()) { + self.postMessage({ + type: "progress", + data: { + status: "complete", + message: "Vector search not available in Firefox - using text search only", + }, + }); + return; + } + if (!vectorIndex) { console.warn( "Streaming requested but vector index not ready. Attempting init.", ); await initWorker(); - if (!vectorIndex) { + if (!vectorIndex || initializationFailed) { self.postMessage({ type: "progress", data: { - status: "error", + status: "complete", message: - "Vector index not available for streaming after init attempt.", + "Vector index not available - using text search only", }, }); return; @@ -306,18 +343,29 @@ async function endStreamingSession() { async function processItems(items: IndexItem[], signal: AbortSignal) { console.debug("Worker received process request."); + if (initializationFailed || isFirefoxWorker()) { + self.postMessage({ + type: "progress", + data: { + status: "complete", + message: "Vector search not available - using text search only", + }, + }); + return; + } + if (!vectorIndex) { console.warn( "Processing requested but vector index not ready. Attempting init.", ); await initWorker(); - if (!vectorIndex) { + if (!vectorIndex || initializationFailed) { self.postMessage({ type: "progress", data: { - status: "error", + status: "complete", message: - "Vector index not available for processing after init attempt.", + "Vector index not available - using text search only", }, }); return; diff --git a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts index c9772efa..f9040996 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/worker/vectorWorkerManager.ts @@ -1,5 +1,6 @@ import { refreshVectorCache } from "../../search/vector/vectorSearch"; import type { IndexItem } from "../types"; +import { isVectorSearchSupported } from "../../utils/browserDetection"; import vectorWorker from "./vectorWorker.ts?inlineWorker"; export type ProgressCallback = (data: { @@ -42,6 +43,13 @@ export class VectorWorkerManager { } private async initWorker(): Promise { + // Skip initialization if vector search is not supported (e.g., Firefox) + if (!isVectorSearchSupported()) { + console.debug("[VectorWorkerManager] Vector search not supported - skipping worker initialization"); + this.isInitialized = false; + return Promise.resolve(); + } + if (this.isInitialized) return Promise.resolve(); if (this.readyPromise) return this.readyPromise; @@ -234,6 +242,17 @@ export class VectorWorkerManager { } async processItems(items: IndexItem[], onProgress?: ProgressCallback) { + // Skip if vector search is not supported + if (!isVectorSearchSupported()) { + if (onProgress) { + onProgress({ + status: "complete", + message: "Vector search not available - using text search only" + }); + } + return; + } + // Only initialize worker if we actually have items to process if (items.length === 0) { if (onProgress) { @@ -298,6 +317,18 @@ export class VectorWorkerManager { batchSize: number = 10, jobId?: string, ): Promise { + // Skip if vector search is not supported + if (!isVectorSearchSupported()) { + console.debug("[VectorWorker] Vector search not supported - skipping streaming session"); + if (onProgress) { + onProgress({ + status: "complete", + message: "Vector search not available - using text search only", + }); + } + return; + } + // Only initialize if we expect items to process if (totalExpectedItems === 0) { console.debug("[VectorWorker] No items expected, not starting streaming session"); diff --git a/src/plugins/built-in/globalSearch/src/search/hybridSearch.ts b/src/plugins/built-in/globalSearch/src/search/hybridSearch.ts new file mode 100644 index 00000000..42360e16 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/search/hybridSearch.ts @@ -0,0 +1,280 @@ +import type { IndexItem } from "../indexing/types"; +import type { CombinedResult } from "../core/types"; +import { searchVectors, type VectorSearchResult } from "./vector/vectorSearch"; +import { jobs } from "../indexing/jobs"; + +/** + * Hybrid Search Implementation + * + * Flow: + * 1. BM25 (Fuse.js) gets top N results fast + * 2. Vector search reranks by semantic similarity + * 3. Apply optional boosting (recency, popularity, tags) + */ + +export interface HybridSearchOptions { + /** Maximum number of BM25 results to retrieve before reranking */ + bm25TopK?: number; + /** Maximum number of final results to return */ + finalLimit?: number; + /** Whether to apply recency boost */ + recencyBoost?: boolean; + /** Weight for BM25 scores (0-1) */ + bm25Weight?: number; + /** Weight for vector similarity scores (0-1) */ + vectorWeight?: number; + /** Weight for recency boost */ + recencyWeight?: number; +} + +const DEFAULT_OPTIONS: Required = { + bm25TopK: 50, // Get top 50 from BM25, then rerank + finalLimit: 10, + recencyBoost: true, + bm25Weight: 0.4, // 40% BM25, 60% vector + vectorWeight: 0.6, + recencyWeight: 0.1, +}; + +/** + * Normalizes a score to 0-1 range + */ +function normalizeScore(score: number, min: number, max: number): number { + if (max === min) return 0.5; + return Math.max(0, Math.min(1, (score - min) / (max - min))); +} + +/** + * Calculates recency boost based on item age + */ +function calculateRecencyBoost(item: IndexItem, now: number): number { + const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); + // Exponential decay: newer items get higher boost + // Items from today get boost of 1, items from 30 days ago get ~0.03 + return 1 / (1 + ageInDays / 7); // Half-life of 7 days +} + +/** + * Calculates popularity boost (can be extended with click tracking, etc.) + */ +function calculatePopularityBoost(item: IndexItem): number { + // For now, boost based on category and metadata + let boost = 0; + + // Boost assignments/assessments + if (item.category === "assignments") { + boost += 0.1; + } + + // Boost upcoming items + if (item.metadata?.isUpcoming) { + boost += 0.15; + } + + // Boost items with subject codes (more structured) + if (item.metadata?.subjectCode) { + boost += 0.05; + } + + return Math.min(boost, 0.3); // Cap at 0.3 +} + +/** + * Reranks BM25 results using vector search + */ +export async function hybridSearch( + bm25Results: CombinedResult[], + query: string, + options: HybridSearchOptions = {}, +): Promise { + const opts = { ...DEFAULT_OPTIONS, ...options }; + const trimmedQuery = query.trim().toLowerCase(); + + // If no BM25 results, return empty + if (bm25Results.length === 0) { + return []; + } + + // Limit BM25 results to top K + const topBm25Results = bm25Results.slice(0, opts.bm25TopK); + + // Get vector search results for reranking + // We'll search the full index and then filter to our BM25 results + let vectorResults: VectorSearchResult[] = []; + + if (trimmedQuery.length > 2) { + try { + // Get more vector results than BM25 results to ensure coverage + // This allows us to find semantic matches that BM25 might have missed + const vectorSearchResults = await searchVectors(trimmedQuery, opts.bm25TopK * 2); + + // Create a map of item ID to vector similarity + const vectorMap = new Map(); + vectorSearchResults.forEach(v => { + // Use the highest similarity if item appears multiple times + const existing = vectorMap.get(v.object.id); + if (!existing || v.similarity > existing) { + vectorMap.set(v.object.id, v.similarity); + } + }); + + // Now rerank BM25 results with vector scores + const now = Date.now(); + + const rerankedResults = topBm25Results.map(result => { + const item = result.item; + + // Normalize BM25 score to 0-1 + // Fuse.js scores: lower is better (0 = perfect match) + // We need to invert: higher score = better match + // Result.score is typically 0-100, where higher = better + // So we normalize it to 0-1 + const normalizedBm25Score = Math.max(0, Math.min(1, result.score / 100)); + + // Get vector similarity (0-1, already normalized) + // If item wasn't in vector results, use a default low score + const vectorSimilarity = vectorMap.get(item.id) || 0.3; // Default to 0.3 if not found + + // Calculate recency boost (0-1 range) + const recencyBoost = opts.recencyBoost + ? calculateRecencyBoost(item, now) * opts.recencyWeight + : 0; + + // Calculate popularity boost (0-1 range) + const popularityBoost = calculatePopularityBoost(item); + + // Apply job-specific boost if available + const job = jobs[item.category]; + let jobBoost = 0; + if (job && typeof job.boostCriteria === 'function') { + const boost = job.boostCriteria(item, trimmedQuery); + if (boost) { + jobBoost = boost / 100; // Normalize boost to 0-1 + } + } + + // Combine scores using weighted average + // BM25 and vector are weighted, boosts are additive + const hybridScore = + (normalizedBm25Score * opts.bm25Weight) + + (vectorSimilarity * opts.vectorWeight) + + recencyBoost + + popularityBoost + + jobBoost; + + return { + ...result, + score: hybridScore * 100, // Scale back to 0-100 for consistency + // Store component scores for debugging (optional, can be removed in production) + _hybridScores: { + bm25: normalizedBm25Score, + vector: vectorSimilarity, + recency: recencyBoost, + popularity: popularityBoost, + jobBoost: jobBoost, + final: hybridScore, + }, + }; + }); + + // Sort by hybrid score descending + rerankedResults.sort((a, b) => b.score - a.score); + + // Return top results + return rerankedResults.slice(0, opts.finalLimit); + + } catch (e) { + console.warn("[Hybrid Search] Vector reranking failed, using BM25 only:", e); + // Fallback to BM25 only + return topBm25Results.slice(0, opts.finalLimit); + } + } + + // If query is too short for vector search, just return BM25 results + return topBm25Results.slice(0, opts.finalLimit); +} + +/** + * Enhanced hybrid search that also includes vector-only results not found by BM25 + */ +export async function hybridSearchWithExpansion( + bm25Results: CombinedResult[], + query: string, + allItems: IndexItem[], + options: HybridSearchOptions = {}, +): Promise { + const opts = { ...DEFAULT_OPTIONS, ...options }; + const trimmedQuery = query.trim().toLowerCase(); + + // First, rerank BM25 results + const rerankedBm25 = await hybridSearch(bm25Results, query, options); + + // If query is too short, skip vector expansion + if (trimmedQuery.length <= 2) { + return rerankedBm25; + } + + // Get vector search results + let vectorResults: VectorSearchResult[] = []; + try { + vectorResults = await searchVectors(trimmedQuery, opts.bm25TopK); + } catch (e) { + console.warn("[Hybrid Search] Vector search failed:", e); + return rerankedBm25; + } + + // Find vector results that weren't in BM25 results + const bm25Ids = new Set(bm25Results.map(r => r.item.id)); + const vectorOnlyResults: CombinedResult[] = []; + + const now = Date.now(); + + vectorResults.forEach(v => { + if (!bm25Ids.has(v.object.id)) { + // This is a semantic match that BM25 missed + const item = v.object; + + // Calculate boosts + const recencyBoost = opts.recencyBoost + ? calculateRecencyBoost(item, now) * opts.recencyWeight + : 0; + const popularityBoost = calculatePopularityBoost(item); + + // Vector-only results get lower base score but high vector similarity + const vectorScore = v.similarity * opts.vectorWeight + recencyBoost + popularityBoost; + + // Apply job-specific boost if available + const job = jobs[item.category]; + let jobBoost = 0; + if (job && typeof job.boostCriteria === 'function') { + const boost = job.boostCriteria(item, trimmedQuery); + if (boost) { + jobBoost = boost / 100; // Normalize boost + } + } + + vectorOnlyResults.push({ + id: item.id, + type: "dynamic" as const, + score: (vectorScore + jobBoost) * 100, + item, + _hybridScores: { + bm25: 0, + vector: v.similarity, + recency: recencyBoost, + popularity: popularityBoost, + final: vectorScore + jobBoost, + }, + }); + } + }); + + // Combine reranked BM25 results with vector-only results + const allResults = [...rerankedBm25, ...vectorOnlyResults]; + + // Sort by score and return top results + allResults.sort((a, b) => b.score - a.score); + + return allResults.slice(0, opts.finalLimit); +} + diff --git a/src/plugins/built-in/globalSearch/src/search/searchUtils.ts b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts index 6982e286..3343e839 100644 --- a/src/plugins/built-in/globalSearch/src/search/searchUtils.ts +++ b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts @@ -6,32 +6,79 @@ import type { IndexItem } from "../indexing/types"; import { searchVectors } from "./vector/vectorSearch"; import type { VectorSearchResult } from "./vector/vectorTypes"; import { jobs } from "../indexing/jobs"; +import { hybridSearchWithExpansion } from "./hybridSearch"; + +// Search result cache for better performance +const searchCache = new Map(); +const CACHE_TTL = 1000 * 60 * 5; // 5 minutes +const MAX_CACHE_SIZE = 100; + +function getCachedResults(query: string): CombinedResult[] | null { + const cached = searchCache.get(query); + if (cached && Date.now() - cached.timestamp < CACHE_TTL) { + return cached.results; + } + return null; +} + +function setCachedResults(query: string, results: CombinedResult[]) { + // Limit cache size + if (searchCache.size >= MAX_CACHE_SIZE) { + const firstKey = searchCache.keys().next().value; + searchCache.delete(firstKey); + } + searchCache.set(query, { results, timestamp: Date.now() }); +} + +/** + * Clears the search result cache + */ +export function clearSearchCache(): void { + searchCache.clear(); + console.debug("[Search] Search result cache cleared"); +} + +// Listen for cache clear events (e.g., on extension update) +if (typeof window !== 'undefined') { + window.addEventListener('betterseqta-clear-search-cache', () => { + clearSearchCache(); + }); +} export function createSearchIndexes() { const commands = getStaticCommands(); const dynamicItems = getDynamicItems(); + // Optimized command search options const commandOptions = { keys: ["text", "category", "keywords"], includeScore: true, includeMatches: true, - threshold: 0.4, + threshold: 0.35, // Slightly more permissive for better recall minMatchCharLength: 2, useExtendedSearch: false, + ignoreLocation: false, + findAllMatches: false, // Performance optimization }; + // Optimized dynamic content search options const dynamicOptions = { keys: [ - { name: "text", weight: 2 }, + { name: "text", weight: 3 }, // Increased weight for title matches { name: "content", weight: 1 }, - { name: "category", weight: 1 }, + { name: "category", weight: 0.5 }, // Lower weight for category + { name: "metadata.subjectName", weight: 1.5 }, // Boost subject name matches + { name: "metadata.subjectCode", weight: 1.5 }, // Boost subject code matches ], includeScore: true, includeMatches: true, - threshold: 0.4, - minMatchCharLength: 2, - distance: 100, + threshold: 0.5, // More permissive for better partial word matching (increased from 0.4) + minMatchCharLength: 2, // Minimum 2 characters for Fuse.js matches (substring fallback handles shorter queries) + distance: 100, // Increased to allow matches across longer strings useExtendedSearch: true, + ignoreLocation: true, // Allow matches anywhere in the string for better partial word matching + findAllMatches: true, // Enable to find all matches for better partial word support + shouldSort: true, }; return { @@ -105,17 +152,63 @@ export function searchDynamicItems( } const now = Date.now(); - const searchResults = dynamicContentFuse.search(query, { limit }); + const queryLower = query.toLowerCase(); + const queryTrimmed = query.trim(); + + // For short queries (3 chars or less), use a more permissive approach + const isShortQuery = queryTrimmed.length <= 3; + const searchLimit = Math.min(limit * 3, 50); + + // First, try Fuse.js search + const searchResults = dynamicContentFuse.search(query, { limit: searchLimit }); + + // For short queries, always do a simple substring match to supplement Fuse.js results + // This ensures we catch partial word matches like "SAT" in "SAT 1: Differential Calculus" + let additionalMatches: IndexItem[] = []; + if (isShortQuery) { + // Always do substring search for short queries to catch partial word matches + for (const item of dynamicIdToItemMap.values()) { + const textLower = item.text.toLowerCase(); + const contentLower = (item.content || '').toLowerCase(); + const subjectNameLower = (item.metadata?.subjectName || '').toLowerCase(); + const subjectCodeLower = (item.metadata?.subjectCode || '').toLowerCase(); + + // Check if query appears anywhere in the text, content, or metadata + if (textLower.includes(queryLower) || + contentLower.includes(queryLower) || + subjectNameLower.includes(queryLower) || + subjectCodeLower.includes(queryLower)) { + // Only add if not already in Fuse.js results + if (!searchResults.find(r => r.item.id === item.id)) { + additionalMatches.push(item); + } + } + } + } - return searchResults.map((result: FuseResult) => { + const results = searchResults.map((result: FuseResult) => { const item = result.item; const fuseScore = 10 * (1 - (result.score || 0.5)); let score = fuseScore; + // Recency boost const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); const recencyBoost = sortByRecent ? 1 / (ageInDays + 1) : 0; score += recencyBoost; + + // Boost for exact text matches (especially at the start) + const textLower = item.text.toLowerCase(); + if (textLower.startsWith(queryLower)) { + score += 5; // Strong boost for prefix matches + } else if (textLower.includes(queryLower)) { + score += 2; // Boost for substring matches + } + + // Boost for category matches + if (item.category.toLowerCase().includes(queryLower)) { + score += 1; + } return { id: item.id, @@ -125,60 +218,124 @@ export function searchDynamicItems( matches: result.matches, }; }); + + // Add additional matches from simple substring search + additionalMatches.forEach((item) => { + // Check if already in results + if (!results.find(r => r.id === item.id)) { + const textLower = item.text.toLowerCase(); + let score = 5; // Base score for substring matches + + // Boost for prefix matches + if (textLower.startsWith(queryLower)) { + score += 5; + } + + // Recency boost + const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); + const recencyBoost = sortByRecent ? 1 / (ageInDays + 1) : 0; + score += recencyBoost; + + results.push({ + id: item.id, + type: "dynamic" as const, + score, + item, + }); + } + }); + + // Sort by score and return top results + return results.sort((a, b) => b.score - a.score).slice(0, limit); } export async function performSearch( query: string, commandsFuse: Fuse, commandIdToItemMap: Map, + dynamicContentFuse?: Fuse, + dynamicIdToItemMap?: Map, + sortByRecent: boolean = true, ): Promise { - // Get all results first + const trimmedQuery = query.trim().toLowerCase(); + + // Check cache first + if (trimmedQuery.length > 2) { + const cached = getCachedResults(trimmedQuery); + if (cached) { + return cached; + } + } + + // Step 1: Get command results (these don't need hybrid search) const commandResults = searchCommands( commandsFuse, - query, + trimmedQuery, commandIdToItemMap, ); - // Get vector results in parallel - let vectorResults: VectorSearchResult[] = []; - try { - vectorResults = await searchVectors(query); - } catch (e) {} + // Step 2: Get BM25 results for dynamic items + let dynamicResults: CombinedResult[] = []; + if (dynamicContentFuse && dynamicIdToItemMap) { + // Get BM25 results first (fast text-based search) + const bm25Results = searchDynamicItems( + dynamicContentFuse, + trimmedQuery, + dynamicIdToItemMap, + 50, // Get top 50 for reranking + sortByRecent, + ); - // Create a map to store our final results, using ID as key to avoid duplicates - const resultMap = new Map(); - - // Add command results first (they keep their original scores) - commandResults.forEach((r) => resultMap.set(r.id, r)); - - // Process dynamic results and vector results together - const seenIds = new Set(); - - vectorResults.forEach((v) => { - const id = v.object.id; - - if (!seenIds.has(id)) { - // This is a semantic match that Fuse missed - add it with the vector similarity as score - let score = v.similarity * 0.5; // High base score for semantic matches - const job = jobs[v.object.category]; - if (job && typeof job.boostCriteria === 'function') { - const boost = job.boostCriteria(v.object, query); - if (boost) { - score += boost; - } + // Step 3: Apply hybrid search (BM25 + Vector reranking + boosting) + if (trimmedQuery.length > 2 && bm25Results.length > 0) { + try { + // Get all items for expansion + const allItems = Array.from(dynamicIdToItemMap.values()); + + // Apply hybrid search with expansion + dynamicResults = await hybridSearchWithExpansion( + bm25Results, + trimmedQuery, + allItems, + { + bm25TopK: 50, + finalLimit: 20, // Return top 20 after reranking + recencyBoost: sortByRecent, + bm25Weight: 0.4, // 40% BM25, 60% vector + vectorWeight: 0.6, + recencyWeight: 0.1, + }, + ); + } catch (e) { + console.warn("[Search] Hybrid search failed, using BM25 only:", e); + // Fallback to BM25 only + dynamicResults = bm25Results.slice(0, 20); } - resultMap.set(id, { - id, - type: "dynamic" as const, - score, - item: v.object, - }); + } else { + // For very short queries or no BM25 results, use BM25 only + dynamicResults = bm25Results.slice(0, 20); } + } + + // Step 4: Combine command and dynamic results + const allResults = [...commandResults, ...dynamicResults]; + + // Sort by score (commands typically have higher priority) + allResults.sort((a, b) => { + // Commands always come first if scores are similar + if (a.type === "command" && b.type === "dynamic") { + return b.score - a.score - 10; // Commands get +10 boost + } + if (a.type === "dynamic" && b.type === "command") { + return b.score - a.score + 10; // Commands get +10 boost + } + return b.score - a.score; }); - // Convert to array and sort by score - const results = Array.from(resultMap.values()); - results.sort((a, b) => b.score - a.score); + // Cache results for queries longer than 2 chars + if (trimmedQuery.length > 2) { + setCachedResults(trimmedQuery, allResults); + } - return results; + return allResults; } diff --git a/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts index ee4bb332..59013d60 100644 --- a/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts +++ b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts @@ -1,16 +1,36 @@ import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia"; import type { IndexItem } from "../../indexing/types"; import type { SearchResult } from "embeddia"; +import { isVectorSearchSupported } from "../../utils/browserDetection"; let vectorIndex: EmbeddingIndex | null = null; +let initializationAttempted = false; +let initializationFailed = false; export async function initVectorSearch() { + // Skip initialization if already attempted and failed, or if not supported + if (initializationFailed || !isVectorSearchSupported()) { + if (!isVectorSearchSupported()) { + console.debug("[Vector Search] Vector search not supported in Firefox - using text search only"); + } + return; + } + + if (initializationAttempted) { + return; + } + + initializationAttempted = true; + try { await initializeModel(); vectorIndex = new EmbeddingIndex([]); vectorIndex.preloadIndexedDB(); + console.debug("[Vector Search] Initialized successfully"); } catch (e) { - console.error("Error initializing vector search", e); + console.warn("[Vector Search] Failed to initialize vector search (will use text search only):", e); + initializationFailed = true; + vectorIndex = null; } } @@ -18,28 +38,111 @@ export interface VectorSearchResult extends SearchResult { object: IndexItem & { embedding: number[] }; } +// Cache for query embeddings to avoid recomputing +const embeddingCache = new Map(); +const EMBEDDING_CACHE_TTL = 1000 * 60 * 30; // 30 minutes +const MAX_EMBEDDING_CACHE_SIZE = 50; + +function getCachedEmbedding(query: string): number[] | null { + const cached = embeddingCache.get(query); + if (cached) { + return cached; + } + return null; +} + +function setCachedEmbedding(query: string, embedding: number[]) { + // Limit cache size + if (embeddingCache.size >= MAX_EMBEDDING_CACHE_SIZE) { + const firstKey = embeddingCache.keys().next().value; + embeddingCache.delete(firstKey); + } + embeddingCache.set(query, embedding); +} + +/** + * Clears the embedding cache + */ +export function clearEmbeddingCache(): void { + embeddingCache.clear(); + console.debug("[Vector Search] Embedding cache cleared"); +} + +// Listen for cache clear events (e.g., on extension update) +if (typeof window !== 'undefined') { + window.addEventListener('betterseqta-clear-embedding-cache', () => { + clearEmbeddingCache(); + }); +} + export async function searchVectors( query: string, topK: number = 20, ): Promise { - if (!vectorIndex) await initVectorSearch(); + // Return empty array if vector search is not supported or failed to initialize + if (!isVectorSearchSupported() || initializationFailed) { + return []; + } - const queryEmbedding = await getEmbedding(query.slice(0, 100)); + if (!vectorIndex) { + await initVectorSearch(); + if (!vectorIndex) { + return []; + } + } - const results = await vectorIndex!.search(queryEmbedding, { - topK, - useStorage: "indexedDB", - dedupeEntries: true, - }); + // Normalize query for caching + const normalizedQuery = query.trim().toLowerCase().slice(0, 100); + + // Check cache first + let queryEmbedding = getCachedEmbedding(normalizedQuery); + + if (!queryEmbedding) { + try { + queryEmbedding = await getEmbedding(normalizedQuery); + setCachedEmbedding(normalizedQuery, queryEmbedding); + } catch (e) { + console.warn("[Vector Search] Failed to get embedding:", e); + return []; + } + } - // filter results with a similarity below 0.81 - const filteredResults = results.filter((r) => r.similarity > 0.81); + try { + const results = await vectorIndex!.search(queryEmbedding, { + topK: Math.min(topK * 2, 30), // Get more results, filter later + useStorage: "indexedDB", + dedupeEntries: true, + }); - return filteredResults as VectorSearchResult[]; + // Filter results with a similarity below 0.80 (slightly more permissive) + // and sort by similarity descending + const filteredResults = results + .filter((r) => r.similarity > 0.80) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, topK); + + return filteredResults as VectorSearchResult[]; + } catch (e) { + console.warn("[Vector Search] Search failed:", e); + return []; + } } export async function refreshVectorCache() { - if (!vectorIndex) await initVectorSearch(); - vectorIndex!.clearIndexedDBCache(); - vectorIndex!.preloadIndexedDB(); + if (!isVectorSearchSupported() || initializationFailed) { + return; + } + + if (!vectorIndex) { + await initVectorSearch(); + } + + if (vectorIndex) { + try { + vectorIndex.clearIndexedDBCache(); + vectorIndex.preloadIndexedDB(); + } catch (e) { + console.warn("[Vector Search] Failed to refresh cache:", e); + } + } } diff --git a/src/plugins/built-in/globalSearch/src/utils/browserDetection.ts b/src/plugins/built-in/globalSearch/src/utils/browserDetection.ts new file mode 100644 index 00000000..7ad87c9f --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/utils/browserDetection.ts @@ -0,0 +1,30 @@ +import browser from "webextension-polyfill"; + +/** + * Detects if the current browser is Firefox + */ +export function isFirefox(): boolean { + try { + // Firefox-specific API + if (typeof (browser.runtime as any).getBrowserInfo === "function") { + return true; + } + // Fallback: check user agent + if (typeof navigator !== "undefined") { + return navigator.userAgent.toLowerCase().includes("firefox"); + } + return false; + } catch { + // If we can't detect, assume not Firefox (safer for Chrome/Edge) + return false; + } +} + +/** + * Checks if vector search is supported in the current browser + * Currently disabled for Firefox due to security restrictions + */ +export function isVectorSearchSupported(): boolean { + return !isFirefox(); +} + diff --git a/src/plugins/built-in/globalSearch/src/utils/versionCheck.ts b/src/plugins/built-in/globalSearch/src/utils/versionCheck.ts new file mode 100644 index 00000000..31f9aa3d --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/utils/versionCheck.ts @@ -0,0 +1,115 @@ +import browser from "webextension-polyfill"; + +const VERSION_STORAGE_KEY = "betterseqta-global-search-version"; +const VERSION_CACHE_KEY = "betterseqta-global-search-cache-version"; + +/** + * Gets the current extension version from the manifest + */ +export function getCurrentVersion(): string { + try { + return browser.runtime.getManifest().version; + } catch (e) { + console.warn("[Version Check] Failed to get manifest version:", e); + return "0.0.0"; + } +} + +/** + * Gets the last stored version from localStorage + */ +export function getStoredVersion(): string | null { + try { + return localStorage.getItem(VERSION_STORAGE_KEY); + } catch (e) { + console.warn("[Version Check] Failed to get stored version:", e); + return null; + } +} + +/** + * Stores the current version in localStorage + */ +export function storeVersion(version: string): void { + try { + localStorage.setItem(VERSION_STORAGE_KEY, version); + localStorage.setItem(VERSION_CACHE_KEY, version); + } catch (e) { + console.warn("[Version Check] Failed to store version:", e); + } +} + +/** + * Checks if the extension has been updated and clears caches if needed + * Returns true if an update was detected + */ +export async function checkAndHandleUpdate(): Promise { + const currentVersion = getCurrentVersion(); + const storedVersion = getStoredVersion(); + + // If no stored version, this is first run - store current version + if (!storedVersion) { + console.debug(`[Version Check] First run detected, storing version ${currentVersion}`); + storeVersion(currentVersion); + return false; + } + + // If versions match, no update + if (storedVersion === currentVersion) { + return false; + } + + // Version mismatch detected - extension was updated + console.log(`[Version Check] Extension updated from ${storedVersion} to ${currentVersion}, clearing caches...`); + + // Clear all caches + await clearAllCaches(); + + // Store new version + storeVersion(currentVersion); + + return true; +} + +/** + * Clears all search-related caches + */ +export async function clearAllCaches(): Promise { + try { + // Clear search result cache (in-memory Map) + if (typeof window !== 'undefined') { + // Dispatch event to clear caches in other modules + window.dispatchEvent(new CustomEvent('betterseqta-clear-search-cache')); + window.dispatchEvent(new CustomEvent('betterseqta-clear-embedding-cache')); + } + + // Also try to directly clear caches if modules are already loaded + // Use setTimeout to avoid blocking and handle CSS preload errors + setTimeout(async () => { + try { + const { clearSearchCache } = await import("../search/searchUtils"); + clearSearchCache(); + } catch (e: any) { + // Module might not be loaded yet, or CSS preload error - that's okay + if (!e?.message?.includes("preload CSS") && !e?.message?.includes("MIME type")) { + console.debug("[Version Check] Could not clear search cache:", e); + } + } + + try { + const { clearEmbeddingCache } = await import("../search/vector/vectorSearch"); + clearEmbeddingCache(); + } catch (e: any) { + // Module might not be loaded yet, or CSS preload error - that's okay + if (!e?.message?.includes("preload CSS") && !e?.message?.includes("MIME type")) { + console.debug("[Version Check] Could not clear embedding cache:", e); + } + } + }, 50); + + console.debug("[Version Check] All caches cleared"); + } catch (e) { + console.error("[Version Check] Error clearing caches:", e); + } +} + diff --git a/src/plugins/core/dynamicLoader.ts b/src/plugins/core/dynamicLoader.ts index 75e32222..fd76b0e8 100644 --- a/src/plugins/core/dynamicLoader.ts +++ b/src/plugins/core/dynamicLoader.ts @@ -47,7 +47,17 @@ export function createLazyPlugin