From 29cfb4c79293bce990ff19a44e144fe55634c593 Mon Sep 17 00:00:00 2001 From: StroepWafel <109832156+StroepWafel@users.noreply.github.com> Date: Thu, 22 Jan 2026 11:28:43 +1030 Subject: [PATCH] improve global search --- .../src/components/SearchBar.svelte | 8 +- .../globalSearch/src/indexing/jobs.ts | 2 + .../src/indexing/jobs/assignments.ts | 257 ++++++++++++++++++ .../globalSearch/src/search/searchUtils.ts | 96 ++++++- .../src/search/vector/vectorSearch.ts | 63 ++++- 5 files changed, 402 insertions(+), 24 deletions(-) create mode 100644 src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts diff --git a/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte index f02d91c8..759bedc3 100644 --- a/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte +++ b/src/plugins/built-in/globalSearch/src/components/SearchBar.svelte @@ -176,13 +176,19 @@ isLoading = false; }; - const debouncedPerformSearch = debounce(performSearch, 20); + // Optimized debounce: shorter delay for better responsiveness + const debouncedPerformSearch = debounce(performSearch, 50); $effect(() => { if (commandPalleteOpen) { if (searchTerm === '') { + // Immediate search for empty query (shows recent items) + performSearch(); + } else if (searchTerm.length <= 2) { + // Immediate search for very short queries performSearch(); } else { + // Debounced search for longer queries debouncedPerformSearch(); } tick().then(() => searchbar?.focus()); diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts index 20cb36e2..659d2bc0 100644 --- a/src/plugins/built-in/globalSearch/src/indexing/jobs.ts +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs.ts @@ -3,10 +3,12 @@ import { messagesJob } from "./jobs/messages"; import { notificationsJob } from "./jobs/notifications"; import { forumsJob } from "./jobs/forums"; import { subjectsJob } from "./jobs/subjects"; +import { assignmentsJob } from "./jobs/assignments"; export const jobs: Record = { messages: messagesJob, notifications: notificationsJob, forums: forumsJob, subjects: subjectsJob, + assignments: assignmentsJob, }; diff --git a/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts b/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts new file mode 100644 index 00000000..9bd06eb7 --- /dev/null +++ b/src/plugins/built-in/globalSearch/src/indexing/jobs/assignments.ts @@ -0,0 +1,257 @@ +import type { Job, IndexItem } from "../types"; +import { htmlToPlainText } from "../utils"; + +const fetchJSON = async (url: string, body: any) => { + const res = await fetch(`${location.origin}${url}`, { + method: "POST", + credentials: "include", + headers: { "Content-Type": "application/json; charset=utf-8" }, + body: JSON.stringify(body), + }); + return res.json(); +}; + +const fetchUpcomingAssessments = async (student: number = 69) => { + try { + const res = await fetchJSON("/seqta/student/assessment/list/upcoming?", { + student, + }); + return res.payload || []; + } catch (e) { + console.error("[Assignments job] Failed to fetch upcoming assessments:", e); + return []; + } +}; + +const fetchSubjects = async () => { + try { + const res = await fetchJSON("/seqta/student/load/subjects?", {}); + return res.payload + ?.filter((s: any) => s.active === 1) + ?.flatMap((s: any) => s.subjects) || []; + } catch (e) { + console.error("[Assignments job] Failed to fetch subjects:", e); + return []; + } +}; + +const fetchPastAssessments = async (student: number = 69, subjects: any[]) => { + const map: Record = {}; + + for (const subject of subjects) { + try { + const res = await fetchJSON("/seqta/student/assessment/list/past?", { + student, + metaclass: subject.metaclass, + programme: subject.programme, + }); + + if (res.payload && Array.isArray(res.payload)) { + res.payload.forEach((assessment: any) => { + if (assessment && assessment.id) { + map[assessment.id] = assessment; + } + }); + } + } catch (e) { + console.warn(`[Assignments job] Failed to fetch past assessments for subject ${subject.code}:`, e); + } + } + + return Object.values(map); +}; + +const fetchAssessmentDetails = async ( + assessmentId: number, + metaclassId: number, + programmeId: number, +): Promise => { + try { + const res = await fetchJSON("/seqta/student/assessment/view?", { + id: assessmentId, + metaclass: metaclassId, + programme: programmeId, + }); + + if (res.payload && res.payload.description) { + return htmlToPlainText(res.payload.description); + } + return null; + } catch (e) { + console.warn(`[Assignments job] Failed to fetch details for assessment ${assessmentId}:`, e); + return null; + } +}; + +export const assignmentsJob: Job = { + id: "assignments", + label: "Assignments", + renderComponentId: "assessment", + frequency: { type: "expiry", afterMs: 1000 * 60 * 60 * 24 }, // Daily + + boostCriteria: (item, searchTerm) => { + if (searchTerm === "") { + return -100; + } + + let score = 0; + + // Boost upcoming assignments + if (item.metadata.dueDate) { + const dueDate = new Date(item.metadata.dueDate).getTime(); + const now = Date.now(); + const daysUntilDue = (dueDate - now) / (1000 * 60 * 60 * 24); + + if (daysUntilDue >= 0 && daysUntilDue <= 7) { + score += 0.05; // Boost assignments due within a week + } + if (daysUntilDue < 0) { + score -= 0.1; // Penalty for overdue assignments + } + } + + // Boost if submitted + if (item.metadata.submitted) { + score += 0.02; + } + + return score; + }, + + run: async (ctx) => { + const existingIds = new Set( + (await ctx.getStoredItems("assignments")).map((i) => i.id), + ); + + const student = 69; // TODO: Get from context if available + + // Fetch data in parallel + const [upcoming, subjects] = await Promise.all([ + fetchUpcomingAssessments(student), + fetchSubjects(), + ]); + + // Fetch past assessments + const past = await fetchPastAssessments(student, subjects); + + // Combine and deduplicate + const allAssessments = new Map(); + + upcoming.forEach((a: any) => { + if (a && a.id) { + allAssessments.set(a.id, { ...a, isUpcoming: true }); + } + }); + + past.forEach((a: any) => { + if (a && a.id) { + const existing = allAssessments.get(a.id); + if (existing) { + Object.assign(existing, a); + } else { + allAssessments.set(a.id, { ...a, isUpcoming: false }); + } + } + }); + + const items: IndexItem[] = []; + const processedIds = new Set(); + + // Process assessments in batches to avoid overwhelming the API + const assessmentArray = Array.from(allAssessments.values()); + const batchSize = 15; // Increased batch size for better performance + + // Only fetch details for upcoming assignments to reduce API calls + const upcomingAssessments = assessmentArray.filter(a => a.isUpcoming); + const detailPromises = new Map>(); + + // Pre-fetch details for upcoming assessments only (most important) + for (const assessment of upcomingAssessments.slice(0, 20)) { + if (assessment.metaclass && assessment.programme) { + const id = `assignment-${assessment.id}`; + detailPromises.set(id, fetchAssessmentDetails( + assessment.id, + assessment.metaclass, + assessment.programme, + )); + } + } + + // Process all assessments + for (let i = 0; i < assessmentArray.length; i += batchSize) { + const batch = assessmentArray.slice(i, i + batchSize); + + const batchItems = await Promise.all( + batch.map(async (assessment) => { + const id = `assignment-${assessment.id}`; + + if (existingIds.has(id) || processedIds.has(id)) { + return null; + } + + processedIds.add(id); + + // Only fetch details for upcoming assignments (already pre-fetched) + let description = ""; + const detailPromise = detailPromises.get(id); + if (detailPromise) { + description = (await detailPromise) || ""; + } + + const subjectName = assessment.subject || assessment.code || "Unknown Subject"; + const dueDate = assessment.due ? new Date(assessment.due).getTime() : null; + + const item: IndexItem = { + id, + text: assessment.title || assessment.name || "Untitled Assignment", + category: "assignments", + content: `${description}\nSubject: ${subjectName}\nDue: ${assessment.due || "No due date"}`.trim(), + dateAdded: dueDate || Date.now(), + metadata: { + assessmentId: assessment.id, + subject: subjectName, + subjectCode: assessment.code, + dueDate: assessment.due, + programmeId: assessment.programme, + metaclassId: assessment.metaclass, + submitted: assessment.submitted || false, + isUpcoming: assessment.isUpcoming || false, + term: assessment.term, + }, + actionId: "assessment", + renderComponentId: "assessment", + }; + + return item; + }) + ); + + // Filter out nulls and add to items + batchItems.forEach(item => { + if (item) { + items.push(item); + } + }); + + // Small delay between batches to avoid rate limiting + if (i + batchSize < assessmentArray.length) { + await new Promise(resolve => setTimeout(resolve, 50)); // Reduced delay + } + } + + console.debug(`[Assignments job] Indexed ${items.length} assignment items`); + return items; + }, + + purge: (items) => { + const oneYearAgo = Date.now() - 365 * 24 * 60 * 60 * 1000; + return items.filter((i) => { + // Keep upcoming assignments and assignments from the last year + if (i.metadata.isUpcoming) { + return true; + } + return i.dateAdded >= oneYearAgo; + }); + }, +}; + diff --git a/src/plugins/built-in/globalSearch/src/search/searchUtils.ts b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts index 6982e286..1d05d77f 100644 --- a/src/plugins/built-in/globalSearch/src/search/searchUtils.ts +++ b/src/plugins/built-in/globalSearch/src/search/searchUtils.ts @@ -7,31 +7,62 @@ import { searchVectors } from "./vector/vectorSearch"; import type { VectorSearchResult } from "./vector/vectorTypes"; import { jobs } from "../indexing/jobs"; +// Search result cache for better performance +const searchCache = new Map(); +const CACHE_TTL = 1000 * 60 * 5; // 5 minutes +const MAX_CACHE_SIZE = 100; + +function getCachedResults(query: string): CombinedResult[] | null { + const cached = searchCache.get(query); + if (cached && Date.now() - cached.timestamp < CACHE_TTL) { + return cached.results; + } + return null; +} + +function setCachedResults(query: string, results: CombinedResult[]) { + // Limit cache size + if (searchCache.size >= MAX_CACHE_SIZE) { + const firstKey = searchCache.keys().next().value; + searchCache.delete(firstKey); + } + searchCache.set(query, { results, timestamp: Date.now() }); +} + export function createSearchIndexes() { const commands = getStaticCommands(); const dynamicItems = getDynamicItems(); + // Optimized command search options const commandOptions = { keys: ["text", "category", "keywords"], includeScore: true, includeMatches: true, - threshold: 0.4, + threshold: 0.35, // Slightly more permissive for better recall minMatchCharLength: 2, useExtendedSearch: false, + ignoreLocation: false, + findAllMatches: false, // Performance optimization }; + // Optimized dynamic content search options const dynamicOptions = { keys: [ - { name: "text", weight: 2 }, + { name: "text", weight: 3 }, // Increased weight for title matches { name: "content", weight: 1 }, - { name: "category", weight: 1 }, + { name: "category", weight: 0.5 }, // Lower weight for category + { name: "metadata.subjectName", weight: 1.5 }, // Boost subject name matches + { name: "metadata.subjectCode", weight: 1.5 }, // Boost subject code matches ], includeScore: true, includeMatches: true, - threshold: 0.4, + threshold: 0.35, // Slightly more permissive minMatchCharLength: 2, - distance: 100, + distance: 50, // Reduced from 100 for better performance useExtendedSearch: true, + ignoreLocation: false, + findAllMatches: false, // Performance optimization + shouldSort: true, }; return { @@ -105,17 +136,30 @@ export function searchDynamicItems( } const now = Date.now(); - const searchResults = dynamicContentFuse.search(query, { limit }); + // Increase limit for better results, then trim later + const searchLimit = Math.min(limit * 3, 50); + const searchResults = dynamicContentFuse.search(query, { limit: searchLimit }); - return searchResults.map((result: FuseResult) => { + const results = searchResults.map((result: FuseResult) => { const item = result.item; const fuseScore = 10 * (1 - (result.score || 0.5)); let score = fuseScore; + // Recency boost const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24); const recencyBoost = sortByRecent ? 1 / (ageInDays + 1) : 0; score += recencyBoost; + + // Boost for exact text matches + if (item.text.toLowerCase().includes(query.toLowerCase())) { + score += 2; + } + + // Boost for category matches + if (item.category.toLowerCase().includes(query.toLowerCase())) { + score += 1; + } return { id: item.id, @@ -125,6 +169,9 @@ export function searchDynamicItems( matches: result.matches, }; }); + + // Sort by score and return top results + return results.sort((a, b) => b.score - a.score).slice(0, limit); } export async function performSearch( @@ -132,18 +179,32 @@ export async function performSearch( commandsFuse: Fuse, commandIdToItemMap: Map, ): Promise { + const trimmedQuery = query.trim().toLowerCase(); + + // Check cache first + if (trimmedQuery.length > 2) { + const cached = getCachedResults(trimmedQuery); + if (cached) { + return cached; + } + } + // Get all results first const commandResults = searchCommands( commandsFuse, - query, + trimmedQuery, commandIdToItemMap, ); - // Get vector results in parallel + // Get vector results in parallel (only for queries longer than 3 chars for performance) let vectorResults: VectorSearchResult[] = []; - try { - vectorResults = await searchVectors(query); - } catch (e) {} + if (trimmedQuery.length > 3) { + try { + vectorResults = await searchVectors(trimmedQuery, 15); // Reduced from 20 for performance + } catch (e) { + console.warn("[Search] Vector search failed:", e); + } + } // Create a map to store our final results, using ID as key to avoid duplicates const resultMap = new Map(); @@ -151,8 +212,9 @@ export async function performSearch( // Add command results first (they keep their original scores) commandResults.forEach((r) => resultMap.set(r.id, r)); - // Process dynamic results and vector results together + // Process vector results const seenIds = new Set(); + commandResults.forEach((r) => seenIds.add(r.id)); vectorResults.forEach((v) => { const id = v.object.id; @@ -162,7 +224,7 @@ export async function performSearch( let score = v.similarity * 0.5; // High base score for semantic matches const job = jobs[v.object.category]; if (job && typeof job.boostCriteria === 'function') { - const boost = job.boostCriteria(v.object, query); + const boost = job.boostCriteria(v.object, trimmedQuery); if (boost) { score += boost; } @@ -173,6 +235,7 @@ export async function performSearch( score, item: v.object, }); + seenIds.add(id); } }); @@ -180,5 +243,10 @@ export async function performSearch( const results = Array.from(resultMap.values()); results.sort((a, b) => b.score - a.score); + // Cache results for queries longer than 2 chars + if (trimmedQuery.length > 2) { + setCachedResults(trimmedQuery, results); + } + return results; } diff --git a/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts index ee4bb332..4bb68590 100644 --- a/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts +++ b/src/plugins/built-in/globalSearch/src/search/vector/vectorSearch.ts @@ -18,24 +18,69 @@ export interface VectorSearchResult extends SearchResult { object: IndexItem & { embedding: number[] }; } +// Cache for query embeddings to avoid recomputing +const embeddingCache = new Map(); +const EMBEDDING_CACHE_TTL = 1000 * 60 * 30; // 30 minutes +const MAX_EMBEDDING_CACHE_SIZE = 50; + +function getCachedEmbedding(query: string): number[] | null { + const cached = embeddingCache.get(query); + if (cached) { + return cached; + } + return null; +} + +function setCachedEmbedding(query: string, embedding: number[]) { + // Limit cache size + if (embeddingCache.size >= MAX_EMBEDDING_CACHE_SIZE) { + const firstKey = embeddingCache.keys().next().value; + embeddingCache.delete(firstKey); + } + embeddingCache.set(query, embedding); +} + export async function searchVectors( query: string, topK: number = 20, ): Promise { if (!vectorIndex) await initVectorSearch(); - const queryEmbedding = await getEmbedding(query.slice(0, 100)); + // Normalize query for caching + const normalizedQuery = query.trim().toLowerCase().slice(0, 100); + + // Check cache first + let queryEmbedding = getCachedEmbedding(normalizedQuery); + + if (!queryEmbedding) { + try { + queryEmbedding = await getEmbedding(normalizedQuery); + setCachedEmbedding(normalizedQuery, queryEmbedding); + } catch (e) { + console.warn("[Vector Search] Failed to get embedding:", e); + return []; + } + } - const results = await vectorIndex!.search(queryEmbedding, { - topK, - useStorage: "indexedDB", - dedupeEntries: true, - }); + try { + const results = await vectorIndex!.search(queryEmbedding, { + topK: Math.min(topK * 2, 30), // Get more results, filter later + useStorage: "indexedDB", + dedupeEntries: true, + }); - // filter results with a similarity below 0.81 - const filteredResults = results.filter((r) => r.similarity > 0.81); + // Filter results with a similarity below 0.80 (slightly more permissive) + // and sort by similarity descending + const filteredResults = results + .filter((r) => r.similarity > 0.80) + .sort((a, b) => b.similarity - a.similarity) + .slice(0, topK); - return filteredResults as VectorSearchResult[]; + return filteredResults as VectorSearchResult[]; + } catch (e) { + console.warn("[Vector Search] Search failed:", e); + return []; + } } export async function refreshVectorCache() {