feat: forums + improvements

This commit is contained in:
SethBurkart123
2025-05-05 19:49:19 +10:00
parent 0f9f618164
commit d3d9b45caa
17 changed files with 217 additions and 136 deletions
@@ -1,50 +0,0 @@
<script lang="ts">
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../utils/highlight';
import type { DynamicContentItem } from '../utils/dynamicItems';
import type { FuseResultMatch } from '../core/types';
const { item, isSelected, searchTerm, matches } = $props<{
item: DynamicContentItem;
isSelected: boolean;
searchTerm: string;
matches?: readonly FuseResultMatch[];
}>();
</script>
<button
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
>
<div class="flex items-center w-full">
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\ue924'}</div>
<span class="ml-4 text-lg truncate">
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
</span>
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
{item.category}
</span>
</div>
{#if item.content}
<div class="mt-1 ml-12 text-sm text-zinc-600 dark:text-zinc-400 line-clamp-2 text-start">
{@html stripHtmlButKeepHighlights(highlightSnippet(item.content, searchTerm, matches))}
</div>
{/if}
</button>
<style>
:global(.highlight) {
background-color: rgba(255, 213, 0, 0.3);
font-weight: 500;
border-radius: 2px;
padding: 0 1px;
margin: 0 -1px;
}
.dark :global(.highlight) {
background-color: rgba(255, 230, 100, 0.4);
}
.due-badge {
font-size: 0.65rem;
}
</style>
@@ -10,8 +10,9 @@
import Fuse from 'fuse.js';
import Calculator from './Calculator.svelte';
import { actionMap } from '../indexing/actions';
import type { IndexItem, HydratedIndexItem } from '../indexing/types';
import type { IndexItem } from '../indexing/types';
import debounce from 'lodash/debounce';
import { renderComponentMap } from '../indexing/renderComponents';
const {
transparencyEffects,
@@ -22,9 +23,9 @@
}>();
let commandsFuse = $state<Fuse<StaticCommandItem>>();
let dynamicContentFuse = $state<Fuse<HydratedIndexItem>>();
let dynamicContentFuse = $state<Fuse<IndexItem>>();
const dynamicIdToItemMap = $state(new Map<string, HydratedIndexItem>());
const dynamicIdToItemMap = $state(new Map<string, IndexItem>());
const commandIdToItemMap = $state(new Map<string, StaticCommandItem>());
let isIndexing = $state(false);
@@ -75,6 +76,7 @@
let combinedResults = $state<CombinedResult[]>([]);
let isLoading = $state(false);
let calculatorResult = $state<string | null>(null);
let resultsList = $state<HTMLUListElement>();
const updateCalculatorState = (hasResult: string | null) => {
calculatorResult = hasResult;
@@ -141,12 +143,20 @@
searchTerm = '';
selectedIndex = 0;
combinedResults = [];
tick().then(() => {
const selectedElement = resultsList?.querySelector(`li:nth-child(1)`);
selectedElement?.scrollIntoView({ block: 'nearest' });
});
}
});
$effect(() => {
if (combinedResults.length === 0 && calculatorResult && commandPalleteOpen) {
selectedIndex = 0;
tick().then(() => {
const selectedElement = resultsList?.querySelector(`li:nth-child(1)`);
selectedElement?.scrollIntoView({ block: 'nearest' });
});
}
});
@@ -154,16 +164,24 @@
const maxIndex = (calculatorResult ? 1 : 0) + combinedResults.length - 1;
if (selectedIndex < maxIndex) {
selectedIndex++;
tick().then(() => {
const selectedElement = resultsList?.querySelector(`li:nth-child(${selectedIndex + 1})`);
selectedElement?.scrollIntoView({ block: 'nearest' });
});
}
};
const selectPrev = () => {
if (selectedIndex > 0) {
selectedIndex--;
tick().then(() => {
const selectedElement = resultsList?.querySelector(`li:nth-child(${selectedIndex + 1})`);
selectedElement?.scrollIntoView({ block: 'nearest' });
});
}
};
function executeItemAction(item: StaticCommandItem | HydratedIndexItem) {
function executeItemAction(item: StaticCommandItem | IndexItem) {
if ('action' in item && typeof item.action === 'function') {
(item as StaticCommandItem).action();
} else if ('actionId' in item && item.actionId && actionMap[item.actionId]) {
@@ -240,7 +258,10 @@
/>
</div>
<ul class="overflow-y-auto max-h-[24rem] text-base scroll-py-2 p-1 gap-0.5 flex flex-col">
<ul
bind:this={resultsList}
class="overflow-y-auto max-h-[24rem] text-base scroll-py-2 p-2 gap-0.5 flex flex-col"
>
<Calculator
searchTerm={searchTerm}
isSelected={selectedIndex === 0}
@@ -270,14 +291,18 @@
{/if}
</button>
{:else if result.type === 'dynamic'}
{@const dynamicItem = item as HydratedIndexItem}
{#if dynamicItem.renderComponent}
<dynamicItem.renderComponent
{@const dynamicItem = item as IndexItem}
{@const RenderComponent = renderComponentMap[dynamicItem.renderComponentId]}
{#if RenderComponent}
<RenderComponent
item={dynamicItem}
isSelected={isSelected}
searchTerm={searchTerm}
matches={result.matches}
on:click={() => executeItemAction(dynamicItem)}
onclick={() => executeItemAction(dynamicItem)}
onkeydown={() => {}}
role="button"
tabindex="0"
/>
{:else}
<button
@@ -0,0 +1,32 @@
<script lang="ts">
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../../utils/highlight';
import type { DynamicContentItem } from '../../utils/dynamicItems';
import type { FuseResultMatch } from '../../core/types';
const { item, isSelected, searchTerm, matches } = $props<{
item: DynamicContentItem;
isSelected: boolean;
searchTerm: string;
matches?: readonly FuseResultMatch[];
}>();
</script>
<button
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
>
<div class="flex items-center w-full">
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\uebee'}</div>
<span class="ml-4 text-lg truncate">
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
</span>
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
{item.category}
</span>
</div>
{#if item.content}
<div class="mt-1 ml-12 text-sm text-zinc-600 dark:text-zinc-400 line-clamp-2 text-start">
{@html stripHtmlButKeepHighlights(highlightSnippet(item.content, searchTerm, matches))}
</div>
{/if}
</button>
@@ -0,0 +1,27 @@
<script lang="ts">
import { highlightMatch, stripHtmlButKeepHighlights } from '../../utils/highlight';
import type { DynamicContentItem } from '../../utils/dynamicItems';
import type { FuseResultMatch } from '../../core/types';
const { item, isSelected, searchTerm, matches } = $props<{
item: DynamicContentItem;
isSelected: boolean;
searchTerm: string;
matches?: readonly FuseResultMatch[];
}>();
</script>
<button
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
>
<div class="flex items-center w-full">
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\uebe7'}</div>
<span class="ml-4 text-lg truncate">
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
</span>
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
{item.category}
</span>
</div>
</button>
@@ -56,3 +56,16 @@
color: #aaa;
}
}
.highlight {
background-color: rgba(255, 213, 0, 0.3);
font-weight: 500;
border-radius: 2px;
padding: 0 1px;
margin: 0 -1px;
}
.dark .highlight {
background-color: rgba(255, 230, 100, 0.4);
}
@@ -1,5 +1,5 @@
import type { StaticCommandItem } from "./commands";
import type { HydratedIndexItem } from "../indexing/types";
import type { IndexItem } from "../indexing/types";
export interface MatchIndices {
readonly 0: number;
@@ -16,7 +16,7 @@ export interface CombinedResult {
id: string;
type: "command" | "dynamic";
score: number;
item: StaticCommandItem | HydratedIndexItem;
item: StaticCommandItem | IndexItem;
matches?: readonly FuseResultMatch[];
}
@@ -1,7 +1,7 @@
import { clear, getAll, get, put, remove } from "./db";
import { jobs } from "./jobs";
import { renderComponentMap } from "./renderComponents";
import type { HydratedIndexItem, IndexItem, Job, JobContext } from "./types";
import type { IndexItem, Job, JobContext } from "./types";
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
const META_STORE = "meta";
@@ -83,21 +83,14 @@ function dispatchProgress(
window.dispatchEvent(event);
}
export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
const all: HydratedIndexItem[] = [];
export async function loadAllStoredItems(): Promise<IndexItem[]> {
const all: IndexItem[] = [];
const jobIds = Object.keys(jobs);
for (const jobId of jobIds) {
try {
const items = (await getAll(jobId)) as IndexItem[];
const job = jobs[jobId];
const renderComponent = renderComponentMap[job.renderComponentId];
if (!renderComponent) {
console.warn(
`Render component not found for job ${jobId} (ID: ${job.renderComponentId})`,
);
}
for (const item of items) {
if (
@@ -108,10 +101,7 @@ export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
item.actionId &&
job.renderComponentId
) {
all.push({
...item,
renderComponent: renderComponent || undefined,
});
all.push(item);
} else {
console.warn(`Skipping invalid item from job ${jobId}:`, item);
}
@@ -144,7 +134,7 @@ export async function runIndexing(): Promise<void> {
const totalSteps = jobIds.length + 1;
dispatchProgress(completedJobs, totalSteps, true, "Starting jobs");
const allItemsFromJobs: HydratedIndexItem[] = [];
const allItemsFromJobs: IndexItem[] = [];
// --- Step 1: Run Fetching/Storing Jobs (Main Thread) ---
for (const jobId of jobIds) {
@@ -225,35 +215,7 @@ export async function runIndexing(): Promise<void> {
await setStoredItems(merged);
await updateLastRunMeta(jobId);
// Hydrate items for vector processing
const renderComponent = renderComponentMap[job.renderComponentId];
if (!renderComponent) {
console.warn(
`Render component not found for job ${jobId} (ID: ${job.renderComponentId}) during hydration`,
);
}
const hydratedItems = merged
.filter(
(item) =>
item &&
item.id &&
item.text &&
item.category &&
item.actionId &&
job.renderComponentId,
) // Filter invalid before hydrating
.map((item) => ({
...item,
renderComponent: renderComponent || undefined, // Assign undefined if not found
}));
if (hydratedItems.length !== merged.length) {
console.warn(
`[Indexer Job ${jobId}] Filtered out ${merged.length - hydratedItems.length} invalid items during hydration.`,
);
}
allItemsFromJobs.push(...hydratedItems);
allItemsFromJobs.push(...newItemsRaw);
console.debug(
`%c[Indexer] ${job.label}: ${newItemsRaw.length} new items from run, ${merged.length} total stored in '${jobId}' store (non-vector).`,
@@ -1,8 +1,10 @@
import type { Job } from "./types";
import { messagesJob } from "./jobs/messages";
import { assessmentsJob } from "./jobs/assessments";
import { forumsJob } from "./jobs/forums";
export const jobs: Record<string, Job> = {
messages: messagesJob,
assessments: assessmentsJob,
forums: forumsJob,
};
@@ -0,0 +1,69 @@
import type { Job, IndexItem } from "../types";
const fetchForums = async () => {
const res = await fetch(`${location.origin}/seqta/student/load/forums`, {
method: "POST",
credentials: "include",
headers: { "Content-Type": "application/json; charset=utf-8" },
body: JSON.stringify({ mode: "list" }),
});
return res.json() as Promise<{
payload: { forums: any[] };
status: string;
}>;
};
export const forumsJob: Job = {
id: "forums",
label: "Forums",
renderComponentId: "forum",
//frequency: { type: "expiry", afterMs: 30 * 24 * 60 * 60 * 1000 }, // 30 days
frequency: { type: "interval", ms: 1000 }, // 1s
run: async (ctx) => {
const existingIds = new Set(
(await ctx.getStoredItems("forums")).map((i) => i.id),
);
let list;
try {
list = await fetchForums();
} catch (e) {
console.error("[Forums job] list fetch failed:", e);
return [];
}
if (list.status !== "200") return [];
const items: IndexItem[] = [];
for (const forum of list.payload.forums) {
const id = forum.id.toString();
if (existingIds.has(id)) continue;
items.push({
id,
text: forum.title,
category: "forums",
content: `${forum.title}`,
dateAdded: Date.now(),
metadata: {
forumId: forum.id,
owner: forum.owner,
title: forum.title,
},
actionId: "forum",
renderComponentId: "forum",
});
}
return items;
},
/** Keep only forums from the last 2years. */
purge: (items) => {
const twoYearsAgo = Date.now() - 2 * 365 * 24 * 60 * 60 * 1000;
return items.filter((i) => i.dateAdded >= twoYearsAgo);
},
};
@@ -1,10 +1,15 @@
import type { SvelteComponent } from "svelte";
import AssessmentComponent from "../components/AssessmentItem.svelte";
import AssessmentItem from "../components/items/AssessmentItem.svelte";
import ForumItem from "../components/items/ForumItem.svelte";
// import other components as needed
export const renderComponentMap: Record<string, typeof SvelteComponent> = {
assessment: AssessmentComponent as unknown as typeof SvelteComponent,
message: AssessmentComponent as unknown as typeof SvelteComponent,
assessment: AssessmentItem as unknown as typeof SvelteComponent,
message: AssessmentItem as unknown as typeof SvelteComponent,
forum: ForumItem as unknown as typeof SvelteComponent,
// subject: SubjectComponent,
// etc...
};
void AssessmentItem;
void ForumItem;
@@ -11,10 +11,6 @@ export interface IndexItem {
renderComponentId: string;
}
export interface HydratedIndexItem extends IndexItem {
renderComponent: typeof SvelteComponent;
}
export type Frequency =
| "pageLoad"
| { type: "interval"; ms: number }
@@ -1,5 +1,5 @@
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
import type { HydratedIndexItem } from "../types";
import type { IndexItem } from "../types";
let vectorIndex: EmbeddingIndex | null = null;
let isInitialized = false;
@@ -35,8 +35,8 @@ async function initWorker() {
}
async function vectorizeItem(
item: HydratedIndexItem,
): Promise<(HydratedIndexItem & { embedding: number[] }) | null> {
item: IndexItem,
): Promise<(IndexItem & { embedding: number[] }) | null> {
// Simplified for brevity - assumes embedding function doesn't need cancellation signal
try {
const textToEmbed = [
@@ -57,7 +57,7 @@ async function vectorizeItem(
}
}
async function processItems(items: HydratedIndexItem[], signal: AbortSignal) {
async function processItems(items: IndexItem[], signal: AbortSignal) {
console.debug("Worker received process request.");
if (!vectorIndex) {
console.warn(
@@ -140,7 +140,7 @@ async function processItems(items: HydratedIndexItem[], signal: AbortSignal) {
const vectorizationResults = await Promise.all(batch.map(vectorizeItem));
const successfullyVectorized = vectorizationResults.filter(
(result) => result !== null,
) as (HydratedIndexItem & { embedding: number[] })[];
) as (IndexItem & { embedding: number[] })[];
if (signal.aborted) {
console.debug("Processing cancelled after vectorization batch.");
@@ -1,5 +1,5 @@
import { refreshVectorCache } from "../../search/vector/vectorSearch";
import type { HydratedIndexItem } from "../types";
import type { IndexItem } from "../types";
import vectorWorker from "./vectorWorker.ts?inlineWorker";
import type { SearchResult } from "embeddia";
@@ -156,7 +156,7 @@ export class VectorWorkerManager {
}
async processItems(
items: HydratedIndexItem[],
items: IndexItem[],
onProgress?: ProgressCallback,
) {
await this.ensureReady(); // Wait for worker to be ready
@@ -168,11 +168,10 @@ export class VectorWorkerManager {
console.debug(`Sending ${items.length} items to worker for processing.`);
const serialisableItems = items.map(({ renderComponent, ...rest }) => rest);
this.worker!.postMessage({
type: "process",
data: { items: serialisableItems },
data: { items: items },
});
}
@@ -2,7 +2,7 @@ import Fuse, { type FuseResult } from "fuse.js";
import { getStaticCommands, type StaticCommandItem } from "../core/commands";
import { getDynamicItems } from "../utils/dynamicItems";
import type { CombinedResult } from "../core/types";
import type { HydratedIndexItem } from "../indexing/types";
import type { IndexItem } from "../indexing/types";
import { searchVectors } from "./vector/vectorSearch";
import type { VectorSearchResult } from "./vector/vectorTypes";
@@ -41,7 +41,7 @@ export function createSearchIndexes() {
dynamicContentFuse: new Fuse(
dynamicItems,
dynamicOptions,
) as Fuse<HydratedIndexItem>,
) as Fuse<IndexItem>,
commands,
dynamicItems,
};
@@ -85,9 +85,9 @@ export function searchCommands(
}
export function searchDynamicItems(
dynamicContentFuse: Fuse<HydratedIndexItem>,
dynamicContentFuse: Fuse<IndexItem>,
query: string,
dynamicIdToItemMap: Map<string, HydratedIndexItem>,
dynamicIdToItemMap: Map<string, IndexItem>,
limit = 10,
sortByRecent: boolean = true, // Added option to control sorting
): CombinedResult[] {
@@ -109,7 +109,7 @@ export function searchDynamicItems(
const now = Date.now();
const searchResults = dynamicContentFuse.search(query, { limit });
return searchResults.map((result: FuseResult<HydratedIndexItem>) => {
return searchResults.map((result: FuseResult<IndexItem>) => {
const item = result.item;
const fuseScore = 10 * (1 - (result.score || 0.5));
const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24);
@@ -129,9 +129,9 @@ export function searchDynamicItems(
export async function performSearch(
query: string,
commandsFuse: Fuse<StaticCommandItem>,
dynamicContentFuse: Fuse<HydratedIndexItem>,
dynamicContentFuse: Fuse<IndexItem>,
commandIdToItemMap: Map<string, StaticCommandItem>,
dynamicIdToItemMap: Map<string, HydratedIndexItem>,
dynamicIdToItemMap: Map<string, IndexItem>,
showRecentFirst: boolean,
): Promise<CombinedResult[]> {
// Get all results first
@@ -182,6 +182,7 @@ export async function performSearch(
// Now add any vector results we haven't seen yet
vectorResults.forEach((v) => {
const id = v.object.id;
if (!seenIds.has(id)) {
// This is a semantic match that Fuse missed - add it with the vector similarity as score
resultMap.set(id, {
@@ -1,5 +1,5 @@
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
import type { HydratedIndexItem } from "../../indexing/types";
import type { IndexItem } from "../../indexing/types";
import type { SearchResult } from "embeddia";
let vectorIndex: EmbeddingIndex | null = null;
@@ -15,7 +15,7 @@ export async function initVectorSearch() {
}
export interface VectorSearchResult extends SearchResult {
object: HydratedIndexItem & { embedding: number[] };
object: IndexItem & { embedding: number[] };
}
export async function searchVectors(
@@ -1,6 +1,6 @@
import type { SearchResult } from "embeddia";
import type { HydratedIndexItem } from "../../indexing/types";
import type { IndexItem } from "../../indexing/types";
export interface VectorSearchResult extends SearchResult {
object: HydratedIndexItem & { embedding: number[] };
object: IndexItem & { embedding: number[] };
}
@@ -1,5 +1,5 @@
import type { SvelteComponent } from "svelte";
import type { HydratedIndexItem } from "./indexing/types";
import type { IndexItem } from "./indexing/types";
export interface DynamicContentItem {
id: string;
@@ -13,18 +13,18 @@ export interface DynamicContentItem {
renderComponent?: typeof SvelteComponent;
}
let dynamicItems: HydratedIndexItem[] = [];
let dynamicItems: IndexItem[] = [];
/**
* Loads a new set of dynamic items.
*/
export function loadDynamicItems(items: HydratedIndexItem[]) {
export function loadDynamicItems(items: IndexItem[]) {
dynamicItems = items;
}
/**
* Returns all currently loaded dynamic items.
*/
export function getDynamicItems(): HydratedIndexItem[] {
export function getDynamicItems(): IndexItem[] {
return dynamicItems;
}