mirror of
https://github.com/BetterSEQTA/BetterSEQTA-Plus.git
synced 2026-06-06 11:44:40 +00:00
feat: forums + improvements
This commit is contained in:
@@ -1,50 +0,0 @@
|
|||||||
<script lang="ts">
|
|
||||||
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../utils/highlight';
|
|
||||||
import type { DynamicContentItem } from '../utils/dynamicItems';
|
|
||||||
import type { FuseResultMatch } from '../core/types';
|
|
||||||
|
|
||||||
const { item, isSelected, searchTerm, matches } = $props<{
|
|
||||||
item: DynamicContentItem;
|
|
||||||
isSelected: boolean;
|
|
||||||
searchTerm: string;
|
|
||||||
matches?: readonly FuseResultMatch[];
|
|
||||||
}>();
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<button
|
|
||||||
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
|
|
||||||
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
|
|
||||||
>
|
|
||||||
<div class="flex items-center w-full">
|
|
||||||
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\ue924'}</div>
|
|
||||||
<span class="ml-4 text-lg truncate">
|
|
||||||
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
|
|
||||||
</span>
|
|
||||||
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
|
|
||||||
{item.category}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
{#if item.content}
|
|
||||||
<div class="mt-1 ml-12 text-sm text-zinc-600 dark:text-zinc-400 line-clamp-2 text-start">
|
|
||||||
{@html stripHtmlButKeepHighlights(highlightSnippet(item.content, searchTerm, matches))}
|
|
||||||
</div>
|
|
||||||
{/if}
|
|
||||||
</button>
|
|
||||||
|
|
||||||
<style>
|
|
||||||
:global(.highlight) {
|
|
||||||
background-color: rgba(255, 213, 0, 0.3);
|
|
||||||
font-weight: 500;
|
|
||||||
border-radius: 2px;
|
|
||||||
padding: 0 1px;
|
|
||||||
margin: 0 -1px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dark :global(.highlight) {
|
|
||||||
background-color: rgba(255, 230, 100, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.due-badge {
|
|
||||||
font-size: 0.65rem;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -10,8 +10,9 @@
|
|||||||
import Fuse from 'fuse.js';
|
import Fuse from 'fuse.js';
|
||||||
import Calculator from './Calculator.svelte';
|
import Calculator from './Calculator.svelte';
|
||||||
import { actionMap } from '../indexing/actions';
|
import { actionMap } from '../indexing/actions';
|
||||||
import type { IndexItem, HydratedIndexItem } from '../indexing/types';
|
import type { IndexItem } from '../indexing/types';
|
||||||
import debounce from 'lodash/debounce';
|
import debounce from 'lodash/debounce';
|
||||||
|
import { renderComponentMap } from '../indexing/renderComponents';
|
||||||
|
|
||||||
const {
|
const {
|
||||||
transparencyEffects,
|
transparencyEffects,
|
||||||
@@ -22,9 +23,9 @@
|
|||||||
}>();
|
}>();
|
||||||
|
|
||||||
let commandsFuse = $state<Fuse<StaticCommandItem>>();
|
let commandsFuse = $state<Fuse<StaticCommandItem>>();
|
||||||
let dynamicContentFuse = $state<Fuse<HydratedIndexItem>>();
|
let dynamicContentFuse = $state<Fuse<IndexItem>>();
|
||||||
|
|
||||||
const dynamicIdToItemMap = $state(new Map<string, HydratedIndexItem>());
|
const dynamicIdToItemMap = $state(new Map<string, IndexItem>());
|
||||||
const commandIdToItemMap = $state(new Map<string, StaticCommandItem>());
|
const commandIdToItemMap = $state(new Map<string, StaticCommandItem>());
|
||||||
|
|
||||||
let isIndexing = $state(false);
|
let isIndexing = $state(false);
|
||||||
@@ -75,6 +76,7 @@
|
|||||||
let combinedResults = $state<CombinedResult[]>([]);
|
let combinedResults = $state<CombinedResult[]>([]);
|
||||||
let isLoading = $state(false);
|
let isLoading = $state(false);
|
||||||
let calculatorResult = $state<string | null>(null);
|
let calculatorResult = $state<string | null>(null);
|
||||||
|
let resultsList = $state<HTMLUListElement>();
|
||||||
|
|
||||||
const updateCalculatorState = (hasResult: string | null) => {
|
const updateCalculatorState = (hasResult: string | null) => {
|
||||||
calculatorResult = hasResult;
|
calculatorResult = hasResult;
|
||||||
@@ -141,12 +143,20 @@
|
|||||||
searchTerm = '';
|
searchTerm = '';
|
||||||
selectedIndex = 0;
|
selectedIndex = 0;
|
||||||
combinedResults = [];
|
combinedResults = [];
|
||||||
|
tick().then(() => {
|
||||||
|
const selectedElement = resultsList?.querySelector(`li:nth-child(1)`);
|
||||||
|
selectedElement?.scrollIntoView({ block: 'nearest' });
|
||||||
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
$effect(() => {
|
$effect(() => {
|
||||||
if (combinedResults.length === 0 && calculatorResult && commandPalleteOpen) {
|
if (combinedResults.length === 0 && calculatorResult && commandPalleteOpen) {
|
||||||
selectedIndex = 0;
|
selectedIndex = 0;
|
||||||
|
tick().then(() => {
|
||||||
|
const selectedElement = resultsList?.querySelector(`li:nth-child(1)`);
|
||||||
|
selectedElement?.scrollIntoView({ block: 'nearest' });
|
||||||
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -154,16 +164,24 @@
|
|||||||
const maxIndex = (calculatorResult ? 1 : 0) + combinedResults.length - 1;
|
const maxIndex = (calculatorResult ? 1 : 0) + combinedResults.length - 1;
|
||||||
if (selectedIndex < maxIndex) {
|
if (selectedIndex < maxIndex) {
|
||||||
selectedIndex++;
|
selectedIndex++;
|
||||||
|
tick().then(() => {
|
||||||
|
const selectedElement = resultsList?.querySelector(`li:nth-child(${selectedIndex + 1})`);
|
||||||
|
selectedElement?.scrollIntoView({ block: 'nearest' });
|
||||||
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const selectPrev = () => {
|
const selectPrev = () => {
|
||||||
if (selectedIndex > 0) {
|
if (selectedIndex > 0) {
|
||||||
selectedIndex--;
|
selectedIndex--;
|
||||||
|
tick().then(() => {
|
||||||
|
const selectedElement = resultsList?.querySelector(`li:nth-child(${selectedIndex + 1})`);
|
||||||
|
selectedElement?.scrollIntoView({ block: 'nearest' });
|
||||||
|
});
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
function executeItemAction(item: StaticCommandItem | HydratedIndexItem) {
|
function executeItemAction(item: StaticCommandItem | IndexItem) {
|
||||||
if ('action' in item && typeof item.action === 'function') {
|
if ('action' in item && typeof item.action === 'function') {
|
||||||
(item as StaticCommandItem).action();
|
(item as StaticCommandItem).action();
|
||||||
} else if ('actionId' in item && item.actionId && actionMap[item.actionId]) {
|
} else if ('actionId' in item && item.actionId && actionMap[item.actionId]) {
|
||||||
@@ -240,7 +258,10 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<ul class="overflow-y-auto max-h-[24rem] text-base scroll-py-2 p-1 gap-0.5 flex flex-col">
|
<ul
|
||||||
|
bind:this={resultsList}
|
||||||
|
class="overflow-y-auto max-h-[24rem] text-base scroll-py-2 p-2 gap-0.5 flex flex-col"
|
||||||
|
>
|
||||||
<Calculator
|
<Calculator
|
||||||
searchTerm={searchTerm}
|
searchTerm={searchTerm}
|
||||||
isSelected={selectedIndex === 0}
|
isSelected={selectedIndex === 0}
|
||||||
@@ -270,14 +291,18 @@
|
|||||||
{/if}
|
{/if}
|
||||||
</button>
|
</button>
|
||||||
{:else if result.type === 'dynamic'}
|
{:else if result.type === 'dynamic'}
|
||||||
{@const dynamicItem = item as HydratedIndexItem}
|
{@const dynamicItem = item as IndexItem}
|
||||||
{#if dynamicItem.renderComponent}
|
{@const RenderComponent = renderComponentMap[dynamicItem.renderComponentId]}
|
||||||
<dynamicItem.renderComponent
|
{#if RenderComponent}
|
||||||
|
<RenderComponent
|
||||||
item={dynamicItem}
|
item={dynamicItem}
|
||||||
isSelected={isSelected}
|
isSelected={isSelected}
|
||||||
searchTerm={searchTerm}
|
searchTerm={searchTerm}
|
||||||
matches={result.matches}
|
matches={result.matches}
|
||||||
on:click={() => executeItemAction(dynamicItem)}
|
onclick={() => executeItemAction(dynamicItem)}
|
||||||
|
onkeydown={() => {}}
|
||||||
|
role="button"
|
||||||
|
tabindex="0"
|
||||||
/>
|
/>
|
||||||
{:else}
|
{:else}
|
||||||
<button
|
<button
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { highlightMatch, highlightSnippet, stripHtmlButKeepHighlights } from '../../utils/highlight';
|
||||||
|
import type { DynamicContentItem } from '../../utils/dynamicItems';
|
||||||
|
import type { FuseResultMatch } from '../../core/types';
|
||||||
|
|
||||||
|
const { item, isSelected, searchTerm, matches } = $props<{
|
||||||
|
item: DynamicContentItem;
|
||||||
|
isSelected: boolean;
|
||||||
|
searchTerm: string;
|
||||||
|
matches?: readonly FuseResultMatch[];
|
||||||
|
}>();
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<button
|
||||||
|
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
|
||||||
|
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
|
||||||
|
>
|
||||||
|
<div class="flex items-center w-full">
|
||||||
|
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\uebee'}</div>
|
||||||
|
<span class="ml-4 text-lg truncate">
|
||||||
|
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
|
||||||
|
</span>
|
||||||
|
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
|
||||||
|
{item.category}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{#if item.content}
|
||||||
|
<div class="mt-1 ml-12 text-sm text-zinc-600 dark:text-zinc-400 line-clamp-2 text-start">
|
||||||
|
{@html stripHtmlButKeepHighlights(highlightSnippet(item.content, searchTerm, matches))}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { highlightMatch, stripHtmlButKeepHighlights } from '../../utils/highlight';
|
||||||
|
import type { DynamicContentItem } from '../../utils/dynamicItems';
|
||||||
|
import type { FuseResultMatch } from '../../core/types';
|
||||||
|
|
||||||
|
const { item, isSelected, searchTerm, matches } = $props<{
|
||||||
|
item: DynamicContentItem;
|
||||||
|
isSelected: boolean;
|
||||||
|
searchTerm: string;
|
||||||
|
matches?: readonly FuseResultMatch[];
|
||||||
|
}>();
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<button
|
||||||
|
class="w-full flex flex-col px-2 py-1.5 rounded-lg select-none cursor-pointer group
|
||||||
|
{isSelected ? 'bg-zinc-900/5 dark:bg-white/10 text-zinc-900 dark:text-white' : 'hover:bg-zinc-500/5 dark:hover:bg-white/5 text-zinc-800 dark:text-zinc-200'}"
|
||||||
|
>
|
||||||
|
<div class="flex items-center w-full">
|
||||||
|
<div class="flex-none w-8 h-8 text-xl font-IconFamily flex items-center justify-center {isSelected ? 'text-zinc-900 dark:text-white' : 'text-zinc-600 dark:text-zinc-400'}">{item.metadata?.icon || '\uebe7'}</div>
|
||||||
|
<span class="ml-4 text-lg truncate">
|
||||||
|
{@html stripHtmlButKeepHighlights(highlightMatch(item.text, searchTerm, matches))}
|
||||||
|
</span>
|
||||||
|
<span class="flex-none ml-auto text-xs text-zinc-500 dark:text-zinc-400">
|
||||||
|
{item.category}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
@@ -56,3 +56,16 @@
|
|||||||
color: #aaa;
|
color: #aaa;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
.highlight {
|
||||||
|
background-color: rgba(255, 213, 0, 0.3);
|
||||||
|
font-weight: 500;
|
||||||
|
border-radius: 2px;
|
||||||
|
padding: 0 1px;
|
||||||
|
margin: 0 -1px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dark .highlight {
|
||||||
|
background-color: rgba(255, 230, 100, 0.4);
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import type { StaticCommandItem } from "./commands";
|
import type { StaticCommandItem } from "./commands";
|
||||||
import type { HydratedIndexItem } from "../indexing/types";
|
import type { IndexItem } from "../indexing/types";
|
||||||
|
|
||||||
export interface MatchIndices {
|
export interface MatchIndices {
|
||||||
readonly 0: number;
|
readonly 0: number;
|
||||||
@@ -16,7 +16,7 @@ export interface CombinedResult {
|
|||||||
id: string;
|
id: string;
|
||||||
type: "command" | "dynamic";
|
type: "command" | "dynamic";
|
||||||
score: number;
|
score: number;
|
||||||
item: StaticCommandItem | HydratedIndexItem;
|
item: StaticCommandItem | IndexItem;
|
||||||
matches?: readonly FuseResultMatch[];
|
matches?: readonly FuseResultMatch[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import { clear, getAll, get, put, remove } from "./db";
|
import { clear, getAll, get, put, remove } from "./db";
|
||||||
import { jobs } from "./jobs";
|
import { jobs } from "./jobs";
|
||||||
import { renderComponentMap } from "./renderComponents";
|
import { renderComponentMap } from "./renderComponents";
|
||||||
import type { HydratedIndexItem, IndexItem, Job, JobContext } from "./types";
|
import type { IndexItem, Job, JobContext } from "./types";
|
||||||
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
|
import { VectorWorkerManager } from "./worker/vectorWorkerManager";
|
||||||
|
|
||||||
const META_STORE = "meta";
|
const META_STORE = "meta";
|
||||||
@@ -83,21 +83,14 @@ function dispatchProgress(
|
|||||||
window.dispatchEvent(event);
|
window.dispatchEvent(event);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
|
export async function loadAllStoredItems(): Promise<IndexItem[]> {
|
||||||
const all: HydratedIndexItem[] = [];
|
const all: IndexItem[] = [];
|
||||||
const jobIds = Object.keys(jobs);
|
const jobIds = Object.keys(jobs);
|
||||||
|
|
||||||
for (const jobId of jobIds) {
|
for (const jobId of jobIds) {
|
||||||
try {
|
try {
|
||||||
const items = (await getAll(jobId)) as IndexItem[];
|
const items = (await getAll(jobId)) as IndexItem[];
|
||||||
const job = jobs[jobId];
|
const job = jobs[jobId];
|
||||||
const renderComponent = renderComponentMap[job.renderComponentId];
|
|
||||||
|
|
||||||
if (!renderComponent) {
|
|
||||||
console.warn(
|
|
||||||
`Render component not found for job ${jobId} (ID: ${job.renderComponentId})`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
if (
|
if (
|
||||||
@@ -108,10 +101,7 @@ export async function loadAllStoredItems(): Promise<HydratedIndexItem[]> {
|
|||||||
item.actionId &&
|
item.actionId &&
|
||||||
job.renderComponentId
|
job.renderComponentId
|
||||||
) {
|
) {
|
||||||
all.push({
|
all.push(item);
|
||||||
...item,
|
|
||||||
renderComponent: renderComponent || undefined,
|
|
||||||
});
|
|
||||||
} else {
|
} else {
|
||||||
console.warn(`Skipping invalid item from job ${jobId}:`, item);
|
console.warn(`Skipping invalid item from job ${jobId}:`, item);
|
||||||
}
|
}
|
||||||
@@ -144,7 +134,7 @@ export async function runIndexing(): Promise<void> {
|
|||||||
const totalSteps = jobIds.length + 1;
|
const totalSteps = jobIds.length + 1;
|
||||||
dispatchProgress(completedJobs, totalSteps, true, "Starting jobs");
|
dispatchProgress(completedJobs, totalSteps, true, "Starting jobs");
|
||||||
|
|
||||||
const allItemsFromJobs: HydratedIndexItem[] = [];
|
const allItemsFromJobs: IndexItem[] = [];
|
||||||
|
|
||||||
// --- Step 1: Run Fetching/Storing Jobs (Main Thread) ---
|
// --- Step 1: Run Fetching/Storing Jobs (Main Thread) ---
|
||||||
for (const jobId of jobIds) {
|
for (const jobId of jobIds) {
|
||||||
@@ -225,35 +215,7 @@ export async function runIndexing(): Promise<void> {
|
|||||||
await setStoredItems(merged);
|
await setStoredItems(merged);
|
||||||
await updateLastRunMeta(jobId);
|
await updateLastRunMeta(jobId);
|
||||||
|
|
||||||
// Hydrate items for vector processing
|
allItemsFromJobs.push(...newItemsRaw);
|
||||||
const renderComponent = renderComponentMap[job.renderComponentId];
|
|
||||||
if (!renderComponent) {
|
|
||||||
console.warn(
|
|
||||||
`Render component not found for job ${jobId} (ID: ${job.renderComponentId}) during hydration`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const hydratedItems = merged
|
|
||||||
.filter(
|
|
||||||
(item) =>
|
|
||||||
item &&
|
|
||||||
item.id &&
|
|
||||||
item.text &&
|
|
||||||
item.category &&
|
|
||||||
item.actionId &&
|
|
||||||
job.renderComponentId,
|
|
||||||
) // Filter invalid before hydrating
|
|
||||||
.map((item) => ({
|
|
||||||
...item,
|
|
||||||
renderComponent: renderComponent || undefined, // Assign undefined if not found
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (hydratedItems.length !== merged.length) {
|
|
||||||
console.warn(
|
|
||||||
`[Indexer Job ${jobId}] Filtered out ${merged.length - hydratedItems.length} invalid items during hydration.`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
allItemsFromJobs.push(...hydratedItems);
|
|
||||||
|
|
||||||
console.debug(
|
console.debug(
|
||||||
`%c[Indexer] ${job.label}: ${newItemsRaw.length} new items from run, ${merged.length} total stored in '${jobId}' store (non-vector).`,
|
`%c[Indexer] ${job.label}: ${newItemsRaw.length} new items from run, ${merged.length} total stored in '${jobId}' store (non-vector).`,
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
import type { Job } from "./types";
|
import type { Job } from "./types";
|
||||||
import { messagesJob } from "./jobs/messages";
|
import { messagesJob } from "./jobs/messages";
|
||||||
import { assessmentsJob } from "./jobs/assessments";
|
import { assessmentsJob } from "./jobs/assessments";
|
||||||
|
import { forumsJob } from "./jobs/forums";
|
||||||
|
|
||||||
export const jobs: Record<string, Job> = {
|
export const jobs: Record<string, Job> = {
|
||||||
messages: messagesJob,
|
messages: messagesJob,
|
||||||
assessments: assessmentsJob,
|
assessments: assessmentsJob,
|
||||||
|
forums: forumsJob,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,69 @@
|
|||||||
|
import type { Job, IndexItem } from "../types";
|
||||||
|
|
||||||
|
const fetchForums = async () => {
|
||||||
|
const res = await fetch(`${location.origin}/seqta/student/load/forums`, {
|
||||||
|
method: "POST",
|
||||||
|
credentials: "include",
|
||||||
|
headers: { "Content-Type": "application/json; charset=utf-8" },
|
||||||
|
body: JSON.stringify({ mode: "list" }),
|
||||||
|
});
|
||||||
|
|
||||||
|
return res.json() as Promise<{
|
||||||
|
payload: { forums: any[] };
|
||||||
|
status: string;
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const forumsJob: Job = {
|
||||||
|
id: "forums",
|
||||||
|
label: "Forums",
|
||||||
|
renderComponentId: "forum",
|
||||||
|
//frequency: { type: "expiry", afterMs: 30 * 24 * 60 * 60 * 1000 }, // 30 days
|
||||||
|
frequency: { type: "interval", ms: 1000 }, // 1s
|
||||||
|
|
||||||
|
run: async (ctx) => {
|
||||||
|
const existingIds = new Set(
|
||||||
|
(await ctx.getStoredItems("forums")).map((i) => i.id),
|
||||||
|
);
|
||||||
|
|
||||||
|
let list;
|
||||||
|
try {
|
||||||
|
list = await fetchForums();
|
||||||
|
} catch (e) {
|
||||||
|
console.error("[Forums job] list fetch failed:", e);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (list.status !== "200") return [];
|
||||||
|
|
||||||
|
const items: IndexItem[] = [];
|
||||||
|
|
||||||
|
for (const forum of list.payload.forums) {
|
||||||
|
const id = forum.id.toString();
|
||||||
|
if (existingIds.has(id)) continue;
|
||||||
|
|
||||||
|
items.push({
|
||||||
|
id,
|
||||||
|
text: forum.title,
|
||||||
|
category: "forums",
|
||||||
|
content: `${forum.title}`,
|
||||||
|
dateAdded: Date.now(),
|
||||||
|
metadata: {
|
||||||
|
forumId: forum.id,
|
||||||
|
owner: forum.owner,
|
||||||
|
title: forum.title,
|
||||||
|
},
|
||||||
|
actionId: "forum",
|
||||||
|
renderComponentId: "forum",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return items;
|
||||||
|
},
|
||||||
|
|
||||||
|
/** Keep only forums from the last 2 years. */
|
||||||
|
purge: (items) => {
|
||||||
|
const twoYearsAgo = Date.now() - 2 * 365 * 24 * 60 * 60 * 1000;
|
||||||
|
return items.filter((i) => i.dateAdded >= twoYearsAgo);
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -1,10 +1,15 @@
|
|||||||
import type { SvelteComponent } from "svelte";
|
import type { SvelteComponent } from "svelte";
|
||||||
import AssessmentComponent from "../components/AssessmentItem.svelte";
|
import AssessmentItem from "../components/items/AssessmentItem.svelte";
|
||||||
|
import ForumItem from "../components/items/ForumItem.svelte";
|
||||||
// import other components as needed
|
// import other components as needed
|
||||||
|
|
||||||
export const renderComponentMap: Record<string, typeof SvelteComponent> = {
|
export const renderComponentMap: Record<string, typeof SvelteComponent> = {
|
||||||
assessment: AssessmentComponent as unknown as typeof SvelteComponent,
|
assessment: AssessmentItem as unknown as typeof SvelteComponent,
|
||||||
message: AssessmentComponent as unknown as typeof SvelteComponent,
|
message: AssessmentItem as unknown as typeof SvelteComponent,
|
||||||
|
forum: ForumItem as unknown as typeof SvelteComponent,
|
||||||
// subject: SubjectComponent,
|
// subject: SubjectComponent,
|
||||||
// etc...
|
// etc...
|
||||||
};
|
};
|
||||||
|
|
||||||
|
void AssessmentItem;
|
||||||
|
void ForumItem;
|
||||||
@@ -11,10 +11,6 @@ export interface IndexItem {
|
|||||||
renderComponentId: string;
|
renderComponentId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface HydratedIndexItem extends IndexItem {
|
|
||||||
renderComponent: typeof SvelteComponent;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type Frequency =
|
export type Frequency =
|
||||||
| "pageLoad"
|
| "pageLoad"
|
||||||
| { type: "interval"; ms: number }
|
| { type: "interval"; ms: number }
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
|
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
|
||||||
import type { HydratedIndexItem } from "../types";
|
import type { IndexItem } from "../types";
|
||||||
|
|
||||||
let vectorIndex: EmbeddingIndex | null = null;
|
let vectorIndex: EmbeddingIndex | null = null;
|
||||||
let isInitialized = false;
|
let isInitialized = false;
|
||||||
@@ -35,8 +35,8 @@ async function initWorker() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function vectorizeItem(
|
async function vectorizeItem(
|
||||||
item: HydratedIndexItem,
|
item: IndexItem,
|
||||||
): Promise<(HydratedIndexItem & { embedding: number[] }) | null> {
|
): Promise<(IndexItem & { embedding: number[] }) | null> {
|
||||||
// Simplified for brevity - assumes embedding function doesn't need cancellation signal
|
// Simplified for brevity - assumes embedding function doesn't need cancellation signal
|
||||||
try {
|
try {
|
||||||
const textToEmbed = [
|
const textToEmbed = [
|
||||||
@@ -57,7 +57,7 @@ async function vectorizeItem(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function processItems(items: HydratedIndexItem[], signal: AbortSignal) {
|
async function processItems(items: IndexItem[], signal: AbortSignal) {
|
||||||
console.debug("Worker received process request.");
|
console.debug("Worker received process request.");
|
||||||
if (!vectorIndex) {
|
if (!vectorIndex) {
|
||||||
console.warn(
|
console.warn(
|
||||||
@@ -140,7 +140,7 @@ async function processItems(items: HydratedIndexItem[], signal: AbortSignal) {
|
|||||||
const vectorizationResults = await Promise.all(batch.map(vectorizeItem));
|
const vectorizationResults = await Promise.all(batch.map(vectorizeItem));
|
||||||
const successfullyVectorized = vectorizationResults.filter(
|
const successfullyVectorized = vectorizationResults.filter(
|
||||||
(result) => result !== null,
|
(result) => result !== null,
|
||||||
) as (HydratedIndexItem & { embedding: number[] })[];
|
) as (IndexItem & { embedding: number[] })[];
|
||||||
|
|
||||||
if (signal.aborted) {
|
if (signal.aborted) {
|
||||||
console.debug("Processing cancelled after vectorization batch.");
|
console.debug("Processing cancelled after vectorization batch.");
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { refreshVectorCache } from "../../search/vector/vectorSearch";
|
import { refreshVectorCache } from "../../search/vector/vectorSearch";
|
||||||
import type { HydratedIndexItem } from "../types";
|
import type { IndexItem } from "../types";
|
||||||
import vectorWorker from "./vectorWorker.ts?inlineWorker";
|
import vectorWorker from "./vectorWorker.ts?inlineWorker";
|
||||||
import type { SearchResult } from "embeddia";
|
import type { SearchResult } from "embeddia";
|
||||||
|
|
||||||
@@ -156,7 +156,7 @@ export class VectorWorkerManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async processItems(
|
async processItems(
|
||||||
items: HydratedIndexItem[],
|
items: IndexItem[],
|
||||||
onProgress?: ProgressCallback,
|
onProgress?: ProgressCallback,
|
||||||
) {
|
) {
|
||||||
await this.ensureReady(); // Wait for worker to be ready
|
await this.ensureReady(); // Wait for worker to be ready
|
||||||
@@ -168,11 +168,10 @@ export class VectorWorkerManager {
|
|||||||
|
|
||||||
console.debug(`Sending ${items.length} items to worker for processing.`);
|
console.debug(`Sending ${items.length} items to worker for processing.`);
|
||||||
|
|
||||||
const serialisableItems = items.map(({ renderComponent, ...rest }) => rest);
|
|
||||||
|
|
||||||
this.worker!.postMessage({
|
this.worker!.postMessage({
|
||||||
type: "process",
|
type: "process",
|
||||||
data: { items: serialisableItems },
|
data: { items: items },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import Fuse, { type FuseResult } from "fuse.js";
|
|||||||
import { getStaticCommands, type StaticCommandItem } from "../core/commands";
|
import { getStaticCommands, type StaticCommandItem } from "../core/commands";
|
||||||
import { getDynamicItems } from "../utils/dynamicItems";
|
import { getDynamicItems } from "../utils/dynamicItems";
|
||||||
import type { CombinedResult } from "../core/types";
|
import type { CombinedResult } from "../core/types";
|
||||||
import type { HydratedIndexItem } from "../indexing/types";
|
import type { IndexItem } from "../indexing/types";
|
||||||
import { searchVectors } from "./vector/vectorSearch";
|
import { searchVectors } from "./vector/vectorSearch";
|
||||||
import type { VectorSearchResult } from "./vector/vectorTypes";
|
import type { VectorSearchResult } from "./vector/vectorTypes";
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ export function createSearchIndexes() {
|
|||||||
dynamicContentFuse: new Fuse(
|
dynamicContentFuse: new Fuse(
|
||||||
dynamicItems,
|
dynamicItems,
|
||||||
dynamicOptions,
|
dynamicOptions,
|
||||||
) as Fuse<HydratedIndexItem>,
|
) as Fuse<IndexItem>,
|
||||||
commands,
|
commands,
|
||||||
dynamicItems,
|
dynamicItems,
|
||||||
};
|
};
|
||||||
@@ -85,9 +85,9 @@ export function searchCommands(
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function searchDynamicItems(
|
export function searchDynamicItems(
|
||||||
dynamicContentFuse: Fuse<HydratedIndexItem>,
|
dynamicContentFuse: Fuse<IndexItem>,
|
||||||
query: string,
|
query: string,
|
||||||
dynamicIdToItemMap: Map<string, HydratedIndexItem>,
|
dynamicIdToItemMap: Map<string, IndexItem>,
|
||||||
limit = 10,
|
limit = 10,
|
||||||
sortByRecent: boolean = true, // Added option to control sorting
|
sortByRecent: boolean = true, // Added option to control sorting
|
||||||
): CombinedResult[] {
|
): CombinedResult[] {
|
||||||
@@ -109,7 +109,7 @@ export function searchDynamicItems(
|
|||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const searchResults = dynamicContentFuse.search(query, { limit });
|
const searchResults = dynamicContentFuse.search(query, { limit });
|
||||||
|
|
||||||
return searchResults.map((result: FuseResult<HydratedIndexItem>) => {
|
return searchResults.map((result: FuseResult<IndexItem>) => {
|
||||||
const item = result.item;
|
const item = result.item;
|
||||||
const fuseScore = 10 * (1 - (result.score || 0.5));
|
const fuseScore = 10 * (1 - (result.score || 0.5));
|
||||||
const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24);
|
const ageInDays = (now - item.dateAdded) / (1000 * 60 * 60 * 24);
|
||||||
@@ -129,9 +129,9 @@ export function searchDynamicItems(
|
|||||||
export async function performSearch(
|
export async function performSearch(
|
||||||
query: string,
|
query: string,
|
||||||
commandsFuse: Fuse<StaticCommandItem>,
|
commandsFuse: Fuse<StaticCommandItem>,
|
||||||
dynamicContentFuse: Fuse<HydratedIndexItem>,
|
dynamicContentFuse: Fuse<IndexItem>,
|
||||||
commandIdToItemMap: Map<string, StaticCommandItem>,
|
commandIdToItemMap: Map<string, StaticCommandItem>,
|
||||||
dynamicIdToItemMap: Map<string, HydratedIndexItem>,
|
dynamicIdToItemMap: Map<string, IndexItem>,
|
||||||
showRecentFirst: boolean,
|
showRecentFirst: boolean,
|
||||||
): Promise<CombinedResult[]> {
|
): Promise<CombinedResult[]> {
|
||||||
// Get all results first
|
// Get all results first
|
||||||
@@ -182,6 +182,7 @@ export async function performSearch(
|
|||||||
// Now add any vector results we haven't seen yet
|
// Now add any vector results we haven't seen yet
|
||||||
vectorResults.forEach((v) => {
|
vectorResults.forEach((v) => {
|
||||||
const id = v.object.id;
|
const id = v.object.id;
|
||||||
|
|
||||||
if (!seenIds.has(id)) {
|
if (!seenIds.has(id)) {
|
||||||
// This is a semantic match that Fuse missed - add it with the vector similarity as score
|
// This is a semantic match that Fuse missed - add it with the vector similarity as score
|
||||||
resultMap.set(id, {
|
resultMap.set(id, {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
|
import { EmbeddingIndex, getEmbedding, initializeModel } from "embeddia";
|
||||||
import type { HydratedIndexItem } from "../../indexing/types";
|
import type { IndexItem } from "../../indexing/types";
|
||||||
import type { SearchResult } from "embeddia";
|
import type { SearchResult } from "embeddia";
|
||||||
|
|
||||||
let vectorIndex: EmbeddingIndex | null = null;
|
let vectorIndex: EmbeddingIndex | null = null;
|
||||||
@@ -15,7 +15,7 @@ export async function initVectorSearch() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface VectorSearchResult extends SearchResult {
|
export interface VectorSearchResult extends SearchResult {
|
||||||
object: HydratedIndexItem & { embedding: number[] };
|
object: IndexItem & { embedding: number[] };
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function searchVectors(
|
export async function searchVectors(
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import type { SearchResult } from "embeddia";
|
import type { SearchResult } from "embeddia";
|
||||||
import type { HydratedIndexItem } from "../../indexing/types";
|
import type { IndexItem } from "../../indexing/types";
|
||||||
|
|
||||||
export interface VectorSearchResult extends SearchResult {
|
export interface VectorSearchResult extends SearchResult {
|
||||||
object: HydratedIndexItem & { embedding: number[] };
|
object: IndexItem & { embedding: number[] };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import type { SvelteComponent } from "svelte";
|
import type { SvelteComponent } from "svelte";
|
||||||
import type { HydratedIndexItem } from "./indexing/types";
|
import type { IndexItem } from "./indexing/types";
|
||||||
|
|
||||||
export interface DynamicContentItem {
|
export interface DynamicContentItem {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -13,18 +13,18 @@ export interface DynamicContentItem {
|
|||||||
renderComponent?: typeof SvelteComponent;
|
renderComponent?: typeof SvelteComponent;
|
||||||
}
|
}
|
||||||
|
|
||||||
let dynamicItems: HydratedIndexItem[] = [];
|
let dynamicItems: IndexItem[] = [];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads a new set of dynamic items.
|
* Loads a new set of dynamic items.
|
||||||
*/
|
*/
|
||||||
export function loadDynamicItems(items: HydratedIndexItem[]) {
|
export function loadDynamicItems(items: IndexItem[]) {
|
||||||
dynamicItems = items;
|
dynamicItems = items;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns all currently loaded dynamic items.
|
* Returns all currently loaded dynamic items.
|
||||||
*/
|
*/
|
||||||
export function getDynamicItems(): HydratedIndexItem[] {
|
export function getDynamicItems(): IndexItem[] {
|
||||||
return dynamicItems;
|
return dynamicItems;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user