Optimize speech search and admin payloads

src/lib/admin-speeches.ts

@@ -0,0 +1,52 @@
+import type { ArchiveEntry } from "./archive";
+
+export type SpeechAdminEntry = {
+  id: string;
+  path: string;
+  title: string;
+  publishedDate: string;
+  speechYear: string;
+  speaker: "rev" | "mrs" | "other";
+  source: string;
+  categories: string[];
+  tags: string[];
+  bodyUrl: string;
+};
+
+function encodePath(value: string): string {
+  return value.split("/").map(encodeURIComponent).join("/");
+}
+
+export function speechAdminEntryFromArchiveEntry(entry: ArchiveEntry, fallbackDate: string): SpeechAdminEntry {
+  const categories = entry.data.categories ?? [];
+  const tags = entry.data.tags ?? [];
+  const publishedDate = entry.data.published?.slice(0, 10) ?? fallbackDate;
+  const speechYear = entry.data.speechYear ?? entry.id.match(/\b(1\d{3}|20\d{2})\b/)?.[1] ?? new Date().getFullYear().toString();
+  const categoryText = categories.join(" ").toLowerCase();
+  const collection = entry.data.speechCollection ?? "";
+  const speaker = categoryText.includes("hak ja han") || collection === "mrs-hak-ja-han-moon"
+    ? "mrs"
+    : categoryText.includes("rev sun myung moon") || /^\d{4}$/.test(collection) || /^\d{4}$/.test(speechYear)
+      ? "rev"
+      : "other";
+
+  return {
+    id: entry.id,
+    path: `content/${entry.id}.md`,
+    title: entry.data.title ?? entry.id,
+    publishedDate,
+    speechYear,
+    speaker,
+    source: entry.data.source ?? "",
+    categories,
+    tags,
+    bodyUrl: `/admin/content/${encodePath(entry.id)}.json`,
+  };
+}
+
+export function speechAdminEntryWithBody(entry: ArchiveEntry, fallbackDate: string) {
+  return {
+    ...speechAdminEntryFromArchiveEntry(entry, fallbackDate),
+    body: entry.body ?? "",
+  };
+}

src/lib/speeches.ts

@@ -32,6 +32,14 @@ export type SpeechArchiveItem = {
   searchText: string;
 };
 
+export type SpeechSearchIndexItem = Omit<SpeechArchiveItem, "searchText">;
+
+export type CompactSpeechSearchIndex = {
+  version: 2;
+  items: SpeechSearchIndexItem[];
+  index: Record<string, string>;
+};
+
 const entityMap: Record<string, string> = {
   amp: "&",
   apos: "'",
@@ -215,6 +223,39 @@ export function getSpeechArchive(): SpeechArchiveItem[] {
     });
 }
 
+function packDocumentIds(documentIds: number[]): string {
+  return documentIds.map((documentId) => documentId.toString(36)).join(",");
+}
+
+export function getCompactSpeechSearchIndex(): CompactSpeechSearchIndex {
+  const items = getSpeechArchive();
+  const index = new Map<string, number[]>();
+
+  items.forEach((item, documentId) => {
+    const terms = new Set(item.searchText.match(/[a-z0-9]+/g) ?? []);
+
+    terms.forEach((term) => {
+      if (term.length < 2) {
+        return;
+      }
+
+      const documentIds = index.get(term) ?? [];
+      documentIds.push(documentId);
+      index.set(term, documentIds);
+    });
+  });
+
+  return {
+    version: 2,
+    items: items.map(({ searchText, ...item }) => item),
+    index: Object.fromEntries(
+      Array.from(index)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([term, documentIds]) => [term, packDocumentIds(documentIds)]),
+    ),
+  };
+}
+
 export function uniqueSorted(values: string[], direction: "asc" | "desc" = "asc"): string[] {
   const unique = Array.from(new Set(values.filter(Boolean)));
   return unique.sort((a, b) => {

src/pages/admin/content/[...id].json.ts

@@ -0,0 +1,23 @@
+import { getArchiveEntries } from "../../../lib/archive";
+import { speechAdminEntryWithBody } from "../../../lib/admin-speeches";
+
+export async function getStaticPaths() {
+  const today = new Date().toISOString().slice(0, 10);
+  const entries = await getArchiveEntries("speech");
+
+  return entries.map((entry) => ({
+    params: { id: entry.id },
+    props: {
+      speech: speechAdminEntryWithBody(entry, today),
+    },
+  }));
+}
+
+export function GET({ props }: { props: { speech: ReturnType<typeof speechAdminEntryWithBody> } }) {
+  return new Response(JSON.stringify(props.speech), {
+    headers: {
+      "content-type": "application/json; charset=utf-8",
+      "cache-control": "public, max-age=300",
+    },
+  });
+}

src/pages/admin/index.astro

@@ -1,5 +1,6 @@
 ---
 import { getArchiveEntries } from "../../lib/archive";
+import { speechAdminEntryFromArchiveEntry } from "../../lib/admin-speeches";
 import { calendarEvents, recurringCalendarEvents } from "../../data/events";
 
 const entries = await getArchiveEntries();
@@ -17,32 +18,7 @@ const latestEntryDate = latestEntry?.data.published
   : "No date";
 const totalEvents = calendarEvents.length + recurringCalendarEvents.length;
 const configuredServices = [contactFormEndpoint, plausibleDomain].filter(Boolean).length;
-const speechAdminEntries = speechEntries.map((entry) => {
+const speechAdminEntries = speechEntries.map((entry) => speechAdminEntryFromArchiveEntry(entry, today));
-  const categories = entry.data.categories ?? [];
-  const tags = entry.data.tags ?? [];
-  const publishedDate = entry.data.published?.slice(0, 10) ?? today;
-  const speechYear = entry.data.speechYear ?? entry.id.match(/\b(1\d{3}|20\d{2})\b/)?.[1] ?? new Date().getFullYear().toString();
-  const categoryText = categories.join(" ").toLowerCase();
-  const collection = entry.data.speechCollection ?? "";
-  const speaker = categoryText.includes("hak ja han") || collection === "mrs-hak-ja-han-moon"
-    ? "mrs"
-    : categoryText.includes("rev sun myung moon") || /^\d{4}$/.test(collection) || /^\d{4}$/.test(speechYear)
-      ? "rev"
-      : "other";
-
-  return {
-    id: entry.id,
-    path: `content/${entry.id}.md`,
-    title: entry.data.title ?? entry.id,
-    publishedDate,
-    speechYear,
-    speaker,
-    source: entry.data.source ?? "",
-    categories,
-    tags,
-    body: entry.body ?? "",
-  };
-});
 const adminConfig = {
   analyticsDashboardUrl,
   contactEndpointConfigured: Boolean(contactFormEndpoint),
@@ -1265,11 +1241,29 @@ const adminConfig = {
           }
         }
 
-        function loadSpeechEditor(speech) {
+        async function loadSpeechEditor(speech) {
           if (!speechForm) {
             return;
           }
 
+          const message = document.querySelector("[data-speech-message]");
+          setMessage(message, "Loading speech body...");
+
+          if (!speech.body && speech.bodyUrl) {
+            try {
+              const response = await fetch(speech.bodyUrl);
+
+              if (!response.ok) {
+                throw new Error(`Speech body request failed: ${response.status}`);
+              }
+
+              speech = { ...speech, ...(await response.json()) };
+            } catch {
+              setMessage(message, "Could not load that speech body. Try refreshing the admin page.", "error");
+              return;
+            }
+          }
+
           setFormValue(speechForm, "title", speech.title);
           setFormValue(speechForm, "publishedDate", speech.publishedDate);
           setFormValue(speechForm, "speechYear", speech.speechYear);

src/pages/speeches/index.astro

@@ -92,6 +92,9 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker));
       }
 
       let speeches = [];
+      let searchIndex = {};
+      let searchTerms = [];
+      const termCache = new Map();
       let renderTimer = 0;
 
       function normalize(value) {
@@ -129,19 +132,82 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker));
         speakerSelect.value = params.get("speaker") || "";
       }
 
-      function matchesSpeech(speech, filters) {
+      function decodeDocumentIds(value) {
+        return new Set(String(value || "").split(",").filter(Boolean).map((documentId) => Number.parseInt(documentId, 36)));
+      }
+
+      function unionSets(sets) {
+        const union = new Set();
+        sets.forEach((set) => {
+          set.forEach((value) => union.add(value));
+        });
+        return union;
+      }
+
+      function intersectSets(left, right) {
+        const smaller = left.size <= right.size ? left : right;
+        const larger = left.size <= right.size ? right : left;
+        const intersection = new Set();
+
+        smaller.forEach((value) => {
+          if (larger.has(value)) {
+            intersection.add(value);
+          }
+        });
+
+        return intersection;
+      }
+
+      function documentsForTerm(term) {
+        if (termCache.has(term)) {
+          return termCache.get(term);
+        }
+
+        const matches = [];
+
+        if (searchIndex[term]) {
+          matches.push(decodeDocumentIds(searchIndex[term]));
+        }
+
+        if (term.length >= 3) {
+          searchTerms.forEach((indexedTerm) => {
+            if (indexedTerm !== term && indexedTerm.startsWith(term)) {
+              matches.push(decodeDocumentIds(searchIndex[indexedTerm]));
+            }
+          });
+        }
+
+        const documentIds = matches.length ? unionSets(matches) : new Set();
+        termCache.set(term, documentIds);
+        return documentIds;
+      }
+
+      function searchCandidates(query) {
+        const terms = normalize(query).match(/[a-z0-9]+/g)?.filter((term) => term.length >= 2) ?? [];
+
+        if (!terms.length) {
+          return null;
+        }
+
+        return terms.reduce((candidateIds, term) => {
+          const termIds = documentsForTerm(term);
+
+          if (!candidateIds) {
+            return termIds;
+          }
+
+          return intersectSets(candidateIds, termIds);
+        }, null);
+      }
+
+      function matchesSpeechMetadata(speech, filters) {
         if (filters.year && speech.year !== filters.year) {
           return false;
         }
         if (filters.speaker && speech.speaker !== filters.speaker) {
           return false;
         }
-        if (!filters.q) {
+        return true;
-          return true;
-        }
-
-        const terms = normalize(filters.q).split(/\s+/).filter(Boolean);
-        return terms.every((term) => speech.searchText.includes(term));
       }
 
       function appendMeta(parent, values) {
@@ -176,7 +242,13 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker));
 
       function render() {
         const filters = selectedFilters();
-        const matched = speeches.filter((speech) => matchesSpeech(speech, filters));
+        const candidateIds = searchCandidates(filters.q);
+        const matched = speeches.filter((speech, index) => {
+          if (candidateIds && !candidateIds.has(index)) {
+            return false;
+          }
+          return matchesSpeechMetadata(speech, filters);
+        });
         const fragment = document.createDocumentFragment();
 
         if (matched.length) {
@@ -208,7 +280,10 @@ const speakers = uniqueSorted(speeches.map((speech) => speech.speaker));
           return response.json();
         })
         .then((data) => {
-          speeches = data;
+          speeches = Array.isArray(data) ? data : data.items ?? [];
+          searchIndex = Array.isArray(data) ? {} : data.index ?? {};
+          searchTerms = Object.keys(searchIndex);
+          termCache.clear();
           render();
           form.addEventListener("input", queueRender);
           form.addEventListener("change", render);

src/pages/speeches/search-index.json.ts

@@ -1,7 +1,7 @@
-import { getSpeechArchive } from "../../lib/speeches";
+import { getCompactSpeechSearchIndex } from "../../lib/speeches";
 
 export function GET() {
-  return new Response(JSON.stringify(getSpeechArchive()), {
+  return new Response(JSON.stringify(getCompactSpeechSearchIndex()), {
     headers: {
       "content-type": "application/json; charset=utf-8",
       "cache-control": "public, max-age=300",