mirror of
https://github.com/Screenly/Anthias.git
synced 2026-07-30 09:15:51 -04:00
feat(website): generate llms.txt and llms-full.txt (#3179)
* feat(website): generate llms.txt and llms-full.txt Add a post-build generator that produces llms.txt (a curated index) and llms-full.txt (the full text of every page) for the marketing site, following the llmstxt.org convention and served from the site root. The generator extracts clean Markdown from the rendered public/ HTML rather than the Markdown/data sources, so it captures the marketing copy that lives in Hugo layouts (home, features, get-started) and never drifts from what visitors actually see. Wired into `bun run build` and the deploy-website workflow. Also fixes four pre-existing `sort $map` calls in the /api templates: in Hugo, sorting a map collapses it to a slice, so the keys were replaced by numeric indices — API paths, HTTP status codes, request body content-types, and schema field names all rendered as 0, 1, 2… on the live page. Hugo already ranges maps in sorted-key order, so dropping `sort` fixes it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(website): escape backslashes before pipes in llms table cells CodeQL js/incomplete-sanitization: the Markdown table-cell escaping replaced "|" but not "\", so a literal backslash in a cell could combine with the pipe escape. Escape backslashes first. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(website): collapse mixed-content whitespace and harden llms entrypoint Address Copilot review: - serializeMixed() now collapses text-node whitespace the same way inline() does, so HTML source indentation/newlines can't leak into list items and other mixed-content blocks. - main() gets a .catch that sets a non-zero exit code, so build/CI failures (missing public/, read/write/parse errors) are deterministic. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Viktor Petersson
8
.github/workflows/deploy-website.yaml
vendored
8
.github/workflows/deploy-website.yaml
vendored
@@ -123,6 +123,14 @@ jobs:
|
||||
- name: Build website
|
||||
run: hugo --source website --minify
|
||||
|
||||
# Generate llms.txt / llms-full.txt from the rendered public/
|
||||
# tree (llmstxt.org convention). Reading the built HTML keeps
|
||||
# the files in sync with what visitors see, including marketing
|
||||
# copy that lives in Hugo layouts rather than content/*.md.
|
||||
- name: Generate llms.txt
|
||||
run: bun run llms:generate
|
||||
working-directory: website
|
||||
|
||||
# Fetch rpi-imager.json from the latest GitHub release. The
|
||||
# file is built by build-balena-disk-image.yaml at release time
|
||||
# and Hugo serves it at /rpi-imager.json.
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
"@fortawesome/fontawesome-free": "^7.2.0",
|
||||
"@tailwindcss/cli": "^4.2.4",
|
||||
"@types/bun": "^1.3.14",
|
||||
"node-html-parser": "^9.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -89,12 +90,28 @@
|
||||
|
||||
"@types/node": ["@types/node@25.7.0", "", { "dependencies": { "undici-types": "~7.21.0" } }, "sha512-z+pdZyxE+RTQE9AcboAZCb4otwcrvgHD+GlBpPgn0emDVt0ohrTMhAwlr2Wd9nZ+nihhYFxO2pThz3C5qSu2Eg=="],
|
||||
|
||||
"boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="],
|
||||
|
||||
"css-select": ["css-select@5.2.2", "", { "dependencies": { "boolbase": "^1.0.0", "css-what": "^6.1.0", "domhandler": "^5.0.2", "domutils": "^3.0.1", "nth-check": "^2.0.1" } }, "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw=="],
|
||||
|
||||
"css-what": ["css-what@6.2.2", "", {}, "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA=="],
|
||||
|
||||
"detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
|
||||
|
||||
"dom-serializer": ["dom-serializer@2.0.0", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.2", "entities": "^4.2.0" } }, "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg=="],
|
||||
|
||||
"domelementtype": ["domelementtype@2.3.0", "", {}, "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw=="],
|
||||
|
||||
"domhandler": ["domhandler@5.0.3", "", { "dependencies": { "domelementtype": "^2.3.0" } }, "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w=="],
|
||||
|
||||
"domutils": ["domutils@3.2.2", "", { "dependencies": { "dom-serializer": "^2.0.0", "domelementtype": "^2.3.0", "domhandler": "^5.0.3" } }, "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw=="],
|
||||
|
||||
"enhanced-resolve": ["enhanced-resolve@5.21.0", "", { "dependencies": { "graceful-fs": "^4.2.4", "tapable": "^2.3.3" } }, "sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA=="],
|
||||
|
||||
"entities": ["entities@8.0.0", "", {}, "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA=="],
|
||||
|
||||
"graceful-fs": ["graceful-fs@4.2.11", "", {}, "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ=="],
|
||||
|
||||
"is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="],
|
||||
@@ -133,6 +150,10 @@
|
||||
|
||||
"node-addon-api": ["node-addon-api@7.1.1", "", {}, "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ=="],
|
||||
|
||||
"node-html-parser": ["node-html-parser@9.0.0", "", { "dependencies": { "css-select": "^5.1.0", "entities": "^8.0.0" } }, "sha512-MhdaHPyxnyYu/sf0TpiRvDnTrkum0UKHC7FdbDGIUQNlx3I7xzwXoyV0eMUMv/XU+lkJT1glOUzpDPq7b2p1Ew=="],
|
||||
|
||||
"nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="],
|
||||
|
||||
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||
|
||||
"picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
|
||||
@@ -156,5 +177,7 @@
|
||||
"@tailwindcss/oxide-wasm32-wasi/@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="],
|
||||
|
||||
"@tailwindcss/oxide-wasm32-wasi/tslib": ["tslib@2.8.1", "", { "bundled": true }, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
|
||||
|
||||
"dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
{{- /* Build sorted list of unique tags */ -}}
|
||||
{{- $allTags := slice -}}
|
||||
{{- range $path, $ops := sort $spec.paths -}}
|
||||
{{- range $path, $ops := $spec.paths -}}
|
||||
{{- range $method := $methods -}}
|
||||
{{- with index $ops $method -}}
|
||||
{{- $op := . -}}
|
||||
@@ -36,7 +36,7 @@
|
||||
<div id="tag-{{ $tag | urlize }}" class="scroll-mt-24">
|
||||
<h2 class="text-2xl font-bold text-brand-near-black capitalize">{{ replace $tag "_" " " }}</h2>
|
||||
<div class="mt-6 space-y-6">
|
||||
{{- range $path, $ops := sort $spec.paths -}}
|
||||
{{- range $path, $ops := $spec.paths -}}
|
||||
{{- range $method := $methods -}}
|
||||
{{- with index $ops $method -}}
|
||||
{{- $op := . -}}
|
||||
@@ -95,7 +95,7 @@
|
||||
{{- $contentType := "application/json" -}}
|
||||
{{- if not $body -}}
|
||||
{{- $content := .content -}}
|
||||
{{- range $ct, $entry := sort $content -}}
|
||||
{{- range $ct, $entry := $content -}}
|
||||
{{- if not $body -}}{{- $body = $entry -}}{{- $contentType = $ct -}}{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
@@ -112,7 +112,10 @@
|
||||
<div>
|
||||
<h3 class="text-xs font-semibold uppercase tracking-wide text-zinc-500 mb-2">Responses</h3>
|
||||
<div class="space-y-3">
|
||||
{{- range $status, $resp := sort $responses -}}
|
||||
{{- /* Map range is already key-sorted; `sort`
|
||||
would drop the keys and make $status a
|
||||
numeric index instead of the HTTP code. */ -}}
|
||||
{{- range $status, $resp := $responses -}}
|
||||
<details class="border border-zinc-200 rounded">
|
||||
<summary class="cursor-pointer px-3 py-2 text-sm font-mono flex items-center gap-3 hover:bg-zinc-50">
|
||||
{{- $statusColor := "bg-zinc-100 text-zinc-700" -}}
|
||||
|
||||
@@ -34,7 +34,11 @@
|
||||
</thead>
|
||||
<tbody>
|
||||
{{- $required := default (slice) $schema.required -}}
|
||||
{{- range $name, $prop := sort $schema.properties -}}
|
||||
{{- /* Range directly over the map: Hugo iterates map keys in
|
||||
sorted order, and `sort` here would collapse the map to a
|
||||
slice — making $name the numeric index instead of the
|
||||
property name. */ -}}
|
||||
{{- range $name, $prop := $schema.properties -}}
|
||||
{{- $propRef := index $prop "$ref" -}}
|
||||
<tr class="border-b border-zinc-100 align-top">
|
||||
<td class="py-2 pr-4 font-mono text-brand-purple">
|
||||
|
||||
@@ -9,12 +9,14 @@
|
||||
"css:build": "bun run assets:install && tailwindcss -i src/main.css -o assets/styles/style.css",
|
||||
"css:watch": "bun run assets:install && tailwindcss -i src/main.css -o assets/styles/style.css --watch",
|
||||
"dev": "bash -c 'bun run css:watch & css=$!; hugo server & hugo=$!; trap \"kill $css $hugo 2>/dev/null\" EXIT INT TERM; wait -n; exit $?'",
|
||||
"build": "bun run css:build && hugo"
|
||||
"llms:generate": "bun run scripts/generate-llms.ts",
|
||||
"build": "bun run css:build && hugo && bun run llms:generate"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@fontsource/plus-jakarta-sans": "^5.2.8",
|
||||
"@fortawesome/fontawesome-free": "^7.2.0",
|
||||
"@tailwindcss/cli": "^4.2.4",
|
||||
"@types/bun": "^1.3.14"
|
||||
"@types/bun": "^1.3.14",
|
||||
"node-html-parser": "^9.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
414
website/scripts/generate-llms.ts
Normal file
414
website/scripts/generate-llms.ts
Normal file
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Generates `llms.txt` and `llms-full.txt` for the marketing site.
|
||||
*
|
||||
* Both files follow the llmstxt.org convention and are served from the
|
||||
* site root so LLM tooling can find them at the well-known paths:
|
||||
*
|
||||
* - /llms.txt — a curated index: H1, a `>` summary, then H2
|
||||
* sections linking to each page.
|
||||
* - /llms-full.txt — the full text of every page concatenated into
|
||||
* one Markdown document, so a model can ingest the
|
||||
* whole site without following links.
|
||||
*
|
||||
* The generator runs AFTER `hugo` against the built `public/` tree and
|
||||
* extracts text from the rendered HTML. Reading the published output
|
||||
* (rather than the Markdown/data sources) means the file never drifts
|
||||
* from what visitors actually see — including the marketing copy that
|
||||
* lives in Hugo layouts (home, features, get-started) rather than in
|
||||
* `content/*.md`.
|
||||
*/
|
||||
|
||||
import { readdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import { join, relative } from 'node:path'
|
||||
|
||||
import {
|
||||
parse,
|
||||
NodeType,
|
||||
type HTMLElement,
|
||||
type Node,
|
||||
type TextNode,
|
||||
} from 'node-html-parser'
|
||||
|
||||
const PUBLIC_DIR = join(import.meta.dir, '..', 'public')
|
||||
const HUGO_CONFIG = join(import.meta.dir, '..', 'hugo.toml')
|
||||
|
||||
// Inline elements that, when they appear on their own at block level,
|
||||
// should be serialized as inline Markdown rather than recursed into
|
||||
// (keeps e.g. permalink "#" anchors from leaking onto their own line).
|
||||
const INLINE_TAGS = new Set([
|
||||
'a',
|
||||
'span',
|
||||
'strong',
|
||||
'b',
|
||||
'em',
|
||||
'i',
|
||||
'code',
|
||||
'small',
|
||||
'sup',
|
||||
'sub',
|
||||
'abbr',
|
||||
'time',
|
||||
'kbd',
|
||||
'mark',
|
||||
])
|
||||
|
||||
// Site chrome and non-prose elements to drop before serializing.
|
||||
const STRIP_TAGS = new Set([
|
||||
'nav',
|
||||
'footer',
|
||||
'script',
|
||||
'style',
|
||||
'noscript',
|
||||
'svg',
|
||||
'head',
|
||||
'template',
|
||||
])
|
||||
|
||||
// Preferred ordering for the top of the file; anything not listed
|
||||
// falls through in alphabetical route order after these.
|
||||
const ROUTE_ORDER = [
|
||||
'/',
|
||||
'/features/',
|
||||
'/get-started/',
|
||||
'/api/',
|
||||
'/faq/',
|
||||
'/docs/',
|
||||
]
|
||||
|
||||
interface Page {
|
||||
route: string
|
||||
url: string
|
||||
title: string
|
||||
description: string
|
||||
body: string
|
||||
}
|
||||
|
||||
/** Read `baseURL` and `title` out of hugo.toml without a TOML parser. */
|
||||
async function readSiteMeta(): Promise<{ baseURL: string; title: string }> {
|
||||
const toml = await readFile(HUGO_CONFIG, 'utf8')
|
||||
const baseURL = toml.match(/^\s*baseURL\s*=\s*(["'])(.*?)\1/m)?.[2] ?? '/'
|
||||
const title = toml.match(/^\s*title\s*=\s*(["'])(.*?)\1/m)?.[2] ?? 'Site'
|
||||
return { baseURL: baseURL.replace(/\/$/, ''), title }
|
||||
}
|
||||
|
||||
/** Recursively collect every `index.html` under `public/`. */
|
||||
async function findHtmlFiles(dir: string): Promise<string[]> {
|
||||
const entries = await readdir(dir, { withFileTypes: true })
|
||||
const files: string[] = []
|
||||
for (const entry of entries) {
|
||||
const full = join(dir, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await findHtmlFiles(full)))
|
||||
} else if (entry.name === 'index.html') {
|
||||
files.push(full)
|
||||
}
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
/** Map a built file path to its site route, e.g. docs/qa/index.html -> /docs/qa/ */
|
||||
function routeForFile(file: string): string {
|
||||
const rel = relative(PUBLIC_DIR, file).replace(/index\.html$/, '')
|
||||
return '/' + rel.replace(/\\/g, '/')
|
||||
}
|
||||
|
||||
const collapse = (s: string) => s.replace(/[ \t\r\n]+/g, ' ')
|
||||
|
||||
/** Serialize an element's inline children (text, links, emphasis) to Markdown. */
|
||||
function inline(node: Node): string {
|
||||
if (node.nodeType === NodeType.TEXT_NODE)
|
||||
return collapse((node as TextNode).text)
|
||||
const el = node as HTMLElement
|
||||
if (!el.tagName) return ''
|
||||
const tag = el.tagName.toLowerCase()
|
||||
if (STRIP_TAGS.has(tag)) return ''
|
||||
const inner = el.childNodes.map(inline).join('')
|
||||
switch (tag) {
|
||||
case 'br':
|
||||
return '\n'
|
||||
case 'a': {
|
||||
const href = el.getAttribute('href') ?? ''
|
||||
const text = inner.trim()
|
||||
if (!text) return ''
|
||||
// Drop bare permalink glyphs (e.g. the "#" heading anchors).
|
||||
if (/^[#¶§]+$/.test(text)) return ''
|
||||
// Keep the visible text but drop the target for in-page anchors.
|
||||
if (!href || href.startsWith('#')) return text
|
||||
return `[${text}](${href})`
|
||||
}
|
||||
case 'strong':
|
||||
case 'b':
|
||||
return inner.trim() ? `**${inner.trim()}**` : ''
|
||||
case 'em':
|
||||
case 'i':
|
||||
return inner.trim() ? `*${inner.trim()}*` : ''
|
||||
case 'code':
|
||||
return inner.trim() ? `\`${inner.trim()}\`` : ''
|
||||
default:
|
||||
return inner
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize an element's children that mix inline text with block
|
||||
* children (e.g. a list item holding prose plus a code block):
|
||||
* consecutive inline nodes are joined into one line, block nodes are
|
||||
* emitted as their own entries.
|
||||
*/
|
||||
function serializeMixed(el: HTMLElement, out: string[]): void {
|
||||
let buf = ''
|
||||
const flush = () => {
|
||||
const t = buf.trim()
|
||||
if (t) out.push(t)
|
||||
buf = ''
|
||||
}
|
||||
for (const child of el.childNodes) {
|
||||
if (child.nodeType === NodeType.TEXT_NODE) {
|
||||
// Collapse the same way inline() does, so HTML source
|
||||
// indentation/newlines don't leak into the output.
|
||||
buf += collapse((child as TextNode).text)
|
||||
continue
|
||||
}
|
||||
const c = child as HTMLElement
|
||||
const tag = c.tagName?.toLowerCase()
|
||||
if (!tag || STRIP_TAGS.has(tag)) continue
|
||||
if (INLINE_TAGS.has(tag)) {
|
||||
buf += inline(c)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
block(c, out)
|
||||
}
|
||||
flush()
|
||||
}
|
||||
|
||||
/** Serialize a block-level element tree to a Markdown string. */
|
||||
function block(node: Node, out: string[]): void {
|
||||
if (node.nodeType === NodeType.TEXT_NODE) {
|
||||
const text = collapse((node as TextNode).text).trim()
|
||||
if (text) out.push(text)
|
||||
return
|
||||
}
|
||||
const el = node as HTMLElement
|
||||
if (!el.tagName) return
|
||||
const tag = el.tagName.toLowerCase()
|
||||
if (STRIP_TAGS.has(tag)) return
|
||||
|
||||
// A standalone inline element at block level: serialize inline so
|
||||
// link/emphasis handling (and permalink-glyph stripping) applies.
|
||||
if (INLINE_TAGS.has(tag)) {
|
||||
const text = inline(el).trim()
|
||||
if (text) out.push(text)
|
||||
return
|
||||
}
|
||||
|
||||
switch (tag) {
|
||||
case 'h1':
|
||||
case 'h2':
|
||||
case 'h3':
|
||||
case 'h4':
|
||||
case 'h5':
|
||||
case 'h6': {
|
||||
const level = Number(tag[1])
|
||||
const text = inline(el).trim()
|
||||
// Down-shift by one so page <h1> becomes "##" under the page's
|
||||
// own "#" heading in llms-full.txt.
|
||||
if (text) out.push(`${'#'.repeat(Math.min(level + 1, 6))} ${text}`)
|
||||
return
|
||||
}
|
||||
case 'p': {
|
||||
const text = inline(el).trim()
|
||||
if (text) out.push(text)
|
||||
return
|
||||
}
|
||||
case 'ul':
|
||||
case 'ol': {
|
||||
const ordered = tag === 'ol'
|
||||
const lines: string[] = []
|
||||
let i = 1
|
||||
for (const li of el.childNodes) {
|
||||
if ((li as HTMLElement).tagName?.toLowerCase() !== 'li') continue
|
||||
const sub: string[] = []
|
||||
serializeMixed(li as HTMLElement, sub)
|
||||
const body = sub.join('\n\n').trim()
|
||||
if (!body) continue
|
||||
const marker = ordered ? `${i++}.` : '-'
|
||||
const [first, ...rest] = body.split('\n')
|
||||
lines.push(`${marker} ${first}`)
|
||||
// Indent any continuation lines (code fences, nested lists)
|
||||
// so they stay attached to the bullet.
|
||||
for (const r of rest) lines.push(r ? ` ${r}` : '')
|
||||
}
|
||||
if (lines.length) out.push(lines.join('\n'))
|
||||
return
|
||||
}
|
||||
case 'pre': {
|
||||
// node-html-parser treats <pre> as a raw-text element, so its
|
||||
// single child holds the inner HTML (Chroma's <code>/<span>
|
||||
// wrappers) as an entity-encoded string. Re-parse that raw
|
||||
// string and take its text: real code angle brackets are
|
||||
// encoded as </> there, so the wrapper tags strip cleanly
|
||||
// while literal "<...>" in the code survives.
|
||||
const rawInner = el.childNodes
|
||||
.map((c) => (c.nodeType === NodeType.TEXT_NODE ? (c as TextNode).rawText : ''))
|
||||
.join('')
|
||||
const code = parse(rawInner).textContent.replace(/\n+$/, '')
|
||||
if (code.trim()) out.push('```\n' + code + '\n```')
|
||||
return
|
||||
}
|
||||
case 'blockquote': {
|
||||
const inner: string[] = []
|
||||
for (const child of el.childNodes) block(child, inner)
|
||||
const text = inner.join('\n\n').trim()
|
||||
if (text)
|
||||
out.push(
|
||||
text
|
||||
.split('\n')
|
||||
.map((l) => `> ${l}`)
|
||||
.join('\n'),
|
||||
)
|
||||
return
|
||||
}
|
||||
case 'table': {
|
||||
const rows: string[][] = []
|
||||
for (const tr of el.querySelectorAll('tr')) {
|
||||
const cells = tr
|
||||
.querySelectorAll('th,td')
|
||||
// Escape backslashes before pipes so a literal "\" in a cell
|
||||
// can't combine with the pipe escape.
|
||||
.map((c) =>
|
||||
inline(c).trim().replace(/\\/g, '\\\\').replace(/\|/g, '\\|'),
|
||||
)
|
||||
if (cells.length) rows.push(cells)
|
||||
}
|
||||
if (rows.length) {
|
||||
const md = [`| ${rows[0].join(' | ')} |`]
|
||||
md.push(`| ${rows[0].map(() => '---').join(' | ')} |`)
|
||||
for (const r of rows.slice(1)) md.push(`| ${r.join(' | ')} |`)
|
||||
out.push(md.join('\n'))
|
||||
}
|
||||
return
|
||||
}
|
||||
default:
|
||||
// Structural container (div, section, header, article, main, …):
|
||||
// recurse into children.
|
||||
for (const child of el.childNodes) block(child, out)
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract a clean Markdown body + metadata from one rendered page. */
|
||||
function extractPage(html: string, route: string, baseURL: string): Page | null {
|
||||
const root = parse(html)
|
||||
|
||||
// Skip Hugo alias/redirect stubs.
|
||||
if (root.querySelector('meta[http-equiv=refresh]')) return null
|
||||
|
||||
const title = (root.querySelector('title')?.textContent ?? route).trim()
|
||||
const description = (
|
||||
root.querySelector('meta[name=description]')?.getAttribute('content') ?? ''
|
||||
).trim()
|
||||
const canonical = root
|
||||
.querySelector('link[rel=canonical]')
|
||||
?.getAttribute('href')
|
||||
const url = canonical ?? baseURL + route
|
||||
|
||||
const body = root.querySelector('body')
|
||||
if (!body) return null
|
||||
|
||||
const out: string[] = []
|
||||
for (const child of body.childNodes) block(child, out)
|
||||
|
||||
// Collapse runs of blank lines the recursion may have produced.
|
||||
const text = out.join('\n\n').replace(/\n{3,}/g, '\n\n').trim()
|
||||
return { route, url, title, description, body: text }
|
||||
}
|
||||
|
||||
function orderPages(pages: Page[]): Page[] {
|
||||
const rank = (route: string) => {
|
||||
const i = ROUTE_ORDER.indexOf(route)
|
||||
if (i !== -1) return i
|
||||
// Keep docs children grouped right after /docs/ but before unknowns.
|
||||
return route.startsWith('/docs/') ? ROUTE_ORDER.length : ROUTE_ORDER.length + 1
|
||||
}
|
||||
return [...pages].sort(
|
||||
(a, b) => rank(a.route) - rank(b.route) || a.route.localeCompare(b.route),
|
||||
)
|
||||
}
|
||||
|
||||
function stripHtmlTitleSuffix(title: string): string {
|
||||
// Layouts append " | Anthias"; drop it for cleaner section headings.
|
||||
return title.replace(/\s*\|\s*Anthias\s*$/, '').trim()
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { baseURL, title } = await readSiteMeta()
|
||||
const files = await findHtmlFiles(PUBLIC_DIR)
|
||||
|
||||
const pages: Page[] = []
|
||||
for (const file of files) {
|
||||
const route = routeForFile(file)
|
||||
const html = await readFile(file, 'utf8')
|
||||
const page = extractPage(html, route, baseURL)
|
||||
if (page && page.body) pages.push(page)
|
||||
}
|
||||
|
||||
const ordered = orderPages(pages)
|
||||
const summary =
|
||||
'Anthias is the world’s most popular open source digital signage ' +
|
||||
'software. It turns a Raspberry Pi, x86 PC, or generic 64-bit ARM board ' +
|
||||
'into a digital sign that displays images, videos, and web pages, ' +
|
||||
'scheduled from a self-hosted web dashboard. No subscriptions, no cloud ' +
|
||||
'lock-in.'
|
||||
|
||||
// --- llms.txt (curated index) ---
|
||||
const indexLines: string[] = [`# ${title}`, '', `> ${summary}`, '']
|
||||
const docs = ordered.filter((p) => p.route.startsWith('/docs/') && p.route !== '/docs/')
|
||||
const primary = ordered.filter((p) => !p.route.startsWith('/docs/') || p.route === '/docs/')
|
||||
indexLines.push('## Pages', '')
|
||||
for (const p of primary) {
|
||||
const name = stripHtmlTitleSuffix(p.title)
|
||||
indexLines.push(`- [${name}](${p.url})${p.description ? `: ${p.description}` : ''}`)
|
||||
}
|
||||
if (docs.length) {
|
||||
indexLines.push('', '## Documentation', '')
|
||||
for (const p of docs) {
|
||||
const name = stripHtmlTitleSuffix(p.title)
|
||||
indexLines.push(`- [${name}](${p.url})${p.description ? `: ${p.description}` : ''}`)
|
||||
}
|
||||
}
|
||||
indexLines.push('', '## Full text', '')
|
||||
indexLines.push(`- [llms-full.txt](${baseURL}/llms-full.txt): the full text of every page on this site, concatenated into one document.`)
|
||||
indexLines.push('')
|
||||
|
||||
// --- llms-full.txt (full content) ---
|
||||
const fullLines: string[] = [
|
||||
`# ${title}`,
|
||||
'',
|
||||
`> ${summary}`,
|
||||
'',
|
||||
`This file concatenates the full text of every page on ${baseURL}. Each section is one page, with its source URL.`,
|
||||
'',
|
||||
]
|
||||
for (const p of ordered) {
|
||||
fullLines.push('---', '')
|
||||
fullLines.push(`# ${stripHtmlTitleSuffix(p.title)}`, '')
|
||||
fullLines.push(`Source: ${p.url}`, '')
|
||||
if (p.description) fullLines.push(`> ${p.description}`, '')
|
||||
fullLines.push(p.body, '')
|
||||
}
|
||||
|
||||
await writeFile(join(PUBLIC_DIR, 'llms.txt'), indexLines.join('\n'))
|
||||
await writeFile(join(PUBLIC_DIR, 'llms-full.txt'), fullLines.join('\n'))
|
||||
|
||||
const fullBytes = Buffer.byteLength(fullLines.join('\n'))
|
||||
console.log(
|
||||
`Wrote llms.txt (${primary.length + docs.length} pages indexed) and ` +
|
||||
`llms-full.txt (${ordered.length} pages, ${(fullBytes / 1024).toFixed(1)} KiB).`,
|
||||
)
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err)
|
||||
process.exitCode = 1
|
||||
})
|
||||
Reference in New Issue
Block a user