feat: llms.txt and llms-full.txt, openapi.json, MCP server
This commit is contained in:
Florian Sylvain committed
1 parent
daba3c04c0
commit
4cff43fefb
25 files changed
+1691
-1
No files matched your search
@@ -0,0 +1,321 @@
|
||||
import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { basename, dirname, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import {
|
||||
API_GROUPS,
|
||||
ERROR_ROWS,
|
||||
PUBLIC_AUTH_ROWS,
|
||||
SESSION_GROUPS,
|
||||
endpointExamples,
|
||||
type ApiEndpoint,
|
||||
type ApiField,
|
||||
} from '../web/src/data/apiCatalogue'
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const outDir = join(root, 'internal/infrastructure/llmdocs/generated')
|
||||
const pagesDir = join(outDir, 'llms')
|
||||
|
||||
const VERSION = '{{VERSION}}'
|
||||
const BASE_URL = '{{BASE_URL}}'
|
||||
|
||||
interface Page {
|
||||
name: string
|
||||
title: string
|
||||
summary: string
|
||||
body: () => string
|
||||
}
|
||||
|
||||
const read = (relative: string) => readFileSync(join(root, relative), 'utf8')
|
||||
|
||||
function section(markdown: string, heading: string) {
|
||||
const lines = markdown.split('\n')
|
||||
const start = lines.findIndex((line) => line.trim() === heading)
|
||||
if (start === -1) throw new Error(`heading not found: ${heading}`)
|
||||
const level = heading.match(/^#+/)![0].length
|
||||
const end = lines.findIndex(
|
||||
(line, index) => index > start && /^#+ /.test(line) && line.match(/^#+/)![0].length <= level,
|
||||
)
|
||||
return lines.slice(start, end === -1 ? undefined : end).join('\n')
|
||||
}
|
||||
|
||||
const docFiles = (directory: string) =>
|
||||
readdirSync(join(root, directory))
|
||||
.filter((file) => file.endsWith('.md'))
|
||||
.sort()
|
||||
.map((file) => `${directory}/${file}`)
|
||||
|
||||
const fieldTable = (fields: ApiField[]) =>
|
||||
[
|
||||
'| Name | Type | Required | Rule |',
|
||||
'| --- | --- | --- | --- |',
|
||||
...fields.map(
|
||||
(field) =>
|
||||
`| \`${field.name}\` | ${field.type} | ${field.required ? 'yes' : 'no'} | ${field.rule.replaceAll('|', '\\|')} |`,
|
||||
),
|
||||
].join('\n')
|
||||
|
||||
function endpointMarkdown(item: ApiEndpoint) {
|
||||
const examples = endpointExamples(item, BASE_URL)
|
||||
const parts = [
|
||||
`### ${item.method} ${item.path}`,
|
||||
`Access: ${item.access}. Success status: ${item.status}.${item.rateLimited ? ' Rate limited.' : ''}`,
|
||||
item.description,
|
||||
]
|
||||
if (item.query.length > 0) parts.push(`Query parameters:\n\n${fieldTable(item.query)}`)
|
||||
if (item.body.length > 0) {
|
||||
const kind = item.bodyKind === 'multipart' ? 'multipart/form-data' : 'JSON'
|
||||
parts.push(`Body (${kind}):\n\n${fieldTable(item.body)}`)
|
||||
}
|
||||
parts.push(`Example:\n\n\`\`\`sh\n${examples.curl}\n\`\`\``)
|
||||
parts.push(`Response:\n\n\`\`\`\n${item.response}\n\`\`\``)
|
||||
if (item.errors.length > 0) {
|
||||
parts.push(
|
||||
`Errors:\n\n${item.errors.map((error) => `- ${error.status}: ${error.caption}`).join('\n')}`,
|
||||
)
|
||||
}
|
||||
return parts.join('\n\n')
|
||||
}
|
||||
|
||||
function apiReference() {
|
||||
const parts = [
|
||||
'# API reference',
|
||||
`Generated from the route catalogue of RenewCMS ${VERSION}. Base URL: ${BASE_URL}. Every path starts with /v1. Send \`Authorization: Bearer rcms_...\` on every route except the public ones.`,
|
||||
]
|
||||
for (const group of API_GROUPS) {
|
||||
parts.push(`## ${group.title}`, group.description)
|
||||
parts.push(...group.endpoints.map(endpointMarkdown))
|
||||
}
|
||||
parts.push(
|
||||
'## Session-only routes',
|
||||
'A key sent to these routes answers 403 "This action needs a real session.", except the sign-in routes, which ignore a key.',
|
||||
)
|
||||
for (const group of SESSION_GROUPS) {
|
||||
parts.push(
|
||||
`### ${group.title}`,
|
||||
group.rows.map((row) => `- ${row.method} ${row.path}: ${row.description}`).join('\n'),
|
||||
)
|
||||
}
|
||||
parts.push(
|
||||
'## Public routes outside the article reads',
|
||||
PUBLIC_AUTH_ROWS.map((row) => `- ${row.method} ${row.path}: ${row.description}`).join('\n'),
|
||||
)
|
||||
parts.push(
|
||||
'## Errors',
|
||||
ERROR_ROWS.map(
|
||||
(row) =>
|
||||
`- ${row.status} ${row.meaning}: ${row.messages.map((message) => `"${message}"`).join(', ')}${row.note ? ` ${row.note}` : ''}`,
|
||||
).join('\n'),
|
||||
)
|
||||
return parts.join('\n\n')
|
||||
}
|
||||
|
||||
const API_PAGE_SUMMARIES: Record<string, string> = {
|
||||
articles: 'Article reads: pagination, search, counts, the format parameter, SEO status.',
|
||||
'api-keys': 'Creating, listing and revoking API keys, and where a key is accepted.',
|
||||
}
|
||||
|
||||
const pages: Page[] = [
|
||||
{
|
||||
name: 'api-reference',
|
||||
title: 'API reference',
|
||||
summary: 'Every key-callable route with parameters, bodies, curl examples, responses and errors.',
|
||||
body: apiReference,
|
||||
},
|
||||
{
|
||||
name: 'api',
|
||||
title: 'API overview',
|
||||
summary: 'Base URL, authentication, API key limits, activity log.',
|
||||
body: () => read('docs/api.md'),
|
||||
},
|
||||
...docFiles('docs/api').map((file) => ({
|
||||
name: basename(file, '.md'),
|
||||
title: read(file).match(/^# (.+)$/m)?.[1] ?? basename(file, '.md'),
|
||||
summary: API_PAGE_SUMMARIES[basename(file, '.md')] ?? '',
|
||||
body: () => read(file),
|
||||
})),
|
||||
...docFiles('docs/llm').map((file) => ({
|
||||
name: basename(file, '.md'),
|
||||
title: read(file).match(/^# (.+)$/m)?.[1] ?? basename(file, '.md'),
|
||||
summary: read(file).match(/^> (.+)$/m)?.[1] ?? '',
|
||||
body: () => read(file).replace(/^> .+\n+/m, ''),
|
||||
})),
|
||||
{
|
||||
name: 'configuration',
|
||||
title: 'Configuration',
|
||||
summary: 'Environment variables of the server.',
|
||||
body: () => `# Configuration\n\n${section(read('docs/configuration.md'), '## Variables')}`,
|
||||
},
|
||||
]
|
||||
|
||||
const pageNames = new Map(pages.map((page) => [page.name, page]))
|
||||
|
||||
function rewriteLinks(markdown: string) {
|
||||
return markdown.replace(/\[([^\]]+)\]\(([^)\s]+)\)/g, (whole, text: string, target: string) => {
|
||||
if (/^(https?:|mailto:|#|\{\{)/.test(target)) return whole
|
||||
const [file, anchor] = target.split('#')
|
||||
const page = pageNames.get(basename(file, '.md'))
|
||||
if (!page || !file.endsWith('.md')) return text
|
||||
return `[${text}](${BASE_URL}/llms/${page.name}.md${anchor ? `#${anchor}` : ''})`
|
||||
})
|
||||
}
|
||||
|
||||
const rendered = pages.map((page) => ({ page, text: rewriteLinks(page.body()).trim() }))
|
||||
|
||||
const index = [
|
||||
'# RenewCMS',
|
||||
`> A small self-hosted CMS. A JSON API under /v1 manages articles, media and sites; articles and sitemaps are readable without credentials. Version ${VERSION}, served by ${BASE_URL}.`,
|
||||
'Authenticate with an API key: `Authorization: Bearer rcms_...`. A key acts as its creator, 120 requests per minute. Some routes need a browser session and answer 403 to a key; see the gotchas page.',
|
||||
'## Docs',
|
||||
rendered
|
||||
.map(({ page }) => `- [${page.title}](${BASE_URL}/llms/${page.name}.md): ${page.summary}`)
|
||||
.join('\n'),
|
||||
'## Machine-readable',
|
||||
[
|
||||
`- [llms-full.txt](${BASE_URL}/llms-full.txt): all of the pages above in one file`,
|
||||
`- [openapi.json](${BASE_URL}/openapi.json): OpenAPI 3.1 description of the routes in the API reference`,
|
||||
].join('\n'),
|
||||
].join('\n\n')
|
||||
|
||||
const full = [
|
||||
`# RenewCMS documentation\n\nVersion ${VERSION}. Served by ${BASE_URL}.`,
|
||||
...rendered.map(({ text }) => text),
|
||||
].join('\n\n---\n\n')
|
||||
|
||||
const SCALARS: Record<string, object> = {
|
||||
string: { type: 'string' },
|
||||
number: { type: 'number' },
|
||||
boolean: { type: 'boolean' },
|
||||
null: { type: 'null' },
|
||||
file: { type: 'string', format: 'binary' },
|
||||
}
|
||||
|
||||
function schemaOf(type: string): object {
|
||||
const alternatives = type.split('|').map((part) => part.trim())
|
||||
if (alternatives.length > 1) return { anyOf: alternatives.map(schemaOf) }
|
||||
if (type.endsWith('[]')) return { type: 'array', items: schemaOf(type.slice(0, -2)) }
|
||||
return SCALARS[type] ?? {}
|
||||
}
|
||||
|
||||
const objectSchema = (fields: ApiField[]) => ({
|
||||
type: 'object',
|
||||
properties: Object.fromEntries(
|
||||
fields.map((field) => [field.name, { ...schemaOf(field.type), description: field.rule }]),
|
||||
),
|
||||
required: fields.filter((field) => field.required).map((field) => field.name),
|
||||
})
|
||||
|
||||
function parseExample(response: string) {
|
||||
try {
|
||||
return JSON.parse(response)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function operation(item: ApiEndpoint) {
|
||||
const pathParameters = [...item.path.matchAll(/\{(\w+)\}/g)].map((match) => ({
|
||||
name: match[1],
|
||||
in: 'path',
|
||||
required: true,
|
||||
schema: match[1] === 'id' ? { type: 'integer', minimum: 0 } : { type: 'string' },
|
||||
}))
|
||||
const queryParameters = item.query.map((field) => ({
|
||||
name: field.name,
|
||||
in: 'query',
|
||||
required: field.required,
|
||||
description: field.rule,
|
||||
schema: schemaOf(field.type),
|
||||
}))
|
||||
const example = parseExample(item.response)
|
||||
const responses: Record<string, object> = {
|
||||
[String(item.status)]:
|
||||
item.status === 204
|
||||
? { description: 'No content' }
|
||||
: {
|
||||
description: 'Success',
|
||||
content: {
|
||||
'application/json': { schema: {}, ...(example === undefined ? {} : { example }) },
|
||||
},
|
||||
},
|
||||
}
|
||||
for (const error of item.errors) {
|
||||
const key = String(error.status)
|
||||
const previous = (responses[key] as { description?: string } | undefined)?.description
|
||||
responses[key] = {
|
||||
description: previous ? `${previous} / ${error.caption}` : error.caption,
|
||||
content: { 'text/plain': { schema: { type: 'string' } } },
|
||||
}
|
||||
}
|
||||
const result: Record<string, unknown> = {
|
||||
operationId: item.id,
|
||||
summary: item.description,
|
||||
'x-access': item.access,
|
||||
parameters: [...pathParameters, ...queryParameters],
|
||||
responses,
|
||||
}
|
||||
if (item.access !== 'Public') result.security = [{ bearerAuth: [] }, { cookieAuth: [] }]
|
||||
if (item.bodyKind === 'json') {
|
||||
const media = {
|
||||
schema: objectSchema(item.body),
|
||||
...(item.bodySample ? { example: item.bodySample } : {}),
|
||||
}
|
||||
result.requestBody = { required: true, content: { 'application/json': media } }
|
||||
}
|
||||
if (item.bodyKind === 'multipart') {
|
||||
result.requestBody = {
|
||||
required: true,
|
||||
content: { 'multipart/form-data': { schema: objectSchema(item.body) } },
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
const paths: Record<string, Record<string, unknown>> = {}
|
||||
for (const group of API_GROUPS) {
|
||||
for (const item of group.endpoints) {
|
||||
paths[item.path] ??= {}
|
||||
paths[item.path][item.method.toLowerCase()] = operation(item)
|
||||
}
|
||||
}
|
||||
for (const group of SESSION_GROUPS) {
|
||||
for (const row of group.rows) {
|
||||
paths[row.path] ??= {}
|
||||
paths[row.path][row.method.toLowerCase()] = {
|
||||
summary: row.description,
|
||||
'x-session-only': row.answers403,
|
||||
security: [{ cookieAuth: [] }],
|
||||
responses: { '200': { description: 'Success' } },
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const row of PUBLIC_AUTH_ROWS) {
|
||||
paths[row.path] ??= {}
|
||||
paths[row.path][row.method.toLowerCase()] = {
|
||||
summary: row.description,
|
||||
responses: { '200': { description: 'Success' } },
|
||||
}
|
||||
}
|
||||
|
||||
const openapi = {
|
||||
openapi: '3.1.0',
|
||||
info: {
|
||||
title: 'RenewCMS API',
|
||||
version: VERSION,
|
||||
description: 'Generated from the route catalogue. Routes listed with x-session-only answer 403 to an API key.',
|
||||
},
|
||||
servers: [{ url: BASE_URL }],
|
||||
paths,
|
||||
components: {
|
||||
securitySchemes: {
|
||||
bearerAuth: { type: 'http', scheme: 'bearer', description: 'An API key starting with rcms_' },
|
||||
cookieAuth: { type: 'apiKey', in: 'cookie', name: 'jwt' },
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
rmSync(pagesDir, { recursive: true, force: true })
|
||||
mkdirSync(pagesDir, { recursive: true })
|
||||
for (const { page, text } of rendered) writeFileSync(join(pagesDir, `${page.name}.md`), `${text}\n`)
|
||||
writeFileSync(join(outDir, 'llms.txt'), `${index}\n`)
|
||||
writeFileSync(join(outDir, 'llms-full.txt'), `${full}\n`)
|
||||
writeFileSync(join(outDir, 'openapi.json'), `${JSON.stringify(openapi, null, 2)}\n`)
|
||||
Reference in new issue
Block a user