Add llms.txt generation for LLM-optimized documentation export (#1023)

* Initial plan

* Add llms.txt generation feature with documentation and changeset

Co-authored-by: mikebarkmin <[email protected]>

* Address code review comments - improve type safety and optimize file system operations

Co-authored-by: mikebarkmin <[email protected]>

* activate llms

* add more instructions to llms.txt

---------

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: mikebarkmin <[email protected]>
Co-authored-by: Mike Barkmin <[email protected]>

Author: Copilot

.changeset/llms-txt-generation.md

@@ -0,0 +1,6 @@
+---
+"hyperbook": minor
+"@hyperbook/types": minor
+---
+
+Add llms.txt file generation feature. When the `llms` property is set to `true` in hyperbook.json, a `llms.txt` file will be generated during build that combines all markdown files in order. The file includes the book name and version in the header. Pages and sections with `hide: true` are automatically excluded from the generated file.

packages/hyperbook/build.ts

@@ -11,6 +11,9 @@ import {
   Link,
   Hyperproject,
   HyperbookContext,
+  HyperbookJson,
+  HyperbookPage,
+  HyperbookSection,
   Navigation,
 } from "@hyperbook/types";
 import lunr from "lunr";
@@ -19,6 +22,139 @@ import packageJson from "./package.json";
 
 export const ASSETS_FOLDER = "__hyperbook_assets";
 
+/**
+ * Generates an llms.txt file by combining all markdown files in order
+ */
+async function generateLlmsTxt(
+  root: string,
+  rootOut: string,
+  hyperbookJson: HyperbookJson,
+  pagesAndSections: Pick<Navigation, "pages" | "sections" | "glossary">,
+  version: string,
+): Promise<void> {
+  const lines: string[] = [];
+
+  // Add header with book name and version
+  lines.push(`<SYSTEM>${hyperbookJson.name} - Version ${version}</SYSTEM>`);
+  lines.push(""); // Empty line after header
+
+  // Get all book files once to avoid repeated file system operations
+  const allFiles = await vfile.listForFolder(root, "book");
+
+  // Helper function to recursively process sections and pages
+  const processSection = async (
+    section: HyperbookSection,
+    level: number = 0,
+  ): Promise<void> => {
+    // Skip if hidden
+    if (section.hide) {
+      return;
+    }
+
+    // Add section header if it has content
+    if (section.href && !section.isEmpty) {
+      const file = allFiles.find((f) => f.path.href === section.href);
+      if (file) {
+        // Add section name as a header
+        lines.push(`# ${section.name}`);
+        lines.push("");
+
+        // Get the markdown content without frontmatter
+        const content = file.markdown.content.trim();
+        if (content) {
+          lines.push(content);
+          lines.push(""); // Empty line after content
+        }
+      }
+    }
+
+    // Process nested pages
+    if (section.pages) {
+      for (const page of section.pages) {
+        await processPage(page);
+      }
+    }
+
+    // Process nested sections
+    if (section.sections) {
+      for (const subsection of section.sections) {
+        await processSection(subsection, level + 1);
+      }
+    }
+  };
+
+  const processPage = async (page: HyperbookPage): Promise<void> => {
+    // Skip if hidden or empty
+    if (page.hide || page.isEmpty) {
+      return;
+    }
+
+    if (page.href) {
+      const file = allFiles.find((f) => f.path.href === page.href);
+      if (file) {
+        // Add page name as a header
+        lines.push(`# ${page.name}`);
+        lines.push("");
+
+        // Get the markdown content without frontmatter
+        const content = file.markdown.content.trim();
+        if (content) {
+          lines.push(content);
+          lines.push(""); // Empty line after content
+        }
+      }
+    }
+  };
+
+  // Process root-level pages first
+  for (const page of pagesAndSections.pages) {
+    await processPage(page);
+  }
+
+  // Process sections
+  for (const section of pagesAndSections.sections) {
+    await processSection(section);
+  }
+
+  // Write the llms.txt file
+  lines.push(`
+When you are writing a hyperbook syntax you must use markdown plus the documented elements, also keep track of colons. Colons act like parentheses in programming languages. Every opening colon must have a closing colon. For example, in the syntax :bold text: the first colon opens the bold formatting and the second colon closes it. If there is a missing colon, it can lead to formatting errors or unexpected behavior in the rendered output. Always ensure that colons are properly paired to maintain the intended structure and appearance of your hyperbook content.
+
+Single colons are inline elements.
+
+Example: :t[Test]
+
+Double colons are block elements.
+
+Example: ::p5{src="sketch.js"}
+
+Triple colons are special elements that can contain other elements inside them.
+
+Example:
+
+:::::alert{info}
+
+::::tabs
+
+:::tab{title="JavaScript"}
+
+Hi
+
+:::
+
+
+::::
+
+:::::
+
+When you want to nest elements you need to increase the number of colons by one for each level of nesting. The outer level should have the most colons.
+
+Also you need to use unique ids when the element supports it.
+`);
+  const llmsTxtContent = lines.join("\n");
+  await fs.writeFile(path.join(rootOut, "llms.txt"), llmsTxtContent);
+}
+
 export async function runBuildProject(
   project: Hyperproject,
   rootProject: Hyperproject,
@@ -390,17 +526,17 @@ async function runBuild(
 
   if (!faviconExists && hyperbookJson.logo) {
     console.log(`${chalk.blue(`[${prefix}]`)} Generating favicons from logo.`);
-    
+
     // Only generate if logo is a local file (not a URL)
     if (!hyperbookJson.logo.includes("://")) {
       let logoPath: string | null = null;
-      
+
       // Resolve logo path by checking multiple locations
       if (hyperbookJson.logo.startsWith("/")) {
         // Absolute path starting with / - check book folder, then public folder
         const bookPath = path.join(root, "book", hyperbookJson.logo);
         const publicPath = path.join(root, "public", hyperbookJson.logo);
-        
+
         try {
           await fs.access(bookPath);
           logoPath = bookPath;
@@ -417,7 +553,7 @@ async function runBuild(
         const rootPath = path.join(root, hyperbookJson.logo);
         const bookPath = path.join(root, "book", hyperbookJson.logo);
         const publicPath = path.join(root, "public", hyperbookJson.logo);
-        
+
         try {
           await fs.access(rootPath);
           logoPath = rootPath;
@@ -435,11 +571,18 @@ async function runBuild(
           }
         }
       }
-      
+
       if (logoPath) {
         try {
-          const { generateFavicons } = await import("./helpers/generate-favicons");
-          await generateFavicons(logoPath, rootOut, hyperbookJson, ASSETS_FOLDER);
+          const { generateFavicons } = await import(
+            "./helpers/generate-favicons"
+          );
+          await generateFavicons(
+            logoPath,
+            rootOut,
+            hyperbookJson,
+            ASSETS_FOLDER,
+          );
           console.log(
             `${chalk.green(`[${prefix}]`)} Favicons generated successfully.`,
           );
@@ -598,5 +741,17 @@ const SEARCH_DOCUMENTS = ${JSON.stringify(documents)};
     ),
   );
 
+  // Generate llms.txt if enabled
+  if (hyperbookJson.llms) {
+    console.log(`${chalk.blue(`[${prefix}]`)} Generating llms.txt`);
+    await generateLlmsTxt(
+      root,
+      rootOut,
+      hyperbookJson,
+      pagesAndSections,
+      packageJson.version,
+    );
+  }
+
   console.log(`${chalk.green(`[${prefix}]`)} Build success: ${rootOut}`);
 }

packages/types/src/index.ts

@@ -93,6 +93,7 @@ export type HyperbookJson = {
   search?: boolean;
   qrcode?: boolean;
   toc?: boolean;
+  llms?: boolean;
   author?: {
     name?: string;
     url?: string;

platforms/vscode/schemas/hyperbook.schema.json

@@ -280,6 +280,9 @@
             },
             "type": "array"
         },
+        "llms": {
+            "type": "boolean"
+        },
         "logo": {
             "type": "string"
         },

website/de/book/configuration/book.md

@@ -38,6 +38,7 @@ von Optionen, die du definieren kannst. Optionen mit einem "\*" müssen gesetzt
 | allowDangerousHtml | Erlaube HTML im Hyperbook. Dies kann zu Inkompatibilität in zukünftigen Versionen führen.                                                                                                       |
 | qrcode             | Zeigt ein Icon, um einen QR-Code zur aktuellen Seite anzuzeigen.                                                                                                                                |
 | toc         | Zeige ein Inhaltsverzeichnis. Diese ist standardmäßig aktiviert für Seiten und deaktiviert für Begriffe im Glossar.                            |
+| llms               | Wenn auf true gesetzt, wird eine llms.txt-Datei generiert, die alle Markdown-Dateien in Reihenfolge kombiniert. Die Datei enthält den Buchnamen und die Version im Header-Format.              |
 | trailingSlash      | Exportiert alle Datei in eigene Verzeichnisse und erzeugt nur index.html Dateien.                                                                                                               |
 | importExport       | Ermöglicht das Importieren und Exportieren des Zustands des Hyperbooks als Datei. Schaltflächen zum Importieren und Exportieren befinden sich am unteren Rand der Seite.                        |
 

website/de/hyperbook.json

@@ -2,6 +2,7 @@
   "name": "Hyperbook Dokumenation",
   "qrcode": true,
   "search": true,
+  "llms": true,
   "importExport": true,
   "description": "Dokumentation für Hyperbook erstellt mit Hyperbook",
   "author": {

website/en/book/configuration/book.md

@@ -37,6 +37,7 @@ can and part wise must set (indicated by a \*).
 | allowDangerousHtml | Allow HTML. This can lead to incompatibilities in future versions.                                                                                              |
 | qrcode             | Shows an icon, which opens a qr code to the current page.                                                                                                       |
 | toc         | Show or hide a table of content for the page. This is on for pages and off for glossary entries by default                          |
+| llms               | When set to true, generates an llms.txt file that combines all markdown files in order. The file includes the book name and version in a header format.         |
 | trailingSlash      | Outputs all files into ther own folders and produces only index.html files.                                                                                     |
 | importExport       | Allows to import and export the state of the Hyperbook as a file. Buttons for importing and exporting will be at the bottom of the page.                        |
 

website/en/hyperbook.json

@@ -1,6 +1,7 @@
 {
   "name": "Hyperbook Documentation",
   "qrcode": true,
+  "llms": true,
   "importExport": true,
   "search": true,
   "description": "Documentation for Hyperbook created with Hyperbook",