Add script for generating llms.txt and llms-full.txt (#222)

* Add script for generating llms.txt and llms-full.txt

* Use the title instead of the filename to identify links

* Don't validate llms links

* Force .html extension on llms.txt links

* Declare the pageheader string pattern as a variable

* Add page headers to header text as a clue in llms-full.txt

Author: bbenligiray

.gitignore

@@ -9,6 +9,9 @@ docs/.vitepress/dist/
 docs/.vitepress/.temp
 .firebase/
 
+docs/public/llms.txt
+docs/public/llms-full.txt
+
 /indexes/content-files/
 
 yarn-error.log

.prettierignore

@@ -14,6 +14,9 @@ pnpm-lock.yaml
 /docs/public/img
 /docs/public/api3-whitepaper-v1.0.3.pdf
 
+docs/public/llms.txt
+docs/public/llms-full.txt
+
 # assets
 /docs/reference/ois/latest/assets
 /docs/reference/airnode/latest/assets

docs/dapps/index.md

@@ -13,6 +13,12 @@ Api3 provides data feeds and pays dApps for using them.
 
 2. Api3 enables the Oracle Extractable Value (OEV) resulting from the usage of these data feeds to be captured, and pays it to the respective dApps in the form of [OEV Rewards.](#oev-rewards)
 
+::: info 💡 Tip
+
+For quick reference, you can copy-paste [`llms-full.txt`](https://docs.api3.org/llms-full.txt) to your choice of AI assistant.
+
+:::
+
 ## Api3 Market
 
 Liquidity is increasingly shifting to newly launched L2 networks, and dApps that are able to branch out to these more quickly are at a significant competitive advantage.

docs/oev-searchers/index.md

@@ -28,6 +28,12 @@ bound by rules enforced on-chain. The auction winner must pay the announced
 amount, which in return allows them to perform the oracle update and capture
 profitable opportunities.
 
+::: info 💡 Tip
+
+For quick reference, you can copy-paste [`llms-full.txt`](https://docs.api3.org/llms-full.txt) to your choice of AI assistant.
+
+:::
+
 ## Practical example
 
 Imagine an overcollateralized lending platform that uses Api3 price feeds.

libs/link-validator-ignore.json

@@ -7,5 +7,7 @@
   "https://orbitlending.io/",
   "https://www.coinbase.com/blog/introducing-the-coinbase-price-oracle",
   "https://x.com",
-  "https://blastscan.io/address/0x5b0cf2b36a65a6BB085D501B971e4c102B9Cd473#readProxyContract#F17"
+  "https://blastscan.io/address/0x5b0cf2b36a65a6BB085D501B971e4c102B9Cd473#readProxyContract#F17",
+  "https://docs.api3.org/llms.txt",
+  "https://docs.api3.org/llms-full.txt"
 ]

package.json

@@ -6,11 +6,12 @@
   "packageManager": "[email protected]",
   "resolutions": {},
   "scripts": {
-    "docs:dev": "pnpm vitepress dev docs",
-    "docs:build": "pnpm vitepress build docs;",
+    "docs:dev": "pnpm generate-llms-files && pnpm vitepress dev docs",
+    "docs:build": "pnpm generate-llms-files && pnpm vitepress build docs;",
     "docs:serve": "vitepress serve docs --port 8082",
     "format": "prettier --write --cache \"./**/*.{js,vue,md,json,yaml}\" --log-level silent",
     "format:check": "prettier --check --cache \"./**/*.{js,vue,md,json,yaml}\"",
+    "generate-llms-files": "node scripts/generate-llms-files.js",
     "prepare": "husky",
     "firebase:emulator": "pnpm docs:build; firebase emulators:start"
   },

scripts/generate-llms-files.js

@@ -0,0 +1,139 @@
+const fs = require('fs');
+const path = require('path');
+const dappsSidebar = require('../docs/dapps/sidebar.js');
+const oevSearchersSidebar = require('../docs/oev-searchers/sidebar.js');
+
+const docsDir = path.join(__dirname, '..', 'docs');
+const staticDir = path.join(__dirname, '..', 'docs', 'public');
+const llmsTxtPath = path.join(staticDir, 'llms.txt');
+const llmsFullTxtPath = path.join(staticDir, 'llms-full.txt');
+
+function getMarkdownFiles(items) {
+  let files = [];
+  for (const item of items) {
+    if (item.link) {
+      files.push(item.link);
+    }
+    if (item.items) {
+      files = files.concat(getMarkdownFiles(item.items));
+    }
+  }
+  return files;
+}
+
+function generateLlmsTxt() {
+  let content = '# Api3 Docs\n\n';
+  content += `> Api3 Docs helps developers build dApps using Api3 data feeds and searchers recapture oracle extractable value (OEV).\n\n`;
+
+  const sidebars = {
+    dapps: dappsSidebar,
+    'oev-searchers': oevSearchersSidebar,
+  };
+
+  for (const key in sidebars) {
+    const sidebar = sidebars[key];
+    if (sidebar) {
+      content += `## ${key}\n\n`;
+      const files = getMarkdownFiles(sidebar);
+      for (const file of files) {
+        const filePath = path.join(docsDir, `${file}.md`);
+        if (fs.existsSync(filePath)) {
+          const fileContent = fs.readFileSync(filePath, 'utf-8');
+          const lines = fileContent.split('\n');
+          let title = path.basename(file, '.md');
+          for (const line of lines) {
+            if (line.startsWith('title: ')) {
+              title = line.substring('title: '.length);
+              break;
+            }
+          }
+          const url = `https://docs.api3.org${file}`;
+          content += `- [${title}](${url.replace(/\/$/, '/index')}.html)\n`;
+        } else {
+          const indexPath = path.join(docsDir, file, 'index.md');
+          if (fs.existsSync(indexPath)) {
+            const fileContent = fs.readFileSync(indexPath, 'utf-8');
+            const lines = fileContent.split('\n');
+            let title = path.basename(file);
+            for (const line of lines) {
+              if (line.startsWith('title: ')) {
+                title = line.substring('title: '.length);
+                break;
+              }
+            }
+            const url = `https://docs.api3.org${file}`;
+            content += `- [${title}](${url.replace(/\/$/, '/index.html')})\n`;
+          }
+        }
+      }
+      content += '\n';
+    }
+  }
+
+  fs.writeFileSync(llmsTxtPath, content);
+  console.log(`Successfully created ${llmsTxtPath}`);
+}
+
+function generateLlmsFullTxt() {
+  const llmsTxtContent = fs.readFileSync(llmsTxtPath, 'utf-8');
+  const links = llmsTxtContent.match(/- \[(.*?)\]\((.*?)\)/g);
+  let fullContent = '';
+  const pageHeader = '<PageHeader/>\n\n';
+
+  if (links) {
+    for (const link of links) {
+      const match = link.match(/- \[(.*?)\]\((.*?)\)/);
+      if (match) {
+        const url = match[2]
+          .replace('https://docs.api3.org', '')
+          .replace('.html', '.md');
+        const filePath = path.join(docsDir, url);
+        if (fs.existsSync(filePath)) {
+          let fileContent = fs.readFileSync(filePath, 'utf-8');
+          const lines = fileContent.split('\n');
+          let pageHeaderValue = '';
+          for (const line of lines) {
+            if (line.startsWith('pageHeader: ')) {
+              pageHeaderValue = line.substring('pageHeader: '.length);
+              break;
+            }
+          }
+
+          const pageHeaderIndex = fileContent.indexOf(pageHeader);
+          if (pageHeaderIndex === -1) {
+            throw new Error(`Could not find PageHeader in ${filePath}`);
+          }
+          fileContent = fileContent.substring(
+            pageHeaderIndex + pageHeader.length
+          );
+          const titleMatch = fileContent.match(/# (.*)/);
+          if (titleMatch && pageHeaderValue) {
+            fileContent = fileContent.replace(
+              /# (.*)/,
+              `# ${titleMatch[1]} (${pageHeaderValue})`
+            );
+          }
+          fullContent += fileContent + '\n\n';
+        }
+      }
+    }
+  }
+
+  fs.writeFileSync(llmsFullTxtPath, fullContent);
+  console.log(`Successfully created ${llmsFullTxtPath}`);
+}
+
+function main() {
+  try {
+    if (!fs.existsSync(staticDir)) {
+      fs.mkdirSync(staticDir, { recursive: true });
+    }
+    generateLlmsTxt();
+    generateLlmsFullTxt();
+  } catch (err) {
+    console.error('Error creating llms files:', err);
+    process.exit(1);
+  }
+}
+
+main();