feat(shiki): expose highlighting utils

src/module.ts

@@ -299,7 +299,7 @@ export default defineNuxtModule<ModuleOptions>({
       config.optimizeDeps = config.optimizeDeps || {}
       config.optimizeDeps.include = config.optimizeDeps.include || []
       config.optimizeDeps.include.push(
-        'html-tags'
+        'html-tags', 'slugify'
       )
     })
 

src/runtime/composables/client-db.ts

@@ -73,7 +73,7 @@ async function initContentDatabase () {
   const _contentDatabase = createDB(contentStorage)
   const integrity = await _contentDatabase.storage.getItem('integrity')
   if (content.integrity !== +(integrity || 0)) {
-    const { contents, navigation } = await $fetch(withContentBase(`cache.${content.integrity}.json`)) as any
+    const { contents, navigation } = await $fetch(withContentBase(content.integrity ? `cache.${content.integrity}.json` : 'cache.json')) as any
 
     await Promise.all(
       contents.map((content: ParsedContent) => _contentDatabase.storage.setItem(`cache:${content._id}`, content))

src/runtime/transformers/shiki/highlighter.ts

@@ -1,12 +1,12 @@
-import { getHighlighter, BUNDLED_LANGUAGES, BUNDLED_THEMES, Lang, Theme, Highlighter } from 'shiki-es'
+import { getHighlighter, BUNDLED_LANGUAGES, BUNDLED_THEMES, Lang, Theme as ShikiTheme, Highlighter } from 'shiki-es'
 import consola from 'consola'
 import type { ModuleOptions } from '../../../module'
-import { HighlightThemedToken } from '../../types'
 import { createSingleton } from '../utils'
 import mdcTMLanguage from './languages/mdc.tmLanguage.json'
+import type { MarkdownNode, HighlighterOptions, Theme, HighlightThemedToken, HighlightThemedTokenLine, TokenColorMap } from './types'
 
 // Re-create logger locally as utils cannot be imported from here
-export const logger = consola.withScope('@nuxt/content')
+const logger = consola.withScope('@nuxt/content')
 
 /**
  * Resolve Shiki compatible lang from string.
@@ -19,7 +19,7 @@ const resolveLang = (lang: string): Lang =>
 /**
  * Resolve Shiki compatible theme from string.
  */
-const resolveTheme = (theme: string | Record<string, string>): Record<string, Theme> | undefined => {
+const resolveTheme = (theme: string | Record<string, string>): Record<string, ShikiTheme> | undefined => {
   if (!theme) {
     return
   }
@@ -32,7 +32,7 @@ const resolveTheme = (theme: string | Record<string, string>): Record<string, Th
   return Object.entries(theme).reduce((acc, [key, value]) => {
     acc[key] = BUNDLED_THEMES.find(t => t === value)!
     return acc
-  }, {} as Record<string, Theme>)
+  }, {} as Record<string, ShikiTheme>)
 }
 
 export const useShikiHighlighter = createSingleton((opts?: Exclude<ModuleOptions['highlight'], false>) => {
@@ -56,6 +56,7 @@ export const useShikiHighlighter = createSingleton((opts?: Exclude<ModuleOptions
           'html',
           'md',
           'yaml',
+          'vue',
           {
             id: 'md',
             scopeName: 'text.markdown.mdc',
@@ -68,13 +69,14 @@ export const useShikiHighlighter = createSingleton((opts?: Exclude<ModuleOptions
     }
     return promise
   }
-  const getHighlightedTokens = async (code: string, lang: Lang, theme: Theme | Record<string, Theme>) => {
+
+  const getHighlightedTokens = async (code: string, lang: Lang, theme: Theme) => {
     const highlighter = await getShikiHighlighter()
     // Remove trailing carriage returns
     code = code.replace(/\n+$/, '')
     // Resolve lang & theme (i.e check if shiki supports them)
     lang = resolveLang(lang || '')
-    theme = resolveTheme(theme || '') || { default: highlighter.getTheme() as any as Theme }
+    theme = resolveTheme(theme || '') || { default: highlighter.getTheme() as any as ShikiTheme }
 
     // Skip highlight if lang is not supported
     if (!lang) {
@@ -130,16 +132,85 @@ export const useShikiHighlighter = createSingleton((opts?: Exclude<ModuleOptions
     return highlightedCode
   }
 
+  const getHighlightedAST = async (code: string, lang: Lang, theme: Theme, opts?: Partial<HighlighterOptions>): Promise<Array<MarkdownNode>> => {
+    const lines = await getHighlightedTokens(code, lang, theme)
+    const { highlights = [], colorMap = {} } = opts || {}
+
+    return lines.map((line, lineIndex) => ({
+      type: 'element',
+      tag: 'span',
+      props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() },
+      children: line.map(tokenSpan)
+    }))
+
+    function getColorProps (token: { color?: string | object }) {
+      if (!token.color) {
+        return {}
+      }
+      if (typeof token.color === 'string') {
+        return { style: { color: token.color } }
+      }
+      const key = Object.values(token.color).join('')
+      if (!colorMap[key]) {
+        colorMap[key] = {
+          colors: token.color,
+          className: 'ct-' + Math.random().toString(16).substring(2, 8) // hash(key)
+        }
+      }
+      return { class: colorMap[key].className }
+    }
+
+    function tokenSpan (token: { content: string, color?: string | object }) {
+      return {
+        type: 'element',
+        tag: 'span',
+        props: getColorProps(token),
+        children: [{ type: 'text', value: token.content }]
+      }
+    }
+  }
+
+  const getHighlightedCode = async (code: string, lang: Lang, theme: Theme, opts?: Partial<HighlighterOptions>) => {
+    const colorMap = opts?.colorMap || {}
+    const highlights = opts?.highlights || []
+    const ast = await getHighlightedAST(code, lang, theme, { colorMap, highlights })
+
+    function renderNode (node: any) {
+      if (node.type === 'text') {
+        return node.value
+      }
+      const children = node.children.map(renderNode).join('')
+      return `<${node.tag} class="${node.props.class}">${children}</${node.tag}>`
+    }
+
+    return {
+      code: ast.map(renderNode).join(''),
+      styles: generateStyles(colorMap)
+    }
+  }
+
+  const generateStyles = (colorMap: TokenColorMap) => {
+    const colors: string[] = []
+    for (const colorClass of Object.values(colorMap)) {
+      Object.entries(colorClass.colors).forEach(([variant, color]) => {
+        if (variant === 'default') {
+          colors.unshift(`.${colorClass.className}{color:${color}}`)
+        } else {
+          colors.push(`.${variant} .${colorClass.className}{color:${color}}`)
+        }
+      })
+    }
+    return colors.join('\n')
+  }
+
   return {
-    getHighlightedTokens
+    getHighlightedTokens,
+    getHighlightedAST,
+    getHighlightedCode,
+    generateStyles
   }
 })
 
-interface HighlightThemedTokenLine {
-  key: string
-  tokens: HighlightThemedToken[]
-}
-
 function mergeLines (line1: HighlightThemedTokenLine, line2: HighlightThemedTokenLine) {
   const mergedTokens: HighlightThemedToken[] = []
   const getColors = (h: HighlightThemedTokenLine, i: number) => typeof h.tokens[i].color === 'string' ? { [h.key]: h.tokens[i].color } : h.tokens[i].color as object

src/runtime/transformers/shiki/index.ts

@@ -1,115 +1,2 @@
-import { visit } from 'unist-util-visit'
-import { MarkdownNode } from '../../types'
-import { defineTransformer } from '../utils'
-import { useShikiHighlighter } from './highlighter'
-
-export default defineTransformer({
-  name: 'highlight',
-  extensions: ['.md'],
-  transform: async (content, options = {}) => {
-    const shikiHighlighter = useShikiHighlighter(options)
-    const tokenColors: Record<string, {colors: any, className: string}> = {}
-    const codeBlocks: any[] = []
-    const inlineCodes: any = []
-    visit(
-      content.body,
-      (node: any) => (node.tag === 'code' && node?.props.code) || (node.tag === 'code-inline' && (node.props?.lang || node.props?.language)),
-      (node) => {
-        if (node.tag === 'code') {
-          codeBlocks.push(node)
-        } else if (node.tag === 'code-inline') {
-          inlineCodes.push(node)
-        }
-      }
-    )
-
-    await Promise.all(codeBlocks.map(highlightBlock))
-    await Promise.all(inlineCodes.map(highlightInline))
-
-    // Inject token colors at the end of the document
-    if (Object.values(tokenColors).length) {
-      const colors: string[] = []
-      for (const colorClass of Object.values(tokenColors)) {
-        Object.entries(colorClass.colors).forEach(([variant, color]) => {
-          if (variant === 'default') {
-            colors.unshift(`.${colorClass.className}{color:${color}}`)
-          } else {
-            colors.push(`.${variant} .${colorClass.className}{color:${color}}`)
-          }
-        })
-      }
-
-      content.body.children.push({
-        type: 'element',
-        tag: 'style',
-        children: [{ type: 'text', value: colors.join('') }]
-      })
-    }
-
-    return content
-
-    /**
-     * Highlight inline code
-     */
-    async function highlightInline (node: MarkdownNode) {
-      const code = node.children![0].value!
-
-      // Fetch highlighted tokens
-      const lines = await shikiHighlighter.getHighlightedTokens(code, node.props!.lang || node.props!.language, options.theme)
-
-      // Generate highlighted children
-      node.children = lines[0].map(tokenSpan)
-
-      node.props = node.props || {}
-      node.props.class = 'colored'
-
-      return node
-    }
-
-    /**
-     * Highlight a code block
-     */
-    async function highlightBlock (node: MarkdownNode) {
-      const { code, language: lang, highlights = [] } = node.props!
-
-      // Fetch highlighted tokens
-      const lines = await shikiHighlighter.getHighlightedTokens(code, lang, options.theme)
-
-      // Generate highlighted children
-      const innerCodeNode = node.children![0].children![0]
-      innerCodeNode.children = lines.map((line, lineIndex) => ({
-        type: 'element',
-        tag: 'span',
-        props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() },
-        children: line.map(tokenSpan)
-      }))
-      return node
-    }
-
-    function getColorProps (token: { color?: string | object }) {
-      if (!token.color) {
-        return {}
-      }
-      if (typeof token.color === 'string') {
-        return { style: { color: token.color } }
-      }
-      const key = Object.values(token.color).join('')
-      if (!tokenColors[key]) {
-        tokenColors[key] = {
-          colors: token.color,
-          className: 'ct-' + Math.random().toString(16).substring(2, 8) // hash(key)
-        }
-      }
-      return { class: tokenColors[key].className }
-    }
-
-    function tokenSpan (token: { content: string, color?: string | object }) {
-      return {
-        type: 'element',
-        tag: 'span',
-        props: getColorProps(token),
-        children: [{ type: 'text', value: token.content }]
-      }
-    }
-  }
-})
+export { default } from './shiki'
+export * from './highlighter'

src/runtime/transformers/shiki/shiki.ts

@@ -0,0 +1,68 @@
+import { visit } from 'unist-util-visit'
+import { defineTransformer } from '../utils'
+import { useShikiHighlighter } from './highlighter'
+import type { TokenColorMap, MarkdownNode } from './types'
+
+export default defineTransformer({
+  name: 'highlight',
+  extensions: ['.md'],
+  transform: async (content, options = {}) => {
+    const shikiHighlighter = useShikiHighlighter(options)
+    const colorMap: TokenColorMap = {}
+    const codeBlocks: any[] = []
+    const inlineCodes: any = []
+    visit(
+      content.body,
+      (node: any) => (node.tag === 'code' && node?.props.code) || (node.tag === 'code-inline' && (node.props?.lang || node.props?.language)),
+      (node) => {
+        if (node.tag === 'code') {
+          codeBlocks.push(node)
+        } else if (node.tag === 'code-inline') {
+          inlineCodes.push(node)
+        }
+      }
+    )
+
+    await Promise.all(codeBlocks.map(highlightBlock))
+    await Promise.all(inlineCodes.map(highlightInline))
+
+    // Inject token colors at the end of the document
+    if (Object.values(colorMap).length) {
+      content.body.children.push({
+        type: 'element',
+        tag: 'style',
+        children: [{ type: 'text', value: shikiHighlighter.generateStyles(colorMap) }]
+      })
+    }
+
+    return content
+
+    /**
+     * Highlight inline code
+     */
+    async function highlightInline (node: MarkdownNode) {
+      const code = node.children![0].value!
+
+      // Fetch highlighted tokens
+      const lines = await shikiHighlighter.getHighlightedAST(code, node.props!.lang || node.props!.language, options.theme, { colorMap })
+
+      // Generate highlighted children
+      node.children = lines[0].children
+      node.props = Object.assign(node.props || {}, { class: 'colored' })
+
+      return node
+    }
+
+    /**
+     * Highlight a code block
+     */
+    async function highlightBlock (node: MarkdownNode) {
+      const { code, language: lang, highlights = [] } = node.props!
+
+      const innerCodeNode = node.children![0].children![0]
+      innerCodeNode.children = await shikiHighlighter.getHighlightedAST(code, lang, options.theme, { colorMap, highlights })
+
+      return node
+    }
+  }
+})

src/runtime/transformers/shiki/types.d.ts

@@ -0,0 +1,27 @@
+import { Theme as ShikiTheme } from 'shiki-es'
+export type { MarkdownNode } from '../../types'
+
+export type Theme = ShikiTheme | Record<string, ShikiTheme>
+
+export type TokenColorMap = Record<string, {colors: any, className: string}>
+
+export interface HighlightParams {
+  code: string
+  lang: string
+  theme: Theme
+}
+
+export interface HighlighterOptions {
+  colorMap: TokenColorMap
+  highlights: Array<number>
+}
+
+export interface HighlightThemedToken {
+  content: string
+  color?: string | Record<string, string>
+}
+
+export interface HighlightThemedTokenLine {
+  key: string
+  tokens: HighlightThemedToken[]
+}

src/runtime/types.d.ts

@@ -501,15 +501,3 @@ export interface NavItem {
 
   [key: string]: any
 }
-
-// Highlight
-export interface HighlightParams {
-  code: string
-  lang: string
-  theme: Theme | Record<string, Theme>
-}
-
-export interface HighlightThemedToken {
-  content: string
-  color?: string | Record<string, string>
-}