From 4fdd92b44697412281bab417bf012e93848671d3 Mon Sep 17 00:00:00 2001 From: Farnabaz Date: Mon, 13 Jun 2022 14:03:57 +0200 Subject: [PATCH 1/4] feat(markdown): support multiple themes for code highlighter --- src/module.ts | 5 +- src/runtime/server/api/highlight.ts | 110 +++++++++++++--- src/runtime/server/transformers/shiki.ts | 161 +++++++++++++++-------- src/runtime/types.d.ts | 4 +- 4 files changed, 208 insertions(+), 72 deletions(-) diff --git a/src/module.ts b/src/module.ts index dfbeaa392..ce9cb15c6 100644 --- a/src/module.ts +++ b/src/module.ts @@ -119,7 +119,10 @@ export interface ModuleOptions { /** * Default theme that will be used for highlighting code blocks. */ - theme?: ShikiTheme, + theme?: ShikiTheme | { + default: ShikiTheme + [theme: string]: ShikiTheme + }, /** * Preloaded languages that will be available for highlighting code blocks. */ diff --git a/src/runtime/server/api/highlight.ts b/src/runtime/server/api/highlight.ts index 13ac077c6..25692374c 100644 --- a/src/runtime/server/api/highlight.ts +++ b/src/runtime/server/api/highlight.ts @@ -15,13 +15,26 @@ const resolveLang = (lang: string): Lang | undefined => /** * Resolve Shiki compatible theme from string. */ -const resolveTheme = (theme: string): Theme | undefined => - BUNDLED_THEMES.find(t => t === theme) +const resolveTheme = (theme: string | Record): Record | undefined => { + if (!theme) { + return + } + if (typeof theme === 'string') { + theme = { + default: theme + } + } + + return Object.entries(theme).reduce((acc, [key, value]) => { + acc[key] = BUNDLED_THEMES.find(t => t === value) + return acc + }, {}) +} /** * Resolve Shiki highlighter compatible payload from request body. */ -const resolveBody = (body: Partial): { code: string, lang?: Lang, theme?: Theme } => { +const resolveBody = (body: Partial) => { // Assert body schema if (typeof body.code !== 'string') { throw createError({ statusMessage: 'Bad Request', statusCode: 400, message: 'Missing code key.' }) } @@ -40,7 +53,7 @@ export default defineLazyEventHandler(async () => { // Initialize highlighter with defaults const highlighter = await getHighlighter({ - theme: theme || 'dark-plus', + theme: theme?.default || theme || 'dark-plus', langs: [ ...(preload || ['json', 'js', 'ts', 'css']), 'shell', @@ -60,7 +73,7 @@ export default defineLazyEventHandler(async () => { return async (event): Promise => { const params = await useBody>(event) - const { code, lang, theme } = resolveBody(params) + const { code, lang, theme = { default: highlighter.getTheme() } } = resolveBody(params) // Skip highlight if lang is not supported if (!lang) { @@ -73,21 +86,88 @@ export default defineLazyEventHandler(async () => { } // Load supported theme on-demand - if (theme && !highlighter.getLoadedThemes().includes(theme)) { - await highlighter.loadTheme(theme) - } + await Promise.all( + Object.values(theme).map(async (theme) => { + if (!highlighter.getLoadedThemes().includes(theme)) { + await highlighter.loadTheme(theme) + } + }) + ) // Highlight code - const highlightedCode = highlighter.codeToThemedTokens(code, lang, theme) - - // Clean up to shorten response payload - for (const line of highlightedCode) { - for (const token of line) { - delete token.fontStyle - delete token.explanation + const coloredTokens = Object.entries(theme).map(([key, theme]) => { + const tokens = highlighter.codeToThemedTokens(code, lang, theme) + return { + key, + theme, + tokens } + }) + + const highlightedCode: HighlightThemedToken[][] = [] + for (const line in coloredTokens[0].tokens) { + highlightedCode[line] = coloredTokens.reduce((acc, color) => { + return mergeLines({ + key: coloredTokens[0].key, + tokens: acc + }, { + key: color.key, + tokens: color.tokens[line] + }) + }, coloredTokens[0].tokens[line]) } return highlightedCode } }) + +function mergeLines (line1, line2) { + const mergedTokens = [] + const getColors = (h, i) => typeof h.tokens[i].color === 'string' ? { [h.key]: h.tokens[i].color } : h.tokens[i].color + + const [big, small] = line1.tokens.length > line2.tokens.length ? [line1, line2] : [line2, line1] + let targetToken = 0 + let targetTokenCharIndex = 0 + big.tokens.forEach((t, i) => { + if (targetTokenCharIndex === 0) { + if (t.content === small.tokens[i]?.content) { + mergedTokens.push({ + content: t.content, + color: { + ...getColors(big, i), + ...getColors(small, i) + } + }) + targetToken = i + 1 + return + } + if (t.content === small.tokens[targetToken]?.content) { + mergedTokens.push({ + content: t.content, + color: { + ...getColors(big, i), + ...getColors(small, targetToken) + } + }) + targetToken += 1 + return + } + } + + if (small.tokens[targetToken]?.content?.substring(targetTokenCharIndex, targetTokenCharIndex + t.content.length) === t.content) { + targetTokenCharIndex += t.content.length + mergedTokens.push({ + content: t.content, + color: { + ...getColors(big, i), + ...getColors(small, targetToken) + } + }) + } + if (small.tokens[targetToken]?.content.length <= targetTokenCharIndex) { + targetToken += 1 + targetTokenCharIndex = 0 + } + }) + return mergedTokens +} diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/server/transformers/shiki.ts index e2c698586..baa7443eb 100644 --- a/src/runtime/server/transformers/shiki.ts +++ b/src/runtime/server/transformers/shiki.ts @@ -2,6 +2,8 @@ import { visit } from 'unist-util-visit' import { withBase } from 'ufo' import { useRuntimeConfig } from '#imports' +const highlightConfig = useRuntimeConfig().content.highlight + const withContentBase = (url: string) => { return withBase(url, `/api/${useRuntimeConfig().public.content.base}`) } @@ -10,74 +12,125 @@ export default { name: 'markdown', extensions: ['.md'], transform: async (content) => { - const codeBlocks = [] + const tokenColors: Record = {} + const codeBlocks: any[] = [] + const inlineCodes: any = [] visit( content.body, - (node: any) => node.tag === 'code' && node?.props.code, - (node) => { codeBlocks.push(node) } + (node: any) => (node.tag === 'code' && node?.props.code) || (node.tag === 'code-inline' && (node.props?.lang || node.props?.language)), + (node) => { + if (node.tag === 'code') { + codeBlocks.push(node) + } else if (node.tag === 'code-inline') { + inlineCodes.push(node) + } + } ) + await Promise.all(codeBlocks.map(highlightBlock)) + await Promise.all(inlineCodes.map(highlightInline)) - const inlineCodes = [] - visit( - content.body, - (node: any) => node.tag === 'code-inline' && (node.props?.lang || node.props?.language), - (node) => { inlineCodes.push(node) } - ) + // Inject token colors at the end of the document + if (Object.values(tokenColors).length) { + const colors: string[] = [] + for (const colorClass of Object.values(tokenColors)) { + Object.entries(colorClass.colors).forEach(([variant, color]) => { + if (variant === 'default') { + colors.unshift(`.${colorClass.className}{color:${color}}`) + } else { + colors.push(`.${variant} .${colorClass.className}{color: ${color}}`) + } + }) + } - await Promise.all(inlineCodes.map(highlightInline)) + content.body.children.push({ + type: 'element', + tag: 'style', + children: [{ type: 'text', value: colors.join('') }] + }) + } return content - } -} -const tokenSpan = ({ content, color }) => ({ - type: 'element', - tag: 'span', - props: { style: { color } }, - children: [{ type: 'text', value: content }] -}) - -const highlightInline = async (node) => { - const code = node.children[0].value - - // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { - method: 'POST', - body: { - code, - lang: node.props.lang || node.props.language + /** + * Highlight inline code + */ + async function highlightInline (node) { + const code = node.children[0].value + + // Fetch highlighted tokens + const lines = await $fetch(withContentBase('highlight'), { + method: 'POST', + body: { + code, + lang: node.props.lang || node.props.language, + theme: highlightConfig.theme + } + }) + + // Generate highlighted children + node.children = lines[0].map(tokenSpan) + + node.props = node.props || {} + node.props.class = 'colored' + + return node } - }) - // Generate highlighted children - node.children = lines[0].map(tokenSpan) + /** + * Highlight a code block + */ + async function highlightBlock (node) { + const { code, language: lang, highlights = [] } = node.props - node.props = node.props || {} - node.props.class = 'colored' + // Fetch highlighted tokens + const lines = await $fetch(withContentBase('highlight'), { + method: 'POST', + body: { + code, + lang, + theme: { + default: highlightConfig.theme, + dark: 'github-light' + } + } + }) - return node -} + // Generate highlighted children + const innerCodeNode = node.children[0].children[0] + innerCodeNode.children = lines.map((line, lineIndex) => ({ + type: 'element', + tag: 'span', + props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() }, + children: line.map(tokenSpan) + })) + return node + } -const highlightBlock = async (node) => { - const { code, language: lang, highlights = [] } = node.props + function getColorProps (token) { + if (!token.color) { + return {} + } + if (typeof token.color === 'string') { + return { style: { color: token.color } } + } + const key = Object.values(token.color).join('') + if (!tokenColors[key]) { + tokenColors[key] = { + colors: token.color, + className: 'ct-' + Math.random().toString(16).substring(2, 8) // hash(key) + } + } + return { class: tokenColors[key].className } + } - // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { - method: 'POST', - body: { - code, - lang + function tokenSpan (token) { + return { + type: 'element', + tag: 'span', + props: getColorProps(token), + children: [{ type: 'text', value: token.content }] + } } - }) - - // Generate highlighted children - const innerCodeNode = node.children[0].children[0] - innerCodeNode.children = lines.map((line, lineIndex) => ({ - type: 'element', - tag: 'span', - props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() }, - children: line.map(tokenSpan) - })) - return node + } } diff --git a/src/runtime/types.d.ts b/src/runtime/types.d.ts index bc51f19e1..47a9b6dad 100644 --- a/src/runtime/types.d.ts +++ b/src/runtime/types.d.ts @@ -283,10 +283,10 @@ export interface NavItem { export interface HighlightParams { code: string lang: string - theme: Theme + theme: Theme | Record } export interface HighlightThemedToken { content: string - color?: string + color?: string | Record } From b0230e1da35b3527197678197af9c0c04caf51ef Mon Sep 17 00:00:00 2001 From: Farnabaz Date: Mon, 13 Jun 2022 14:30:31 +0200 Subject: [PATCH 2/4] chore: remove test params --- src/runtime/server/transformers/shiki.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/server/transformers/shiki.ts index baa7443eb..cb2b0ad44 100644 --- a/src/runtime/server/transformers/shiki.ts +++ b/src/runtime/server/transformers/shiki.ts @@ -89,10 +89,7 @@ export default { body: { code, lang, - theme: { - default: highlightConfig.theme, - dark: 'github-light' - } + theme: highlightConfig.theme } }) From a92cf1602638689cde5c2fa6db773a8179f45e5c Mon Sep 17 00:00:00 2001 From: Farnabaz Date: Mon, 13 Jun 2022 15:16:12 +0200 Subject: [PATCH 3/4] test: add test --- src/runtime/server/transformers/shiki.ts | 2 +- test/basic.test.ts | 3 ++ test/features/highlighter.ts | 62 ++++++++++++++++++++++++ test/fixtures/basic/nuxt.config.ts | 6 +++ 4 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 test/features/highlighter.ts diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/server/transformers/shiki.ts index cb2b0ad44..0fb2417d1 100644 --- a/src/runtime/server/transformers/shiki.ts +++ b/src/runtime/server/transformers/shiki.ts @@ -38,7 +38,7 @@ export default { if (variant === 'default') { colors.unshift(`.${colorClass.className}{color:${color}}`) } else { - colors.push(`.${variant} .${colorClass.className}{color: ${color}}`) + colors.push(`.${variant} .${colorClass.className}{color:${color}}`) } }) } diff --git a/test/basic.test.ts b/test/basic.test.ts index da14a5a3a..324f223bc 100644 --- a/test/basic.test.ts +++ b/test/basic.test.ts @@ -14,6 +14,7 @@ import { testMarkdownParserExcerpt } from './features/parser-markdown-excerpt' import { testParserHooks } from './features/parser-hooks' import { testModuleOption } from './features/module-options' import { testContentQuery } from './features/content-query' +import { testHighlighter } from './features/highlighter' describe('fixtures:basic', async () => { await setup({ @@ -129,4 +130,6 @@ describe('fixtures:basic', async () => { testParserHooks() testModuleOption() + + testHighlighter() }) diff --git a/test/features/highlighter.ts b/test/features/highlighter.ts new file mode 100644 index 000000000..9755b136c --- /dev/null +++ b/test/features/highlighter.ts @@ -0,0 +1,62 @@ +import { describe, test, expect, assert } from 'vitest' +import { $fetch } from '@nuxt/test-utils' + +const content = [ + '```ts', + 'const a: number = 1', + '```' +].join('\n') + +export const testHighlighter = () => { + describe('highlighter', () => { + test('themed', async () => { + const parsed = await $fetch('/api/parse', { + method: 'POST', + body: { + id: 'content:index.md', + content + } + }) + + expect(parsed).toHaveProperty('_id') + assert(parsed._id === 'content:index.md') + + const styleElement = parsed.body.children.pop() + expect(styleElement.tag).toBe('style') + const style = styleElement.children[0].value + console.log(style) + + const code = parsed.body.children[0].children[0].children[0].children[0].children + + expect(style).toContain(`.${code[0].props.class}{color:#CF222E}`) + expect(style).toContain(`.dark .${code[0].props.class}{color:#FF7B72}`) + + expect(style).toContain(`.${code[1].props.class}{color:#24292F}`) + expect(style).toContain(`.dark .${code[1].props.class}{color:#C9D1D9}`) + + expect(style).toContain(`.${code[2].props.class}{color:#0550AE}`) + expect(style).toContain(`.dark .${code[2].props.class}{color:#79C0FF}`) + + expect(style).toContain(`.${code[3].props.class}{color:#CF222E}`) + expect(style).toContain(`.dark .${code[3].props.class}{color:#FF7B72}`) + + expect(style).toContain(`.${code[4].props.class}{color:#24292F}`) + expect(style).toContain(`.dark .${code[4].props.class}{color:#C9D1D9}`) + + expect(style).toContain(`.${code[5].props.class}{color:#0550AE}`) + expect(style).toContain(`.dark .${code[5].props.class}{color:#79C0FF}`) + + expect(style).toContain(`.${code[6].props.class}{color:#24292F}`) + expect(style).toContain(`.dark .${code[6].props.class}{color:#C9D1D9}`) + + expect(style).toContain(`.${code[7].props.class}{color:#CF222E}`) + expect(style).toContain(`.dark .${code[7].props.class}{color:#FF7B72}`) + + expect(style).toContain(`.${code[8].props.class}{color:#24292F}`) + expect(style).toContain(`.dark .${code[8].props.class}{color:#C9D1D9}`) + + expect(style).toContain(`.${code[9].props.class}{color:#0550AE}`) + expect(style).toContain(`.dark .${code[9].props.class}{color:#79C0FF}`) + }) + }) +} diff --git a/test/fixtures/basic/nuxt.config.ts b/test/fixtures/basic/nuxt.config.ts index 81b46bcde..599054d09 100644 --- a/test/fixtures/basic/nuxt.config.ts +++ b/test/fixtures/basic/nuxt.config.ts @@ -30,6 +30,12 @@ export default defineNuxtConfig({ navigation: { fields: ['icon'] }, + highlight: { + theme: { + default: 'github-light', + dark: 'github-dark' + } + }, markdown: { // Object syntax can be used to override default options remarkPlugins: { From cacc7e010f08e9738cb9d59e3e800d128b660ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ya=C3=ABl=20Guilloux?= Date: Tue, 14 Jun 2022 12:55:44 +0200 Subject: [PATCH 4/4] docs(highlight): document highlight.theme option --- docs/content/4.api/3.configuration.md | 30 ++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/content/4.api/3.configuration.md b/docs/content/4.api/3.configuration.md index 2908b95da..aea86a0d8 100644 --- a/docs/content/4.api/3.configuration.md +++ b/docs/content/4.api/3.configuration.md @@ -181,9 +181,37 @@ Nuxt Content uses [Shiki](https://github.com/shikijs/shiki) to provide syntax hi | Option | Default | Description | | ----------------- | :--------: | :-------- | -| `theme` | `ShikiTheme` | The [color theme](https://github.com/shikijs/shiki/blob/main/docs/themes.md) to use | +| `theme` | `ShikiTheme` or `Record` | The [color theme](https://github.com/shikijs/shiki/blob/main/docs/themes.md) to use. | | `preload` | `ShikiLang[]` | The [preloaded languages](https://github.com/shikijs/shiki/blob/main/docs/languages.md) available for highlighting. | +#### `highlight.theme` + +Theme can be specified by a single string but also supports an object with multiple themes. + +This option is compatible with [Color Mode module](https://color-mode.nuxtjs.org/). + +If you are using multiple themes, it's recommended to always have a `default` theme specified. + +```ts +export default defineNuxtConfig({ + content: { + highlight: { + // Theme used in all color schemes. + theme: 'github-light' + // OR + theme: { + // Default theme (same as single string) + default: 'github-light', + // Theme used if `html.dark` + dark: 'github-dark' + // Theme used if `html.sepia` + sepia: 'monokai' + } + } + } +}) +``` + ## `yaml` - Type: `false | Object`{lang=ts}