From db14372193c00191110e92b24274e065f2d84495 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 14:59:21 +0200 Subject: [PATCH 01/10] refactor: extract nitro logic from transformers --- package.json | 4 + src/module.ts | 12 +- src/runtime/server/index.ts | 3 +- src/runtime/server/storage.ts | 31 +++- src/runtime/server/transformers/csv.ts | 20 --- src/runtime/server/transformers/index.ts | 37 ----- src/runtime/server/transformers/shiki.ts | 133 ------------------ src/runtime/transformers/csv.ts | 18 +++ src/runtime/transformers/index.ts | 62 ++++++++ src/runtime/{server => }/transformers/json.ts | 7 +- .../{server => }/transformers/markdown.ts | 16 +-- .../{server => }/transformers/path-meta.ts | 15 +- src/runtime/transformers/types.d.ts | 16 +++ src/runtime/transformers/utils.ts | 5 + src/runtime/{server => }/transformers/yaml.ts | 5 +- src/runtime/types.d.ts | 7 - 16 files changed, 162 insertions(+), 229 deletions(-) delete mode 100644 src/runtime/server/transformers/csv.ts delete mode 100644 src/runtime/server/transformers/index.ts delete mode 100644 src/runtime/server/transformers/shiki.ts create mode 100644 src/runtime/transformers/csv.ts create mode 100644 src/runtime/transformers/index.ts rename src/runtime/{server => }/transformers/json.ts (88%) rename src/runtime/{server => }/transformers/markdown.ts (68%) rename src/runtime/{server => }/transformers/path-meta.ts (88%) create mode 100644 src/runtime/transformers/types.d.ts create mode 100644 src/runtime/transformers/utils.ts rename src/runtime/{server => }/transformers/yaml.ts (87%) diff --git a/package.json b/package.json index b78d2eabd..f88f3039f 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,10 @@ ".": { "import": "./dist/module.mjs", "require": "./dist/module.cjs" + }, + "./transformers": { + "import": "./dist/runtime/transformers/index.mjs", + "require": "./dist/runtime/transformers/index.cjs" } }, "main": "./dist/module.cjs", diff --git a/src/module.ts b/src/module.ts index e1bb82031..19829decd 100644 --- a/src/module.ts +++ b/src/module.ts @@ -227,11 +227,11 @@ export default defineNuxtModule({ const contentContext: ContentContext = { transformers: [ // Register internal content plugins - resolveRuntimeModule('./server/transformers/markdown'), - resolveRuntimeModule('./server/transformers/yaml'), - resolveRuntimeModule('./server/transformers/json'), - resolveRuntimeModule('./server/transformers/csv'), - resolveRuntimeModule('./server/transformers/path-meta') + // resolveRuntimeModule('./server/transformers/markdown'), + // resolveRuntimeModule('./server/transformers/yaml'), + // resolveRuntimeModule('./server/transformers/json'), + // resolveRuntimeModule('./server/transformers/csv'), + // resolveRuntimeModule('./server/transformers/path-meta') ], ...options } @@ -313,7 +313,7 @@ export default defineNuxtModule({ nitroConfig.virtual['#content/virtual/transformers'] = [ // TODO: remove kit usage templateUtils.importSources(contentContext.transformers), - `const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`, + `export const transformers = [${contentContext.transformers.map(templateUtils.importName).join(', ')}]`, 'export const getParser = (ext) => transformers.find(p => ext.match(new RegExp(p.extensions.join("|"), "i")) && p.parse)', 'export const getTransformers = (ext) => transformers.filter(p => ext.match(new RegExp(p.extensions.join("|"), "i")) && p.transform)', 'export default () => {}' diff --git a/src/runtime/server/index.ts b/src/runtime/server/index.ts index bfc3d50b6..a5e0ad631 100644 --- a/src/runtime/server/index.ts +++ b/src/runtime/server/index.ts @@ -1,2 +1 @@ -export { serverQueryContent } from './storage' -export { parseContent } from './transformers' +export { serverQueryContent, parseContent } from './storage' diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index 0d4363e44..006c59ab8 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -5,10 +5,11 @@ import type { CompatibilityEvent } from 'h3' import type { QueryBuilderParams, ParsedContent, QueryBuilder } from '../types' import { createQuery } from '../query/query' import { createPipelineFetcher } from '../query/match/pipeline' -import { parseContent } from './transformers' +import { transformContent } from '../transformers' import { getPreview, isPreview } from './preview' // eslint-disable-next-line import/named -import { useRuntimeConfig, useStorage } from '#imports' +import { useNitroApp, useRuntimeConfig, useStorage } from '#imports' +import { transformers } from '#content/virtual/transformers' export const sourceStorage = prefixStorage(useStorage(), 'content:source') export const cacheStorage = prefixStorage(useStorage(), 'cache:content') @@ -127,13 +128,37 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise return { _id: contentId, body: null } } - const parsed = await parseContent(contentId, body as string) + const parsed = await parseContent(contentId, body as string) as ParsedContent await cacheParsedStorage.setItem(id, { parsed, hash }).catch(() => {}) return parsed } +/** + * Parse content file using registered plugins + */ +export async function parseContent (id: string, content: string) { + const nitroApp = useNitroApp() + const { markdown, csv, yaml } = useRuntimeConfig().content + + // Call hook before parsing the file + const file = { _id: id, body: content } + await nitroApp.hooks.callHook('content:file:beforeParse', file) + + const result = transformContent(id, file.body, { + transformers, + markdown, + csv, + yaml + }) + + // Call hook after parsing the file + await nitroApp.hooks.callHook('content:file:afterParse', result) + + return result +} + export const createServerQueryFetch = (event: CompatibilityEvent, path?: string) => (query: QueryBuilder) => { if (path) { if (query.params().first) { diff --git a/src/runtime/server/transformers/csv.ts b/src/runtime/server/transformers/csv.ts deleted file mode 100644 index 949d887b2..000000000 --- a/src/runtime/server/transformers/csv.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { useRuntimeConfig } from '#imports' - -export default { - name: 'csv', - extensions: ['.csv'], - parse: async (_id, content) => { - const config = { ...useRuntimeConfig().content?.csv || {} } - - const csvToJson: any = await import('csvtojson').then(m => m.default || m) - - const parsed = await csvToJson({ output: 'json', ...config }) - .fromString(content) - - return { - _id, - _type: 'csv', - body: parsed - } - } -} diff --git a/src/runtime/server/transformers/index.ts b/src/runtime/server/transformers/index.ts deleted file mode 100644 index c4fbbd2fc..000000000 --- a/src/runtime/server/transformers/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { extname } from 'pathe' -import type { ParsedContent, ContentTransformer } from '../../types' -import { getParser, getTransformers } from '#content/virtual/transformers' -// eslint-disable-next-line import/named -import { useNitroApp } from '#imports' -/** - * Parse content file using registered plugins - */ -export async function parseContent (id: string, content: string) { - const nitroApp = useNitroApp() - - // Call hook before parsing the file - const file = { _id: id, body: content } - await nitroApp.hooks.callHook('content:file:beforeParse', file) - - const ext = extname(id) - const plugin: ContentTransformer = getParser(ext) - if (!plugin) { - // eslint-disable-next-line no-console - console.warn(`${ext} files are not supported, "${id}" falling back to raw content`) - return file - } - - const parsed: ParsedContent = await plugin.parse!(file._id, file.body) - - const transformers = getTransformers(ext) - const result = await transformers.reduce(async (prev, cur) => { - const next = (await prev) || parsed - - return cur.transform!(next) - }, Promise.resolve(parsed)) - - // Call hook after parsing the file - await nitroApp.hooks.callHook('content:file:afterParse', result) - - return result -} diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/server/transformers/shiki.ts deleted file mode 100644 index 0fb2417d1..000000000 --- a/src/runtime/server/transformers/shiki.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { visit } from 'unist-util-visit' -import { withBase } from 'ufo' -import { useRuntimeConfig } from '#imports' - -const highlightConfig = useRuntimeConfig().content.highlight - -const withContentBase = (url: string) => { - return withBase(url, `/api/${useRuntimeConfig().public.content.base}`) -} - -export default { - name: 'markdown', - extensions: ['.md'], - transform: async (content) => { - const tokenColors: Record = {} - const codeBlocks: any[] = [] - const inlineCodes: any = [] - visit( - content.body, - (node: any) => (node.tag === 'code' && node?.props.code) || (node.tag === 'code-inline' && (node.props?.lang || node.props?.language)), - (node) => { - if (node.tag === 'code') { - codeBlocks.push(node) - } else if (node.tag === 'code-inline') { - inlineCodes.push(node) - } - } - ) - - await Promise.all(codeBlocks.map(highlightBlock)) - await Promise.all(inlineCodes.map(highlightInline)) - - // Inject token colors at the end of the document - if (Object.values(tokenColors).length) { - const colors: string[] = [] - for (const colorClass of Object.values(tokenColors)) { - Object.entries(colorClass.colors).forEach(([variant, color]) => { - if (variant === 'default') { - colors.unshift(`.${colorClass.className}{color:${color}}`) - } else { - colors.push(`.${variant} .${colorClass.className}{color:${color}}`) - } - }) - } - - content.body.children.push({ - type: 'element', - tag: 'style', - children: [{ type: 'text', value: colors.join('') }] - }) - } - - return content - - /** - * Highlight inline code - */ - async function highlightInline (node) { - const code = node.children[0].value - - // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { - method: 'POST', - body: { - code, - lang: node.props.lang || node.props.language, - theme: highlightConfig.theme - } - }) - - // Generate highlighted children - node.children = lines[0].map(tokenSpan) - - node.props = node.props || {} - node.props.class = 'colored' - - return node - } - - /** - * Highlight a code block - */ - async function highlightBlock (node) { - const { code, language: lang, highlights = [] } = node.props - - // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { - method: 'POST', - body: { - code, - lang, - theme: highlightConfig.theme - } - }) - - // Generate highlighted children - const innerCodeNode = node.children[0].children[0] - innerCodeNode.children = lines.map((line, lineIndex) => ({ - type: 'element', - tag: 'span', - props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() }, - children: line.map(tokenSpan) - })) - return node - } - - function getColorProps (token) { - if (!token.color) { - return {} - } - if (typeof token.color === 'string') { - return { style: { color: token.color } } - } - const key = Object.values(token.color).join('') - if (!tokenColors[key]) { - tokenColors[key] = { - colors: token.color, - className: 'ct-' + Math.random().toString(16).substring(2, 8) // hash(key) - } - } - return { class: tokenColors[key].className } - } - - function tokenSpan (token) { - return { - type: 'element', - tag: 'span', - props: getColorProps(token), - children: [{ type: 'text', value: token.content }] - } - } - } -} diff --git a/src/runtime/transformers/csv.ts b/src/runtime/transformers/csv.ts new file mode 100644 index 000000000..2e0906a60 --- /dev/null +++ b/src/runtime/transformers/csv.ts @@ -0,0 +1,18 @@ +import { defineTransformer } from './utils' + +export default defineTransformer({ + name: 'csv', + extensions: ['.csv'], + parse: async (_id, content, options = {}) => { + const csvToJson: any = await import('csvtojson').then(m => m.default || m) + + const parsed = await csvToJson({ output: 'json', ...options }) + .fromString(content) + + return { + _id, + _type: 'csv', + body: parsed + } + } +}) diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts new file mode 100644 index 000000000..d6c98d2f7 --- /dev/null +++ b/src/runtime/transformers/index.ts @@ -0,0 +1,62 @@ +import { extname } from 'pathe' +import { pascalCase } from 'scule' +import type { ContentTransformer, TransformContentOptions } from './types' +import csv from './csv' +import markdown from './markdown' +import yaml from './yaml' +import pathMeta from './path-meta' +import json from './json' + +const TRANSFORMERS = [ + csv, + markdown, + json, + yaml, + pathMeta +] + +function getParser (ext, additionalTransformers: ContentTransformer[] = []): ContentTransformer { + const parser = additionalTransformers.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) + if (parser) { + return parser + } + return TRANSFORMERS.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) +} + +function getTransformers (ext, additionalTransformers: ContentTransformer[] = []) { + const transformr = additionalTransformers.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) + if (transformr) { + return transformr + } + return TRANSFORMERS.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) +} + +/** + * Parse content file using registered plugins + */ +export async function transformContent (id, content, options: TransformContentOptions = {}) { + const { transformers = [] } = options + // Call hook before parsing the file + const file = { _id: id, body: content } + + const ext = extname(id) + const parser: ContentTransformer = getParser(ext, transformers) + if (!parser) { + // eslint-disable-next-line no-console + console.warn(`${ext} files are not supported, "${id}" falling back to raw content`) + return file + } + + const parserOptions = options[pascalCase(parser.name)] || {} + const parsed = await parser.parse!(file._id, file.body, parserOptions) + + const matchedTransformers = getTransformers(ext, transformers) + const result = await matchedTransformers.reduce(async (prev, cur) => { + const next = (await prev) || parsed + + const transformOptions = options[pascalCase(cur.name)] || {} + return cur.transform!(next, transformOptions) + }, Promise.resolve(parsed)) + + return result +} diff --git a/src/runtime/server/transformers/json.ts b/src/runtime/transformers/json.ts similarity index 88% rename from src/runtime/server/transformers/json.ts rename to src/runtime/transformers/json.ts index fd5139630..6e87d9258 100644 --- a/src/runtime/server/transformers/json.ts +++ b/src/runtime/transformers/json.ts @@ -1,10 +1,11 @@ import destr from 'destr' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'Json', extensions: ['.json', '.json5'], parse: async (_id, content) => { - let parsed = content + let parsed if (typeof content === 'string') { if (_id.endsWith('json5')) { @@ -30,4 +31,4 @@ export default { _type: 'json' } } -} +}) diff --git a/src/runtime/server/transformers/markdown.ts b/src/runtime/transformers/markdown.ts similarity index 68% rename from src/runtime/server/transformers/markdown.ts rename to src/runtime/transformers/markdown.ts index a9103025b..f3342832e 100644 --- a/src/runtime/server/transformers/markdown.ts +++ b/src/runtime/transformers/markdown.ts @@ -1,13 +1,13 @@ -import { parse } from '../../markdown-parser' -import type { MarkdownOptions, MarkdownPlugin } from '../../types' -import { MarkdownParsedContent } from '../../types' -import { useRuntimeConfig } from '#imports' +import { parse } from '../markdown-parser' +import type { MarkdownOptions, MarkdownPlugin } from '../types' +import { MarkdownParsedContent } from '../types' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'markdown', extensions: ['.md'], - parse: async (_id, content) => { - const config: MarkdownOptions = { ...useRuntimeConfig().content?.markdown || {} } + parse: async (_id, content, options = {}) => { + const config = { ...options } as MarkdownOptions config.rehypePlugins = await importPlugins(config.rehypePlugins) config.remarkPlugins = await importPlugins(config.remarkPlugins) @@ -20,7 +20,7 @@ export default { _id } } -} +}) async function importPlugins (plugins: Record = {}) { const resolvedPlugins = {} diff --git a/src/runtime/server/transformers/path-meta.ts b/src/runtime/transformers/path-meta.ts similarity index 88% rename from src/runtime/server/transformers/path-meta.ts rename to src/runtime/transformers/path-meta.ts index ea73f6d74..c9d7fbebd 100644 --- a/src/runtime/server/transformers/path-meta.ts +++ b/src/runtime/transformers/path-meta.ts @@ -1,8 +1,7 @@ import { pascalCase } from 'scule' import slugify from 'slugify' import { withoutTrailingSlash, withLeadingSlash } from 'ufo' -import { ParsedContentMeta } from '../../types' -import { useRuntimeConfig } from '#imports' +import { defineTransformer } from './utils' const SEMVER_REGEX = /^(\d+)(\.\d+)*(\.x)?$/ @@ -13,7 +12,7 @@ const describeId = (_id: string) => { parts[parts.length - 1] = filename const _path = parts.join('/') - return > { + return { _source, _path, _extension, @@ -21,11 +20,11 @@ const describeId = (_id: string) => { } } -export default { +export default defineTransformer({ name: 'path-meta', extensions: ['.*'], - transform (content) { - const { locales, defaultLocale } = useRuntimeConfig().content || {} + transform (content, options: any = {}) { + const { locales = [], defaultLocale = 'en' } = options const { _source, _file, _path, _extension } = describeId(content._id) const parts = _path.split('/') @@ -34,7 +33,7 @@ export default { const filePath = parts.join('/') - return { + return { _path: generatePath(filePath), _draft: isDraft(filePath), _partial: isPartial(filePath), @@ -47,7 +46,7 @@ export default { _extension } } -} +}) /** * When file name ends with `.draft` then it will mark as draft. diff --git a/src/runtime/transformers/types.d.ts b/src/runtime/transformers/types.d.ts new file mode 100644 index 000000000..21253cb63 --- /dev/null +++ b/src/runtime/transformers/types.d.ts @@ -0,0 +1,16 @@ +interface TransformedContent { + [key: string]: any; +} + +export interface ContentTransformer { + name: string + extensions: string[] + parse?(id: string, content: string, options: any): TransformedContent + transform?(content: TransformedContent, options: any): TransformedContent +} + +export interface TransformContentOptions { + transformers?: ContentTransformer[] + + [key: string]: any +} diff --git a/src/runtime/transformers/utils.ts b/src/runtime/transformers/utils.ts new file mode 100644 index 000000000..61ffa8e0a --- /dev/null +++ b/src/runtime/transformers/utils.ts @@ -0,0 +1,5 @@ +import { ContentTransformer } from './types' + +export const defineTransformer = (transformer: ContentTransformer) => { + return transformer +} diff --git a/src/runtime/server/transformers/yaml.ts b/src/runtime/transformers/yaml.ts similarity index 87% rename from src/runtime/server/transformers/yaml.ts rename to src/runtime/transformers/yaml.ts index b7c875316..e90757f51 100644 --- a/src/runtime/server/transformers/yaml.ts +++ b/src/runtime/transformers/yaml.ts @@ -1,6 +1,7 @@ import { parseFrontMatter } from 'remark-mdc' +import { defineTransformer } from './utils' -export default { +export default defineTransformer({ name: 'Yaml', extensions: ['.yml', '.yaml'], parse: async (_id, content) => { @@ -20,4 +21,4 @@ export default { _type: 'yaml' } } -} +}) diff --git a/src/runtime/types.d.ts b/src/runtime/types.d.ts index 47a9b6dad..0a42f0833 100644 --- a/src/runtime/types.d.ts +++ b/src/runtime/types.d.ts @@ -135,13 +135,6 @@ export interface MarkdownParsedContent extends ParsedContent { } } -export interface ContentTransformer { - name: string - extensions: string[] - parse?(id: string, content: string): Promise | ParsedContent - transform?: ((content: ParsedContent) => Promise) | ((content: ParsedContent) => ParsedContent) -} - /** * Query */ From a05ad161dba3924caca933a40ef0dabd10e4a641 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 15:03:05 +0200 Subject: [PATCH 02/10] chore: shiki --- src/module.ts | 9 +- src/runtime/server/transformers/shiki.ts | 133 +++++++++++++++++++++++ 2 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 src/runtime/server/transformers/shiki.ts diff --git a/src/module.ts b/src/module.ts index 19829decd..d1b58516e 100644 --- a/src/module.ts +++ b/src/module.ts @@ -225,14 +225,7 @@ export default defineNuxtModule({ const { resolve } = createResolver(import.meta.url) const resolveRuntimeModule = (path: string) => resolveModule(path, { paths: resolve('./runtime') }) const contentContext: ContentContext = { - transformers: [ - // Register internal content plugins - // resolveRuntimeModule('./server/transformers/markdown'), - // resolveRuntimeModule('./server/transformers/yaml'), - // resolveRuntimeModule('./server/transformers/json'), - // resolveRuntimeModule('./server/transformers/csv'), - // resolveRuntimeModule('./server/transformers/path-meta') - ], + transformers: [], ...options } diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/server/transformers/shiki.ts new file mode 100644 index 000000000..0fb2417d1 --- /dev/null +++ b/src/runtime/server/transformers/shiki.ts @@ -0,0 +1,133 @@ +import { visit } from 'unist-util-visit' +import { withBase } from 'ufo' +import { useRuntimeConfig } from '#imports' + +const highlightConfig = useRuntimeConfig().content.highlight + +const withContentBase = (url: string) => { + return withBase(url, `/api/${useRuntimeConfig().public.content.base}`) +} + +export default { + name: 'markdown', + extensions: ['.md'], + transform: async (content) => { + const tokenColors: Record = {} + const codeBlocks: any[] = [] + const inlineCodes: any = [] + visit( + content.body, + (node: any) => (node.tag === 'code' && node?.props.code) || (node.tag === 'code-inline' && (node.props?.lang || node.props?.language)), + (node) => { + if (node.tag === 'code') { + codeBlocks.push(node) + } else if (node.tag === 'code-inline') { + inlineCodes.push(node) + } + } + ) + + await Promise.all(codeBlocks.map(highlightBlock)) + await Promise.all(inlineCodes.map(highlightInline)) + + // Inject token colors at the end of the document + if (Object.values(tokenColors).length) { + const colors: string[] = [] + for (const colorClass of Object.values(tokenColors)) { + Object.entries(colorClass.colors).forEach(([variant, color]) => { + if (variant === 'default') { + colors.unshift(`.${colorClass.className}{color:${color}}`) + } else { + colors.push(`.${variant} .${colorClass.className}{color:${color}}`) + } + }) + } + + content.body.children.push({ + type: 'element', + tag: 'style', + children: [{ type: 'text', value: colors.join('') }] + }) + } + + return content + + /** + * Highlight inline code + */ + async function highlightInline (node) { + const code = node.children[0].value + + // Fetch highlighted tokens + const lines = await $fetch(withContentBase('highlight'), { + method: 'POST', + body: { + code, + lang: node.props.lang || node.props.language, + theme: highlightConfig.theme + } + }) + + // Generate highlighted children + node.children = lines[0].map(tokenSpan) + + node.props = node.props || {} + node.props.class = 'colored' + + return node + } + + /** + * Highlight a code block + */ + async function highlightBlock (node) { + const { code, language: lang, highlights = [] } = node.props + + // Fetch highlighted tokens + const lines = await $fetch(withContentBase('highlight'), { + method: 'POST', + body: { + code, + lang, + theme: highlightConfig.theme + } + }) + + // Generate highlighted children + const innerCodeNode = node.children[0].children[0] + innerCodeNode.children = lines.map((line, lineIndex) => ({ + type: 'element', + tag: 'span', + props: { class: ['line', highlights.includes(lineIndex + 1) ? 'highlight' : ''].join(' ').trim() }, + children: line.map(tokenSpan) + })) + return node + } + + function getColorProps (token) { + if (!token.color) { + return {} + } + if (typeof token.color === 'string') { + return { style: { color: token.color } } + } + const key = Object.values(token.color).join('') + if (!tokenColors[key]) { + tokenColors[key] = { + colors: token.color, + className: 'ct-' + Math.random().toString(16).substring(2, 8) // hash(key) + } + } + return { class: tokenColors[key].className } + } + + function tokenSpan (token) { + return { + type: 'element', + tag: 'span', + props: getColorProps(token), + children: [{ type: 'text', value: token.content }] + } + } + } +} From d9dfd8d8f3c08e4588e4501adf9a93a3ebb60da4 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 15:27:34 +0200 Subject: [PATCH 03/10] fix: transformers detection --- src/runtime/server/navigation.ts | 2 +- src/runtime/server/storage.ts | 10 +++++++--- src/runtime/transformers/index.ts | 15 +++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/runtime/server/navigation.ts b/src/runtime/server/navigation.ts index 31c897428..ee94da9aa 100644 --- a/src/runtime/server/navigation.ts +++ b/src/runtime/server/navigation.ts @@ -1,5 +1,5 @@ import { NavItem, ParsedContentMeta } from '../types' -import { generateTitle } from './transformers/path-meta' +import { generateTitle } from '../transformers/path-meta' import { useRuntimeConfig } from '#imports' type PrivateNavItem = NavItem & { path?: string } diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index 006c59ab8..b4f0a73d4 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -140,17 +140,21 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise */ export async function parseContent (id: string, content: string) { const nitroApp = useNitroApp() - const { markdown, csv, yaml } = useRuntimeConfig().content + const { markdown, csv, yaml, defaultLocale, locales } = useRuntimeConfig().content // Call hook before parsing the file const file = { _id: id, body: content } await nitroApp.hooks.callHook('content:file:beforeParse', file) - const result = transformContent(id, file.body, { + const result = await transformContent(id, file.body, { transformers, markdown, csv, - yaml + yaml, + pathMeta: { + defaultLocale, + locales + } }) // Call hook after parsing the file diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts index d6c98d2f7..603d836a4 100644 --- a/src/runtime/transformers/index.ts +++ b/src/runtime/transformers/index.ts @@ -1,5 +1,5 @@ import { extname } from 'pathe' -import { pascalCase } from 'scule' +import { camelCase } from 'scule' import type { ContentTransformer, TransformContentOptions } from './types' import csv from './csv' import markdown from './markdown' @@ -24,11 +24,10 @@ function getParser (ext, additionalTransformers: ContentTransformer[] = []): Con } function getTransformers (ext, additionalTransformers: ContentTransformer[] = []) { - const transformr = additionalTransformers.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) - if (transformr) { - return transformr - } - return TRANSFORMERS.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) + return [ + ...additionalTransformers.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform), + ...TRANSFORMERS.filter(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.transform) + ] } /** @@ -47,14 +46,14 @@ export async function transformContent (id, content, options: TransformContentOp return file } - const parserOptions = options[pascalCase(parser.name)] || {} + const parserOptions = options[camelCase(parser.name)] || {} const parsed = await parser.parse!(file._id, file.body, parserOptions) const matchedTransformers = getTransformers(ext, transformers) const result = await matchedTransformers.reduce(async (prev, cur) => { const next = (await prev) || parsed - const transformOptions = options[pascalCase(cur.name)] || {} + const transformOptions = options[camelCase(cur.name)] || {} return cur.transform!(next, transformOptions) }, Promise.resolve(parsed)) From cc82da7fb6161b6ebe2089c601e80127afd84d25 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 15:38:12 +0200 Subject: [PATCH 04/10] chore: cleanup --- package.json | 4 ---- src/runtime/transformers/index.ts | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index f88f3039f..b78d2eabd 100644 --- a/package.json +++ b/package.json @@ -14,10 +14,6 @@ ".": { "import": "./dist/module.mjs", "require": "./dist/module.cjs" - }, - "./transformers": { - "import": "./dist/runtime/transformers/index.mjs", - "require": "./dist/runtime/transformers/index.cjs" } }, "main": "./dist/module.cjs", diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts index 603d836a4..ea42114bb 100644 --- a/src/runtime/transformers/index.ts +++ b/src/runtime/transformers/index.ts @@ -16,11 +16,12 @@ const TRANSFORMERS = [ ] function getParser (ext, additionalTransformers: ContentTransformer[] = []): ContentTransformer { - const parser = additionalTransformers.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) - if (parser) { - return parser + let parser = additionalTransformers.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) + if (!parser) { + parser = TRANSFORMERS.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) } - return TRANSFORMERS.find(p => ext.match(new RegExp(p.extensions.join('|'), 'i')) && p.parse) + + return parser } function getTransformers (ext, additionalTransformers: ContentTransformer[] = []) { From 6e73529db35dcdf0e247cc96262f18df36fa420d Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 15:56:56 +0200 Subject: [PATCH 05/10] fix: update types --- src/runtime/transformers/csv.ts | 3 ++- src/runtime/transformers/index.ts | 2 +- src/runtime/transformers/json.ts | 3 ++- src/runtime/transformers/path-meta.ts | 3 ++- src/runtime/transformers/types.d.ts | 16 ---------------- src/runtime/transformers/utils.ts | 2 +- src/runtime/transformers/yaml.ts | 3 ++- src/runtime/types.d.ts | 13 +++++++++++++ 8 files changed, 23 insertions(+), 22 deletions(-) delete mode 100644 src/runtime/transformers/types.d.ts diff --git a/src/runtime/transformers/csv.ts b/src/runtime/transformers/csv.ts index 2e0906a60..3ddd3d45c 100644 --- a/src/runtime/transformers/csv.ts +++ b/src/runtime/transformers/csv.ts @@ -1,3 +1,4 @@ +import { ParsedContent } from '../types' import { defineTransformer } from './utils' export default defineTransformer({ @@ -9,7 +10,7 @@ export default defineTransformer({ const parsed = await csvToJson({ output: 'json', ...options }) .fromString(content) - return { + return { _id, _type: 'csv', body: parsed diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts index ea42114bb..c6909a618 100644 --- a/src/runtime/transformers/index.ts +++ b/src/runtime/transformers/index.ts @@ -1,6 +1,6 @@ import { extname } from 'pathe' import { camelCase } from 'scule' -import type { ContentTransformer, TransformContentOptions } from './types' +import type { ContentTransformer, TransformContentOptions } from '../types' import csv from './csv' import markdown from './markdown' import yaml from './yaml' diff --git a/src/runtime/transformers/json.ts b/src/runtime/transformers/json.ts index 6e87d9258..677eed2eb 100644 --- a/src/runtime/transformers/json.ts +++ b/src/runtime/transformers/json.ts @@ -1,4 +1,5 @@ import destr from 'destr' +import { ParsedContent } from '../types' import { defineTransformer } from './utils' export default defineTransformer({ @@ -25,7 +26,7 @@ export default defineTransformer({ } } - return { + return { ...parsed, _id, _type: 'json' diff --git a/src/runtime/transformers/path-meta.ts b/src/runtime/transformers/path-meta.ts index c9d7fbebd..a01baa0dd 100644 --- a/src/runtime/transformers/path-meta.ts +++ b/src/runtime/transformers/path-meta.ts @@ -1,6 +1,7 @@ import { pascalCase } from 'scule' import slugify from 'slugify' import { withoutTrailingSlash, withLeadingSlash } from 'ufo' +import { ParsedContent } from '../types' import { defineTransformer } from './utils' const SEMVER_REGEX = /^(\d+)(\.\d+)*(\.x)?$/ @@ -33,7 +34,7 @@ export default defineTransformer({ const filePath = parts.join('/') - return { + return { _path: generatePath(filePath), _draft: isDraft(filePath), _partial: isPartial(filePath), diff --git a/src/runtime/transformers/types.d.ts b/src/runtime/transformers/types.d.ts deleted file mode 100644 index 21253cb63..000000000 --- a/src/runtime/transformers/types.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -interface TransformedContent { - [key: string]: any; -} - -export interface ContentTransformer { - name: string - extensions: string[] - parse?(id: string, content: string, options: any): TransformedContent - transform?(content: TransformedContent, options: any): TransformedContent -} - -export interface TransformContentOptions { - transformers?: ContentTransformer[] - - [key: string]: any -} diff --git a/src/runtime/transformers/utils.ts b/src/runtime/transformers/utils.ts index 61ffa8e0a..169348a70 100644 --- a/src/runtime/transformers/utils.ts +++ b/src/runtime/transformers/utils.ts @@ -1,4 +1,4 @@ -import { ContentTransformer } from './types' +import { ContentTransformer } from '../types' export const defineTransformer = (transformer: ContentTransformer) => { return transformer diff --git a/src/runtime/transformers/yaml.ts b/src/runtime/transformers/yaml.ts index e90757f51..70af82bec 100644 --- a/src/runtime/transformers/yaml.ts +++ b/src/runtime/transformers/yaml.ts @@ -1,4 +1,5 @@ import { parseFrontMatter } from 'remark-mdc' +import { ParsedContent } from '../types' import { defineTransformer } from './utils' export default defineTransformer({ @@ -15,7 +16,7 @@ export default defineTransformer({ parsed = { body: data } } - return { + return { ...parsed, _id, _type: 'yaml' diff --git a/src/runtime/types.d.ts b/src/runtime/types.d.ts index 0a42f0833..e621c4696 100644 --- a/src/runtime/types.d.ts +++ b/src/runtime/types.d.ts @@ -135,6 +135,19 @@ export interface MarkdownParsedContent extends ParsedContent { } } +export interface ContentTransformer { + name: string + extensions: string[] + parse?(id: string, content: string, options: any): Promise | ParsedContent + transform?(content: ParsedContent, options: any): Promise | ParsedContent +} + +export interface TransformContentOptions { + transformers?: ContentTransformer[] + + [key: string]: any +} + /** * Query */ From 940acec1a61f94f367adf6fcbd7edaf4afbe39b3 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Wed, 13 Jul 2022 17:21:24 +0200 Subject: [PATCH 06/10] fix: highlight --- src/module.ts | 4 ++- src/runtime/server/storage.ts | 5 ++-- src/runtime/transformers/index.ts | 4 ++- .../{server => }/transformers/shiki.ts | 25 +++++++------------ 4 files changed, 18 insertions(+), 20 deletions(-) rename src/runtime/{server => }/transformers/shiki.ts (85%) diff --git a/src/module.ts b/src/module.ts index d1b58516e..370f442ec 100644 --- a/src/module.ts +++ b/src/module.ts @@ -391,7 +391,9 @@ export default defineNuxtModule({ // Register highlighter if (options.highlight) { - contentContext.transformers.push(resolveRuntimeModule('./server/transformers/shiki')) + contentContext.transformers.push(resolveRuntimeModule('./transformers/shiki')) + // @ts-ignore + contentContext.highlight.apiURL = `/api/${options.base}/highlight` nuxt.hook('nitro:config', (nitroConfig) => { nitroConfig.handlers = nitroConfig.handlers || [] diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index b4f0a73d4..2a806ef65 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -140,7 +140,7 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise */ export async function parseContent (id: string, content: string) { const nitroApp = useNitroApp() - const { markdown, csv, yaml, defaultLocale, locales } = useRuntimeConfig().content + const { markdown, csv, yaml, defaultLocale, locales, highlight } = useRuntimeConfig().content // Call hook before parsing the file const file = { _id: id, body: content } @@ -154,7 +154,8 @@ export async function parseContent (id: string, content: string) { pathMeta: { defaultLocale, locales - } + }, + highlight }) // Call hook after parsing the file diff --git a/src/runtime/transformers/index.ts b/src/runtime/transformers/index.ts index c6909a618..0991d2f4c 100644 --- a/src/runtime/transformers/index.ts +++ b/src/runtime/transformers/index.ts @@ -6,13 +6,15 @@ import markdown from './markdown' import yaml from './yaml' import pathMeta from './path-meta' import json from './json' +import highlight from './shiki' const TRANSFORMERS = [ csv, markdown, json, yaml, - pathMeta + pathMeta, + highlight ] function getParser (ext, additionalTransformers: ContentTransformer[] = []): ContentTransformer { diff --git a/src/runtime/server/transformers/shiki.ts b/src/runtime/transformers/shiki.ts similarity index 85% rename from src/runtime/server/transformers/shiki.ts rename to src/runtime/transformers/shiki.ts index 0fb2417d1..cbb246679 100644 --- a/src/runtime/server/transformers/shiki.ts +++ b/src/runtime/transformers/shiki.ts @@ -1,17 +1,10 @@ import { visit } from 'unist-util-visit' -import { withBase } from 'ufo' -import { useRuntimeConfig } from '#imports' +import { defineTransformer } from './utils' -const highlightConfig = useRuntimeConfig().content.highlight - -const withContentBase = (url: string) => { - return withBase(url, `/api/${useRuntimeConfig().public.content.base}`) -} - -export default { - name: 'markdown', +export default defineTransformer({ + name: 'highlight', extensions: ['.md'], - transform: async (content) => { + transform: async (content, options = {}) => { const tokenColors: Record = {} const codeBlocks: any[] = [] const inlineCodes: any = [] @@ -59,12 +52,12 @@ export default { const code = node.children[0].value // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { + const lines = await $fetch(options.apiURL, { method: 'POST', body: { code, lang: node.props.lang || node.props.language, - theme: highlightConfig.theme + theme: options.theme } }) @@ -84,12 +77,12 @@ export default { const { code, language: lang, highlights = [] } = node.props // Fetch highlighted tokens - const lines = await $fetch(withContentBase('highlight'), { + const lines = await $fetch(options.apiURL, { method: 'POST', body: { code, lang, - theme: highlightConfig.theme + theme: options.theme } }) @@ -130,4 +123,4 @@ export default { } } } -} +}) From fcced71003a11bd52ba33893390447c2e44afa23 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Tue, 19 Jul 2022 12:34:46 +0200 Subject: [PATCH 07/10] feat: allow overriding parser options --- src/runtime/server/storage.ts | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index 2a806ef65..8161b365f 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -2,14 +2,26 @@ import { prefixStorage } from 'unstorage' import { joinURL, withLeadingSlash, withoutTrailingSlash } from 'ufo' import { hash as ohash } from 'ohash' import type { CompatibilityEvent } from 'h3' -import type { QueryBuilderParams, ParsedContent, QueryBuilder } from '../types' +import defu from 'defu' +import type { QueryBuilderParams, ParsedContent, QueryBuilder, ContentTransformer } from '../types' import { createQuery } from '../query/query' import { createPipelineFetcher } from '../query/match/pipeline' import { transformContent } from '../transformers' +import type { ModuleOptions } from '../../module' import { getPreview, isPreview } from './preview' // eslint-disable-next-line import/named import { useNitroApp, useRuntimeConfig, useStorage } from '#imports' -import { transformers } from '#content/virtual/transformers' +import { transformers as customTransformers } from '#content/virtual/transformers' + +interface ParseContentOptions { + csv?: ModuleOptions['csv'] + yaml?: ModuleOptions['yaml'] + highlight?: ModuleOptions['highlight'] + locales?: ModuleOptions['locales'] + defaultLocale?: ModuleOptions['defaultLocale'] + markdown?: ModuleOptions['markdown'] + transformers?: ContentTransformer[] +} export const sourceStorage = prefixStorage(useStorage(), 'content:source') export const cacheStorage = prefixStorage(useStorage(), 'cache:content') @@ -138,9 +150,20 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise /** * Parse content file using registered plugins */ -export async function parseContent (id: string, content: string) { +export async function parseContent (id: string, content: string, options: ParseContentOptions = {}) { const nitroApp = useNitroApp() - const { markdown, csv, yaml, defaultLocale, locales, highlight } = useRuntimeConfig().content + const { transformers, markdown, csv, yaml, defaultLocale, locales, highlight } = defu( + options, + { + markdown: contentConfig.markdown, + csv: contentConfig.csv, + yaml: contentConfig.yaml, + defaultLocale: contentConfig.defaultLocale, + locales: contentConfig.locales, + highlight: contentConfig.highlight, + transformers: customTransformers + } + ) // Call hook before parsing the file const file = { _id: id, body: content } From 4b566ed724ed4d9537c337e2948e27998376ffaa Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Tue, 19 Jul 2022 12:47:53 +0200 Subject: [PATCH 08/10] chore: update options structure --- src/runtime/server/storage.ts | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index 8161b365f..d1b3d56bb 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -17,10 +17,14 @@ interface ParseContentOptions { csv?: ModuleOptions['csv'] yaml?: ModuleOptions['yaml'] highlight?: ModuleOptions['highlight'] - locales?: ModuleOptions['locales'] - defaultLocale?: ModuleOptions['defaultLocale'] markdown?: ModuleOptions['markdown'] transformers?: ContentTransformer[] + pathMeta?: { + locales?: ModuleOptions['locales'] + defaultLocale?: ModuleOptions['defaultLocale'] + } + // Allow passing options for custom transformers + [key: string]: any } export const sourceStorage = prefixStorage(useStorage(), 'content:source') @@ -150,18 +154,20 @@ export const getContent = async (event: CompatibilityEvent, id: string): Promise /** * Parse content file using registered plugins */ -export async function parseContent (id: string, content: string, options: ParseContentOptions = {}) { +export async function parseContent (id: string, content: string, opts: ParseContentOptions = {}) { const nitroApp = useNitroApp() - const { transformers, markdown, csv, yaml, defaultLocale, locales, highlight } = defu( - options, + const options = defu( + opts, { markdown: contentConfig.markdown, csv: contentConfig.csv, yaml: contentConfig.yaml, - defaultLocale: contentConfig.defaultLocale, - locales: contentConfig.locales, highlight: contentConfig.highlight, - transformers: customTransformers + transformers: customTransformers, + pathMeta: { + defaultLocale: contentConfig.defaultLocale, + locales: contentConfig.locales + } } ) @@ -169,17 +175,7 @@ export async function parseContent (id: string, content: string, options: ParseC const file = { _id: id, body: content } await nitroApp.hooks.callHook('content:file:beforeParse', file) - const result = await transformContent(id, file.body, { - transformers, - markdown, - csv, - yaml, - pathMeta: { - defaultLocale, - locales - }, - highlight - }) + const result = await transformContent(id, file.body, options) // Call hook after parsing the file await nitroApp.hooks.callHook('content:file:afterParse', result) From 291961bb6e5b50b3df98f7eb4cf3d88ef438a011 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Tue, 19 Jul 2022 13:07:29 +0200 Subject: [PATCH 09/10] test: add tests --- test/basic.test.ts | 3 ++ test/features/parser-options.ts | 42 +++++++++++++++++++++++++ test/fixtures/basic/server/api/parse.ts | 4 +-- 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 test/features/parser-options.ts diff --git a/test/basic.test.ts b/test/basic.test.ts index 02a40cc8c..3e2c8b907 100644 --- a/test/basic.test.ts +++ b/test/basic.test.ts @@ -16,6 +16,7 @@ import { testModuleOptions } from './features/module-options' import { testContentQuery } from './features/content-query' import { testHighlighter } from './features/highlighter' import { testMarkdownRenderer } from './features/renderer-markdown' +import { testParserOptions } from './features/parser-options' const spyConsoleWarn = vi.spyOn(global.console, 'warn') @@ -141,4 +142,6 @@ describe('Basic usage', async () => { testModuleOptions() testHighlighter() + + testParserOptions() }) diff --git a/test/features/parser-options.ts b/test/features/parser-options.ts new file mode 100644 index 000000000..a46c9bb7e --- /dev/null +++ b/test/features/parser-options.ts @@ -0,0 +1,42 @@ +import { describe, test, expect, assert } from 'vitest' +import { $fetch } from '@nuxt/test-utils' + +export const testParserOptions = () => { + describe('Parser Options', () => { + test('disable MDC syntax', async () => { + const parsed = await $fetch('/api/parse', { + method: 'POST', + body: { + id: 'content:index.md', + content: ':component', + options: { + markdown: { + mdc: false + } + } + } + }) + expect(parsed).toHaveProperty('_id') + assert(parsed.body.children[0].tag === 'p') + assert(parsed.body.children[0].children[0].type === 'text') + assert(parsed.body.children[0].children[0].value === ':component') + }) + + test('custom locale', async () => { + const parsed = await $fetch('/api/parse', { + method: 'POST', + body: { + id: 'content:index.md', + content: ':component', + options: { + pathMeta: { + defaultLocale: 'jp' + } + } + } + }) + expect(parsed).toHaveProperty('_id') + expect(parsed.locale).toBe('jp') + }) + }) +} diff --git a/test/fixtures/basic/server/api/parse.ts b/test/fixtures/basic/server/api/parse.ts index 73d9c96de..4665378cb 100644 --- a/test/fixtures/basic/server/api/parse.ts +++ b/test/fixtures/basic/server/api/parse.ts @@ -2,10 +2,10 @@ import { defineEventHandler, useBody } from 'h3' import { parseContent } from '#content/server' export default defineEventHandler(async (event) => { - const { id, content } = await useBody(event) + const { id, content, options } = await useBody(event) // @ts-ignore - const parsedContent = await parseContent(id, content) + const parsedContent = await parseContent(id, content, options) return parsedContent }) From 54bd6bfb779d8594e315f22ffd5380e74637d823 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Tue, 19 Jul 2022 13:12:27 +0200 Subject: [PATCH 10/10] chore: typo --- test/features/parser-options.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/features/parser-options.ts b/test/features/parser-options.ts index a46c9bb7e..acf297433 100644 --- a/test/features/parser-options.ts +++ b/test/features/parser-options.ts @@ -36,7 +36,7 @@ export const testParserOptions = () => { } }) expect(parsed).toHaveProperty('_id') - expect(parsed.locale).toBe('jp') + expect(parsed._locale).toBe('jp') }) }) }