Skip to content

Commit 4ddfdb1

Browse files
antfuantfu
committed
feat: upgrade oniguruma-to-js, support more languages
1 parent 930dddc commit 4ddfdb1

25 files changed

+1306
-316
lines changed

‎docs/references/engine-js-compat.md‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# JavaScript RegExp Engine Compatibility References
22

3-
> Genreated on Sunday, September 1, 2024
3+
> Genreated on Wednesday, September 4, 2024
44
>
5-
> Version `1.15.2`
5+
> Version `1.16.1`
66
>
77
> Runtime: Node.js v20.12.2
88
@@ -11,9 +11,9 @@
1111
| | Count |
1212
| :-------------- | --------------------------------: |
1313
| Total Languages | 213 |
14-
| Fully Supported | [178](#fully-supported-languages) |
15-
| Mismatched | [30](#mismatched-languages) |
16-
| Unsupported | [5](#unsupported-languages) |
14+
| Fully Supported | [187](#fully-supported-languages) |
15+
| Mismatched | [18](#mismatched-languages) |
16+
| Unsupported | [8](#unsupported-languages) |
1717

1818
## Fully Supported Languages
1919

@@ -28,6 +28,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
2828
| apl | ✅ OK | 179 | - |
2929
| applescript | ✅ OK | 151 | - |
3030
| ara | ✅ OK | 54 | - |
31+
| asciidoc | ✅ OK | 262 | - |
3132
| asm | ✅ OK | 297 | - |
3233
| astro | ✅ OK | 59 | - |
3334
| awk | ✅ OK | 36 | - |
@@ -38,6 +39,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
3839
| bibtex | ✅ OK | 19 | - |
3940
| bicep | ✅ OK | 28 | - |
4041
| c | ✅ OK | 158 | - |
42+
| cadence | ✅ OK | 71 | - |
4143
| clarity | ✅ OK | 43 | - |
4244
| clj | ✅ OK | 38 | - |
4345
| clojure | ✅ OK | 38 | - |
@@ -47,10 +49,13 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
4749
| codeql | ✅ OK | 150 | - |
4850
| coffee | ✅ OK | 120 | - |
4951
| common-lisp | ✅ OK | 57 | - |
52+
| coq | ✅ OK | 25 | - |
5053
| cpp | ✅ OK | 220 | - |
5154
| crystal | ✅ OK | 140 | - |
5255
| css | ✅ OK | 141 | - |
5356
| csv | ✅ OK | 1 | - |
57+
| cue | ✅ OK | 85 | - |
58+
| cypher | ✅ OK | 39 | - |
5459
| d | ✅ OK | 270 | - |
5560
| dart | ✅ OK | 71 | - |
5661
| dax | ✅ OK | 23 | - |
@@ -133,6 +138,8 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
133138
| perl | ✅ OK | 156 | - |
134139
| plsql | ✅ OK | 43 | - |
135140
| postcss | ✅ OK | 47 | - |
141+
| powerquery | ✅ OK | 30 | - |
142+
| powershell | ✅ OK | 88 | - |
136143
| prisma | ✅ OK | 26 | - |
137144
| prolog | ✅ OK | 26 | - |
138145
| proto | ✅ OK | 33 | - |
@@ -171,6 +178,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
171178
| tcl | ✅ OK | 33 | - |
172179
| templ | ✅ OK | 74 | - |
173180
| terraform | ✅ OK | 62 | - |
181+
| tex | ✅ OK | 38 | - |
174182
| toml | ✅ OK | 40 | - |
175183
| ts-tags | ✅ OK | - | - |
176184
| tsv | ✅ OK | 1 | - |
@@ -196,6 +204,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
196204
| wolfram | ✅ OK | 501 | - |
197205
| xml | ✅ OK | 30 | - |
198206
| xsl | ✅ OK | 5 | - |
207+
| yaml | ✅ OK | 46 | - |
199208
| zenscript | ✅ OK | 21 | - |
200209
| zig | ✅ OK | 51 | - |
201210
| zsh | ✅ OK | 146 | - |
@@ -208,43 +217,34 @@ Languages that does not throw with the JavaScript RegExp engine, but will produc
208217
| ------------ | :-------------- | ----------------: | --------------: |
209218
| angular-html | ⚠️ Mismatch | 2 | - |
210219
| apex | ⚠️ Mismatch | 189 | - |
211-
| asciidoc | ⚠️ Mismatch | 262 | - |
212220
| beancount | ⚠️ Mismatch | 39 | - |
213221
| blade | ⚠️ Mismatch | 330 | - |
214-
| cadence | ⚠️ Mismatch | 71 | - |
215-
| coq | ⚠️ Mismatch | 25 | - |
216-
| cue | ⚠️ Mismatch | 85 | - |
217-
| cypher | ⚠️ Mismatch | 39 | - |
218222
| haml | ⚠️ Mismatch | 64 | - |
219-
| haskell | ⚠️ Mismatch | 157 | - |
220223
| kusto | ⚠️ Mismatch | 60 | - |
221224
| latex | ⚠️ Mismatch | 183 | - |
222225
| markdown | ⚠️ Mismatch | 103 | - |
223226
| mdc | ⚠️ Mismatch | 27 | - |
224-
| mdx | ⚠️ Mismatch | 180 | - |
225227
| mermaid | ⚠️ Mismatch | 129 | - |
226228
| nginx | ⚠️ Mismatch | 102 | - |
227229
| php | ⚠️ Mismatch | 328 | - |
228230
| po | ⚠️ Mismatch | 23 | - |
229-
| powerquery | ⚠️ Mismatch | 30 | - |
230-
| powershell | ⚠️ Mismatch | 88 | - |
231231
| pug | ⚠️ Mismatch | 92 | - |
232-
| purescript | ⚠️ Mismatch | 72 | - |
233232
| rst | ⚠️ Mismatch | 61 | - |
234233
| splunk | ⚠️ Mismatch | 17 | - |
235234
| stata | ⚠️ Mismatch | 189 | - |
236235
| systemd | ⚠️ Mismatch | 32 | - |
237-
| tex | ⚠️ Mismatch | 38 | - |
238-
| yaml | ⚠️ Mismatch | 46 | - |
239236

240237
## Unsupported Languages
241238

242239
Languages that throws with the JavaScript RegExp engine (contains syntaxes that we can't polyfill yet). If you need to use these languages, please use the Oniguruma engine.
243240

244-
| Language | Highlight Match | Patterns Parsable | Patterns Failed |
245-
| -------- | :-------------- | ----------------: | --------------: |
246-
| ada | ✅ OK | 199 | 1 |
247-
| csharp | ⚠️ Mismatch | 298 | 1 |
248-
| razor | ⚠️ Mismatch | 83 | 2 |
249-
| swift | ❌ Error | 302 | 4 |
250-
| julia | ❌ Error | 77 | 18 |
241+
| Language | Highlight Match | Patterns Parsable | Patterns Failed |
242+
| ---------- | :-------------- | ----------------: | --------------: |
243+
| ada | ✅ OK | 199 | 1 |
244+
| csharp | ⚠️ Mismatch | 298 | 1 |
245+
| razor | ⚠️ Mismatch | 83 | 2 |
246+
| mdx | ❌ Error | 177 | 4 |
247+
| julia | ❌ Error | 90 | 5 |
248+
| swift | ❌ Error | 301 | 5 |
249+
| purescript | ❌ Error | 64 | 8 |
250+
| haskell | ❌ Error | 113 | 44 |

‎packages/compat/test/fixtures.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import getHighlighter from '../src'
88
describe('fixtures', () => {
99
const files = import.meta.glob('./input/*.*', { as: 'raw', eager: true })
1010
const filter = process.env.FILTER
11-
Object.entries(files)
11+
Object
12+
.entries(files)
1213
.forEach(([path, content]) => {
1314
const run = !filter || path.includes(filter)
1415
? it

‎packages/core/src/highlight/code-to-tokens-ansi.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ function dimColor(color: string) {
7070
if (hexMatch) {
7171
if (hexMatch[3]) {
7272
// convert from #rrggbbaa to #rrggbb(aa/2)
73-
const alpha = Math.round(Number.parseInt(hexMatch[3], 16) / 2)
73+
const alpha = Math
74+
.round(Number.parseInt(hexMatch[3], 16) / 2)
7475
.toString(16)
7576
.padStart(2, '0')
7677
return `#${hexMatch[1]}${hexMatch[2]}${alpha}`
@@ -81,7 +82,8 @@ function dimColor(color: string) {
8182
}
8283
else {
8384
// convert from #rgb to #rrggbb80
84-
return `#${Array.from(hexMatch[1])
85+
return `#${Array
86+
.from(hexMatch[1])
8587
.map(x => `${x}${x}`)
8688
.join('')}80`
8789
}

‎packages/core/src/highlight/code-to-tokens-themes.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ export function codeToTokensWithThemes(
1414
code: string,
1515
options: CodeToTokensWithThemesOptions,
1616
): ThemedTokenWithVariants[][] {
17-
const themes = Object.entries(options.themes)
17+
const themes = Object
18+
.entries(options.themes)
1819
.filter(i => i[1])
1920
.map(i => ({ color: i[0], theme: i[1]! }))
2021

‎packages/core/src/highlight/code-to-tokens.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ export function codeToTokens(
2626
cssVariablePrefix = '--shiki-',
2727
} = options
2828

29-
const themes = Object.entries(options.themes)
29+
const themes = Object
30+
.entries(options.themes)
3031
.filter(i => i[1])
3132
.map(i => ({ color: i[0], theme: i[1]! }))
3233
.sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0)

‎packages/core/src/utils.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ export function splitToken<
137137
export function splitTokens<
138138
T extends Pick<ThemedToken, 'content' | 'offset'>,
139139
>(tokens: T[][], breakpoints: number[] | Set<number>) {
140-
const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
140+
const sorted = Array
141+
.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
141142
.sort((a, b) => a - b)
142143

143144
if (!sorted.length)

‎packages/rehype/test/core.test.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ it('run with rehype-raw', async () => {
4848
],
4949
})
5050

51-
// eslint-disable-next-line unicorn/consistent-function-scoping
5251
const rehypeMetaString = () => (tree: Root) => {
5352
visit(tree, 'element', (node) => {
5453
if (node.tagName === 'code' && node.data?.meta) {

‎packages/shiki/scripts/prepare/langs.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,13 @@ export default langs
118118

119119
const bundled = Array.from(bundledIds).map(id => grammars.find(i => i.name === id)!).filter(Boolean)
120120

121-
const info = bundled.map(i => ({
122-
id: i.name,
123-
name: i.displayName || i.name,
124-
aliases: i.aliases,
125-
import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
126-
}) as const)
121+
const info = bundled
122+
.map(i => ({
123+
id: i.name,
124+
name: i.displayName || i.name,
125+
aliases: i.aliases,
126+
import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
127+
}) as const)
127128
.sort((a, b) => a.id.localeCompare(b.id))
128129

129130
const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => ` | '${i}'`).join('\n')

‎packages/shiki/test/core.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,9 @@ describe('errors', () => {
158158
})
159159

160160
const code = shiki.codeToHtml('console.log("Hi")', { lang: 'javascript', theme: mtp })
161-
expect.soft(code)
161+
162+
expect
163+
.soft(code)
162164
.toMatchInlineSnapshot(`"<pre class="shiki material-theme-palenight" style="background-color:#292D3E;color:#babed8" tabindex="0"><code><span class="line"><span style="color:#BABED8">console</span><span style="color:#89DDFF">.</span><span style="color:#82AAFF">log</span><span style="color:#BABED8">(</span><span style="color:#89DDFF">"</span><span style="color:#C3E88D">Hi</span><span style="color:#89DDFF">"</span><span style="color:#BABED8">)</span></span></code></pre>"`)
163165

164166
expect.soft(shiki.getLoadedThemes()).toContain('material-theme-palenight')

‎packages/shiki/test/decorations.test.ts‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ describe('decorations errors', () => {
105105
{ start: 10, end: 0 },
106106
],
107107
})
108-
}).rejects
108+
})
109+
.rejects
109110
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration range: {"line":1,"character":6,"offset":10} - {"line":0,"character":0,"offset":0}]`)
110111
})
111112

@@ -119,8 +120,7 @@ describe('decorations errors', () => {
119120
{ start: 5, end: 15 },
120121
],
121122
})
122-
}).rejects
123-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
123+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
124124
})
125125

126126
it('throws when lines overflow', async () => {
@@ -132,8 +132,7 @@ describe('decorations errors', () => {
132132
{ start: { line: 100, character: 0 }, end: { line: 100, character: 1 } },
133133
],
134134
})
135-
}).rejects
136-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
135+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
137136
})
138137

139138
it('throws when chars overflow', async () => {
@@ -145,8 +144,7 @@ describe('decorations errors', () => {
145144
{ start: { line: 0, character: 0 }, end: { line: 0, character: 10 } },
146145
],
147146
})
148-
}).rejects
149-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
147+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
150148

151149
expect(async () => {
152150
await codeToHtml(code, {
@@ -159,8 +157,7 @@ describe('decorations errors', () => {
159157
},
160158
],
161159
})
162-
}).rejects
163-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
160+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
164161
})
165162

166163
it('throws when offset underflows/overflows', async () => {
@@ -170,16 +167,14 @@ describe('decorations errors', () => {
170167
lang: 'ts',
171168
decorations: [{ start: 1, end: 1000 }],
172169
})
173-
}).rejects
174-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
170+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
175171

176172
expect(async () => {
177173
await codeToHtml(code, {
178174
theme: 'vitesse-light',
179175
lang: 'ts',
180176
decorations: [{ start: -3, end: 5 }],
181177
})
182-
}).rejects
183-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
178+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
184179
})
185180
})

0 commit comments

Comments
 (0)