feat: allow importing CSS and assets inside external dependencies (#3880)

Author: sheremet-va

docs/config/index.md

@@ -196,14 +196,69 @@ When Vitest encounters the external library listed in `include`, it will be bund
 - Your `alias` configuration is now respected inside bundled packages
 - Code in your tests is running closer to how it's running in the browser
 
-Be aware that only packages in `deps.optimizer?.[mode].include` option are bundled (some plugins populate this automatically, like Svelte). You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs. By default, Vitest uses `optimizer.web` for `jsdom` and `happy-dom` environments, and `optimizer.ssr` for `node` and `edge` environments, but it is configurable by [`transformMode`](#transformmode).
+Be aware that only packages in `deps.optimizer?.[mode].include` option are bundled (some plugins populate this automatically, like Svelte). You can read more about available options in [Vite](https://vitejs.dev/config/dep-optimization-options.html) docs (Vitest doesn't support `disable` and `noDiscovery` options). By default, Vitest uses `optimizer.web` for `jsdom` and `happy-dom` environments, and `optimizer.ssr` for `node` and `edge` environments, but it is configurable by [`transformMode`](#transformmode).
 
 This options also inherits your `optimizeDeps` configuration (for web Vitest will extend `optimizeDeps`, for ssr - `ssr.optimizeDeps`). If you redefine `include`/`exclude` option in `deps.optimizer` it will extend your `optimizeDeps` when running tests. Vitest automatically removes the same options from `include`, if they are listed in `exclude`.
 
 ::: tip
 You will not be able to edit your `node_modules` code for debugging, since the code is actually located in your `cacheDir` or `test.cache.dir` directory. If you want to debug with `console.log` statements, edit it directly or force rebundling with `deps.optimizer?.[mode].force` option.
 :::
 
+#### deps.optimizer.{mode}.enabled
+
+- **Type:** `boolean`
+- **Default:** `true`
+
+Enable dependency optimization.
+
+#### deps.web
+
+- **Type:** `{ transformAssets?, ... }`
+- **Version:** Since Vite 0.34.2
+
+Options that are applied to external files when transform mode is set to `web`. By default, `jsdom` and `happy-dom` use `web` mode, while `node` and `edge` environments use `ssr` transform mode, so these options will have no affect on files inside those environments.
+
+Usually, files inside `node_modules` are externalized, but these options also affect files in [`server.deps.external`](#server-deps-external).
+
+#### deps.web.transformAssets
+
+- **Type:** `boolean`
+- **Default:** `true`
+
+Should Vitest process assets (.png, .svg, .jpg, etc) files and resolve them like Vite does in the browser.
+
+hese module will have a default export equal to the path to the asset, if no query is specified.
+
+::: warning
+At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
+:::
+
+#### deps.web.transformCss
+
+- **Type:** `boolean`
+- **Default:** `true`
+
+Should Vitest process CSS (.css, .scss, .sass, etc) files and resolve them like Vite does in the browser.
+
+If CSS files are disabled with [`css`](#css) options, this option will just silence `UNKNOWN_EXTENSION` errors.
+
+::: warning
+At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
+:::
+
+#### deps.web.transformGlobPattern
+
+- **Type:** `RegExp | RegExp[]`
+- **Default:** `[]`
+
+Regexp pattern to match external files that should be transformed.
+
+By default, files inside `node_modules` are externalized and not transformed, unless it's CSS or an asset, and corresponding option is not disabled.
+
+::: warning
+At the moment, this option only works with [`experimentalVmThreads`](#experimentalvmthreads) pool.
+:::
+
 #### deps.registerNodeLoader<NonProjectOption />
 
 - **Type:** `boolean`

packages/vite-node/src/server.ts

@@ -150,6 +150,19 @@ export class ViteNodeServer {
     return this.transformPromiseMap.get(id)!
   }
 
+  async transformModule(id: string, transformMode?: 'web' | 'ssr') {
+    if (transformMode !== 'web')
+      throw new Error('`transformModule` only supports `transformMode: "web"`.')
+
+    const normalizedId = normalizeModuleId(id)
+    const mod = this.server.moduleGraph.getModuleById(normalizedId)
+    const result = mod?.transformResult || await this.server.transformRequest(normalizedId)
+
+    return {
+      code: result?.code,
+    }
+  }
+
   getTransformMode(id: string) {
     const withoutQuery = id.split('?')[0]
 

packages/vitest/src/node/config.ts

@@ -137,6 +137,11 @@ export function resolveConfig(
   if (!resolved.deps.moduleDirectories.includes('/node_modules/'))
     resolved.deps.moduleDirectories.push('/node_modules/')
 
+  resolved.deps.web ??= {}
+  resolved.deps.web.transformAssets ??= true
+  resolved.deps.web.transformCss ??= true
+  resolved.deps.web.transformGlobPattern ??= []
+
   resolved.server ??= {}
   resolved.server.deps ??= {}
 

packages/vitest/src/node/pools/rpc.ts

@@ -30,6 +30,9 @@ export function createMethodsRPC(project: WorkspaceProject): RuntimeRPC {
     resolveId(id, importer, transformMode) {
       return project.vitenode.resolveId(id, importer, transformMode)
     },
+    transform(id, environment) {
+      return project.vitenode.transformModule(id, environment)
+    },
     onPathsCollected(paths) {
       ctx.state.collectPaths(paths)
       project.report('onPathsCollected', paths)

packages/vitest/src/node/pools/vm-threads.ts

@@ -14,7 +14,7 @@ import type { WorkspaceProject } from '../workspace'
 import { createMethodsRPC } from './rpc'
 
 const workerPath = pathToFileURL(resolve(distDir, './vm.js')).href
-const suppressLoaderWarningsPath = resolve(rootDir, './suppress-warnings.cjs')
+const suppressWarningsPath = resolve(rootDir, './suppress-warnings.cjs')
 
 function createWorkerChannel(project: WorkspaceProject) {
   const channel = new MessageChannel()
@@ -61,7 +61,7 @@ export function createVmThreadsPool(ctx: Vitest, { execArgv, env }: PoolProcessO
       '--experimental-import-meta-resolve',
       '--experimental-vm-modules',
       '--require',
-      suppressLoaderWarningsPath,
+      suppressWarningsPath,
       ...execArgv,
     ],
 

packages/vitest/src/node/workspace.ts

@@ -293,10 +293,10 @@ export class WorkspaceProject {
         ...this.config.deps,
         optimizer: {
           web: {
-            enabled: this.config.deps?.optimizer?.web?.enabled ?? false,
+            enabled: this.config.deps?.optimizer?.web?.enabled ?? true,
           },
           ssr: {
-            enabled: this.config.deps?.optimizer?.ssr?.enabled ?? false,
+            enabled: this.config.deps?.optimizer?.ssr?.enabled ?? true,
           },
         },
       },

packages/vitest/src/runtime/execute.ts

@@ -6,7 +6,7 @@ import type { ViteNodeRunnerOptions } from 'vite-node'
 import { normalize, relative, resolve } from 'pathe'
 import { processError } from '@vitest/utils/error'
 import type { MockMap } from '../types/mocker'
-import type { ResolvedConfig, ResolvedTestEnvironment, WorkerGlobalState } from '../types'
+import type { ResolvedConfig, ResolvedTestEnvironment, RuntimeRPC, WorkerGlobalState } from '../types'
 import { distDir } from '../paths'
 import { getWorkerState } from '../utils/global'
 import { VitestMocker } from './mocker'
@@ -21,6 +21,7 @@ export interface ExecuteOptions extends ViteNodeRunnerOptions {
   moduleDirectories?: string[]
   context?: vm.Context
   state: WorkerGlobalState
+  transform: RuntimeRPC['transform']
 }
 
 export async function createVitestExecutor(options: ExecuteOptions) {
@@ -100,6 +101,9 @@ export async function startVitestExecutor(options: ContextExecutorOptions) {
     resolveId(id, importer) {
       return rpc().resolveId(id, importer, getTransformMode())
     },
+    transform(id) {
+      return rpc().transform(id, 'web')
+    },
     packageCache,
     moduleCache,
     mockMap,
@@ -174,12 +178,6 @@ export class VitestExecutor extends ViteNodeRunner {
       }
     }
     else {
-      this.externalModules = new ExternalModulesExecutor({
-        ...options,
-        fileMap,
-        context: options.context,
-        packageCache: options.packageCache,
-      })
       const clientStub = vm.runInContext(
         `(defaultClient) => ({ ...defaultClient, updateStyle: ${updateStyle.toString()}, removeStyle: ${removeStyle.toString()} })`,
         options.context,
@@ -189,6 +187,12 @@ export class VitestExecutor extends ViteNodeRunner {
         '@vite/client': clientStub,
       }
       this.primitives = vm.runInContext('({ Object, Reflect, Symbol })', options.context)
+      this.externalModules = new ExternalModulesExecutor({
+        ...options,
+        fileMap,
+        context: options.context,
+        packageCache: options.packageCache,
+      })
     }
   }
 

packages/vitest/src/runtime/external-executor.ts

@@ -12,6 +12,7 @@ import { CommonjsExecutor } from './vm/commonjs-executor'
 import type { FileMap } from './vm/file-map'
 import { EsmExecutor } from './vm/esm-executor'
 import { interopCommonJsModule } from './vm/utils'
+import { ViteExecutor } from './vm/vite-executor'
 
 const SyntheticModule: typeof VMSyntheticModule = (vm as any).SyntheticModule
 
@@ -23,12 +24,20 @@ export interface ExternalModulesExecutorOptions extends ExecuteOptions {
   packageCache: Map<string, any>
 }
 
+interface ModuleInformation {
+  type: 'data' | 'builtin' | 'vite' | 'module' | 'commonjs'
+  url: string
+  path: string
+}
+
 // TODO: improve Node.js strict mode support in #2854
 export class ExternalModulesExecutor {
   private cjs: CommonjsExecutor
   private esm: EsmExecutor
+  private vite: ViteExecutor
   private context: vm.Context
   private fs: FileMap
+  private resolvers: ((id: string, parent: string) => string | undefined)[] = []
 
   constructor(private options: ExternalModulesExecutorOptions) {
     this.context = options.context
@@ -42,6 +51,13 @@ export class ExternalModulesExecutor {
       importModuleDynamically: this.importModuleDynamically,
       fileMap: options.fileMap,
     })
+    this.vite = new ViteExecutor({
+      esmExecutor: this.esm,
+      context: this.context,
+      transform: options.transform,
+      viteClientModule: options.requestStubs!['/@vite/client'],
+    })
+    this.resolvers = [this.vite.resolve]
   }
 
   // dynamic import can be used in both ESM and CJS, so we have it in the executor
@@ -56,6 +72,11 @@ export class ExternalModulesExecutor {
   }
 
   public async resolve(specifier: string, parent: string) {
+    for (const resolver of this.resolvers) {
+      const id = resolver(specifier, parent)
+      if (id)
+        return id
+    }
     return nativeResolve(specifier, parent)
   }
 
@@ -124,44 +145,59 @@ export class ExternalModulesExecutor {
     return m
   }
 
-  private async createModule(identifier: string): Promise<VMModule> {
+  private getModuleInformation(identifier: string): ModuleInformation {
     if (identifier.startsWith('data:'))
-      return this.esm.createDataModule(identifier)
+      return { type: 'data', url: identifier, path: identifier }
 
     const extension = extname(identifier)
-
-    if (extension === '.node' || isNodeBuiltin(identifier)) {
-      const exports = this.require(identifier)
-      return this.wrapCoreSynteticModule(identifier, exports)
-    }
+    if (extension === '.node' || isNodeBuiltin(identifier))
+      return { type: 'builtin', url: identifier, path: identifier }
 
     const isFileUrl = identifier.startsWith('file://')
-    const fileUrl = isFileUrl ? identifier : pathToFileURL(identifier).toString()
     const pathUrl = isFileUrl ? fileURLToPath(identifier.split('?')[0]) : identifier
+    const fileUrl = isFileUrl ? identifier : pathToFileURL(pathUrl).toString()
 
-    // TODO: support wasm in the future
-    // if (extension === '.wasm') {
-    //   const source = this.readBuffer(pathUrl)
-    //   const wasm = this.loadWebAssemblyModule(source, fileUrl)
-    //   this.moduleCache.set(fileUrl, wasm)
-    //   return wasm
-    // }
-
-    if (extension === '.cjs') {
-      const exports = this.require(pathUrl)
-      return this.wrapCommonJsSynteticModule(fileUrl, exports)
+    let type: 'module' | 'commonjs' | 'vite'
+    if (this.vite.canResolve(fileUrl)) {
+      type = 'vite'
+    }
+    else if (extension === '.mjs') {
+      type = 'module'
+    }
+    else if (extension === '.cjs') {
+      type = 'commonjs'
+    }
+    else {
+      const pkgData = this.findNearestPackageData(normalize(pathUrl))
+      type = pkgData.type === 'module' ? 'module' : 'commonjs'
     }
 
-    if (extension === '.mjs')
-      return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl))
-
-    const pkgData = this.findNearestPackageData(normalize(pathUrl))
-
-    if (pkgData.type === 'module')
-      return await this.esm.createEsmModule(fileUrl, this.fs.readFile(pathUrl))
+    return { type, path: pathUrl, url: fileUrl }
+  }
 
-    const exports = this.cjs.require(pathUrl)
-    return this.wrapCommonJsSynteticModule(fileUrl, exports)
+  private async createModule(identifier: string): Promise<VMModule> {
+    const { type, url, path } = this.getModuleInformation(identifier)
+
+    switch (type) {
+      case 'data':
+        return this.esm.createDataModule(identifier)
+      case 'builtin': {
+        const exports = this.require(identifier)
+        return this.wrapCoreSynteticModule(identifier, exports)
+      }
+      case 'vite':
+        return await this.vite.createViteModule(url)
+      case 'module':
+        return await this.esm.createEsModule(url, this.fs.readFile(path))
+      case 'commonjs': {
+        const exports = this.require(path)
+        return this.wrapCommonJsSynteticModule(identifier, exports)
+      }
+      default: {
+        const _deadend: never = type
+        return _deadend
+      }
+    }
   }
 
   async import(identifier: string) {

packages/vitest/src/runtime/vm/commonjs-executor.ts

@@ -21,7 +21,8 @@ interface PrivateNodeModule extends NodeModule {
 
 export class CommonjsExecutor {
   private context: vm.Context
-  private requireCache: Record<string, NodeModule> = Object.create(null)
+  private requireCache = new Map<string, NodeModule>()
+  private publicRequireCache = this.createProxyCache()
 
   private moduleCache = new Map<string, VMModule | Promise<VMModule>>()
   private builtinCache: Record<string, NodeModule> = Object.create(null)
@@ -75,7 +76,7 @@ export class CommonjsExecutor {
         script.identifier = filename
         const fn = script.runInContext(executor.context)
         const __dirname = dirname(filename)
-        executor.requireCache[filename] = this
+        executor.requireCache.set(filename, this)
         try {
           fn(this.exports, this.require, this, filename, __dirname)
           return this.exports
@@ -165,14 +166,31 @@ export class CommonjsExecutor {
       set: () => {},
       configurable: true,
     })
-    require.main = _require.main
-    require.cache = this.requireCache
+    require.main = undefined // there is no main, since we are running tests using ESM
+    require.cache = this.publicRequireCache
     return require
   }
 
+  private createProxyCache() {
+    return new Proxy(Object.create(null), {
+      defineProperty: () => true,
+      deleteProperty: () => true,
+      set: () => true,
+      get: (_, key: string) => this.requireCache.get(key),
+      has: (_, key: string) => this.requireCache.has(key),
+      ownKeys: () => Array.from(this.requireCache.keys()),
+      getOwnPropertyDescriptor() {
+        return {
+          configurable: true,
+          enumerable: true,
+        }
+      },
+    })
+  }
+
   // very naive implementation for Node.js require
   private loadCommonJSModule(module: NodeModule, filename: string): Record<string, unknown> {
-    const cached = this.requireCache[filename]
+    const cached = this.requireCache.get(filename)
     if (cached)
       return cached.exports