perf(experimental): if fsCacheModule is enabled, read from the memory when possible (#9076)

Author: sheremet-va

packages/vitest/src/node/cache/fsModuleCache.ts

@@ -19,6 +19,8 @@ const cacheCommentLength = cacheComment.length
 
 const METADATA_FILE = '_metadata.json'
 
+const parallelFsCacheRead = new Map<string, Promise<{ code: string; meta: CachedInlineModuleMeta } | undefined>>()
+
 /**
  * @experimental
  */
@@ -68,28 +70,38 @@ export class FileSystemModuleCache {
  }
  }
 
+ private readCachedFileConcurrently(cachedFilePath: string) {
+ if (!parallelFsCacheRead.has(cachedFilePath)) {
+ parallelFsCacheRead.set(cachedFilePath, readFile(cachedFilePath, 'utf-8').then((code) => {
+ const matchIndex = code.lastIndexOf(cacheComment)
+ if (matchIndex === -1) {
+ debugFs?.(`${c.red('[empty]')} ${cachedFilePath} exists, but doesn't have a ${cacheComment} comment, transforming by vite instead`)
+ return
+ }
+
+ return { code, meta: this.fromBase64(code.slice(matchIndex + cacheCommentLength)) }
+ }).finally(() => {
+ parallelFsCacheRead.delete(cachedFilePath)
+ }))
+ }
+ return parallelFsCacheRead.get(cachedFilePath)!
+ }
+
  async getCachedModule(cachedFilePath: string): Promise<
  CachedInlineModuleMeta
- | Extract<FetchResult, { externalize: string }>
  | undefined
  > {
  if (!existsSync(cachedFilePath)) {
  debugFs?.(`${c.red('[empty]')} ${cachedFilePath} doesn't exist, transforming by vite instead`)
  return
  }
 
- const code = await readFile(cachedFilePath, 'utf-8')
- const matchIndex = code.lastIndexOf(cacheComment)
- if (matchIndex === -1) {
- debugFs?.(`${c.red('[empty]')} ${cachedFilePath} exists, but doesn't have a ${cacheComment} comment, transforming by vite instead`)
+ const fileResult = await this.readCachedFileConcurrently(cachedFilePath)
+ if (!fileResult) {
  return
  }
+ const { code, meta } = fileResult
 
- const meta = this.fromBase64(code.slice(matchIndex + cacheCommentLength))
- if (meta.externalize) {
- debugFs?.(`${c.green('[read]')} ${meta.externalize} is externalized inside ${cachedFilePath}`)
- return { externalize: meta.externalize, type: meta.type }
- }
  debugFs?.(`${c.green('[read]')} ${meta.id} is cached in ${cachedFilePath}`)
 
  return {
@@ -108,11 +120,7 @@ export class FileSystemModuleCache {
  importers: string[] = [],
  mappings: boolean = false,
  ): Promise<void> {
- if ('externalize' in fetchResult) {
- debugFs?.(`${c.yellow('[write]')} ${fetchResult.externalize} is externalized inside ${cachedFilePath}`)
- await atomicWriteFile(cachedFilePath, `${cacheComment}${this.toBase64(fetchResult)}`)
- }
- else if ('code' in fetchResult) {
+ if ('code' in fetchResult) {
  const result = {
  file: fetchResult.file,
  id: fetchResult.id,
@@ -169,8 +177,6 @@ export class FileSystemModuleCache {
  id: string,
  fileContent: string,
  ): string | null {
- let hashString = ''
-
  // bail out if file has import.meta.glob because it depends on other files
  // TODO: figure out a way to still support it
  if (fileContent.includes('import.meta.glob(')) {
@@ -179,6 +185,8 @@ export class FileSystemModuleCache {
  return null
  }
 
+ let hashString = ''
+
  for (const generator of this.fsCacheKeyGenerators) {
  const result = generator({ environment, id, sourceCode: fileContent })
  if (typeof result === 'string') {
@@ -214,13 +222,6 @@ export class FileSystemModuleCache {
  environment: environment.name,
  // this affects Vitest CSS plugin
  css: vitestConfig.css,
- // this affect externalization
- resolver: {
- inline: resolver.options.inline,
- external: resolver.options.external,
- inlineFiles: resolver.options.inlineFiles,
- moduleDirectories: resolver.options.moduleDirectories,
- },
  },
  (_, value) => {
  if (typeof value === 'function' || value instanceof RegExp) {

packages/vitest/src/node/environments/fetchModule.ts

@@ -139,7 +139,7 @@ class ModuleFetcher {
  private recordResult(trace: Span, result: FetchResult | FetchCachedFileSystemResult): void {
  if ('externalize' in result) {
  trace.setAttributes({
- 'vitest.module.external': result.externalize,
+ 'vitest.fetched_module.external': result.externalize,
  'vitest.fetched_module.type': result.type,
  })
  }
@@ -213,44 +213,55 @@ class ModuleFetcher {
  environment: DevEnvironment,
  moduleGraphModule: EnvironmentModuleNode,
  ): Promise<FetchResult | FetchCachedFileSystemResult | undefined> {
- const cachedModule = await this.fsCache.getCachedModule(cachePath)
-
- if (cachedModule && 'code' in cachedModule) {
- // keep the module graph in sync
- if (!moduleGraphModule.transformResult) {
- let map: Rollup.SourceMap | null | { mappings: '' } = extractSourceMap(cachedModule.code)
- if (map && cachedModule.file) {
- map.file = cachedModule.file
- }
- // mappings is a special source map identifier in rollup
- if (!map && cachedModule.mappings) {
- map = { mappings: '' }
- }
- moduleGraphModule.transformResult = {
- code: cachedModule.code,
- map,
- ssr: true,
- }
-
- // we populate the module graph to make the watch mode work because it relies on importers
- cachedModule.importers.forEach((importer) => {
- const environmentNode = environment.moduleGraph.getModuleById(importer)
- if (environmentNode) {
- moduleGraphModule.importers.add(environmentNode)
- }
- })
- }
+ if (moduleGraphModule.transformResult?.__vitestTmp) {
  return {
  cached: true as const,
- file: cachedModule.file,
- id: cachedModule.id,
- tmp: cachePath,
- url: cachedModule.url,
+ file: moduleGraphModule.file,
+ id: moduleGraphModule.id!,
+ tmp: moduleGraphModule.transformResult.__vitestTmp,
+ url: moduleGraphModule.url,
  invalidate: false,
  }
  }
 
- return cachedModule
+ const cachedModule = await this.fsCache.getCachedModule(cachePath)
+
+ if (!cachedModule) {
+ return
+ }
+
+ // keep the module graph in sync
+ let map: Rollup.SourceMap | null | { mappings: '' } = extractSourceMap(cachedModule.code)
+ if (map && cachedModule.file) {
+ map.file = cachedModule.file
+ }
+ // mappings is a special source map identifier in rollup
+ if (!map && cachedModule.mappings) {
+ map = { mappings: '' }
+ }
+ moduleGraphModule.transformResult = {
+ code: cachedModule.code,
+ map,
+ ssr: true,
+ __vitestTmp: cachePath,
+ }
+
+ // we populate the module graph to make the watch mode work because it relies on importers
+ cachedModule.importers.forEach((importer) => {
+ const environmentNode = environment.moduleGraph.getModuleById(importer)
+ if (environmentNode) {
+ moduleGraphModule.importers.add(environmentNode)
+ }
+ })
+
+ return {
+ cached: true as const,
+ file: cachedModule.file,
+ id: cachedModule.id,
+ tmp: cachePath,
+ url: cachedModule.url,
+ invalidate: false,
+ }
  }
 
  private async fetchAndProcess(
@@ -485,3 +496,9 @@ export function handleRollupError(e: unknown): never {
  }
  throw e
 }
+
+declare module 'vite' {
+ export interface TransformResult {
+ __vitestTmp?: string
+ }
+}