Simplify file parsing and project reference parsing (#1536)

Co-authored-by: Copilot <[email protected]>

Author: sheetalkamat

internal/compiler/fileloader.go

@@ -22,9 +22,8 @@ type fileLoader struct {
 	comparePathsOptions tspath.ComparePathsOptions
 	supportedExtensions []string
 
-	parseTasks                 *fileLoaderWorker[*parseTask]
-	projectReferenceParseTasks *fileLoaderWorker[*projectReferenceParseTask]
-	rootTasks                  []*parseTask
+	filesParser *filesParser
+	rootTasks   []*parseTask
 
 	totalFileCount atomic.Int32
 	libFileCount   atomic.Int32
@@ -80,17 +79,14 @@ func processAllProgramFiles(
 			UseCaseSensitiveFileNames: opts.Host.FS().UseCaseSensitiveFileNames(),
 			CurrentDirectory:          opts.Host.GetCurrentDirectory(),
 		},
-		parseTasks: &fileLoaderWorker[*parseTask]{
+		filesParser: &filesParser{
 			wg:       core.NewWorkGroup(singleThreaded),
 			maxDepth: maxNodeModuleJsDepth,
 		},
-		projectReferenceParseTasks: &fileLoaderWorker[*projectReferenceParseTask]{
-			wg: core.NewWorkGroup(singleThreaded),
-		},
 		rootTasks:           make([]*parseTask, 0, len(rootFiles)+len(compilerOptions.Lib)),
 		supportedExtensions: core.Flatten(tsoptions.GetSupportedExtensionsWithJsonIfResolveJsonModule(compilerOptions, supportedExtensions)),
 	}
-	loader.addProjectReferenceTasks()
+	loader.addProjectReferenceTasks(singleThreaded)
 	loader.resolver = module.NewResolver(loader.projectReferenceFileMapper.host, compilerOptions, opts.TypingsLocation, opts.ProjectName)
 
 	var libs []string
@@ -115,7 +111,7 @@ func processAllProgramFiles(
 		loader.addAutomaticTypeDirectiveTasks()
 	}
 
-	loader.parseTasks.runAndWait(&loader, loader.rootTasks)
+	loader.filesParser.parse(&loader, loader.rootTasks)
 	// Clear out loader and host to ensure its not used post program creation
 	loader.projectReferenceFileMapper.loader = nil
 	loader.projectReferenceFileMapper.host = nil
@@ -138,7 +134,7 @@ func processAllProgramFiles(
 	var libFileSet collections.Set[tspath.Path]
 	fileLoadDiagnostics := &ast.DiagnosticsCollection{}
 
-	loader.parseTasks.collect(&loader, loader.rootTasks, func(task *parseTask, _ []tspath.Path) {
+	loader.filesParser.collect(&loader, loader.rootTasks, func(task *parseTask) {
 		if task.isRedirected {
 			return
 		}
@@ -280,7 +276,7 @@ func (p *fileLoader) resolveAutomaticTypeDirectives(containingFileName string) (
 	return toParse, typeResolutionsInFile
 }
 
-func (p *fileLoader) addProjectReferenceTasks() {
+func (p *fileLoader) addProjectReferenceTasks(singleThreaded bool) {
 	p.projectReferenceFileMapper = &projectReferenceFileMapper{
 		opts: p.opts,
 		host: p.opts.Host,
@@ -290,9 +286,12 @@ func (p *fileLoader) addProjectReferenceTasks() {
 		return
 	}
 
+	parser := &projectReferenceParser{
+		loader: p,
+		wg:     core.NewWorkGroup(singleThreaded),
+	}
 	rootTasks := createProjectReferenceParseTasks(projectReferences)
-	p.projectReferenceParseTasks.runAndWait(p, rootTasks)
-	p.projectReferenceFileMapper.init(p, rootTasks)
+	parser.parse(rootTasks)
 
 	// Add files from project references as root if the module kind is 'none'.
 	// This ensures that files from project references are included in the root tasks
@@ -567,11 +566,6 @@ func getInferredLibraryNameResolveFrom(options *core.CompilerOptions, currentDir
 	return tspath.CombinePaths(containingDirectory, "__lib_node_modules_lookup_"+libFileName+"__.ts")
 }
 
-type resolution struct {
-	node           *ast.Node
-	resolvedModule *module.ResolvedModule
-}
-
 func getModeForTypeReferenceDirectiveInFile(ref *ast.FileReference, file *ast.SourceFile, meta ast.SourceFileMetaData, options *core.CompilerOptions) core.ResolutionMode {
 	if ref.ResolutionMode != core.ResolutionModeNone {
 		return ref.ResolutionMode

internal/compiler/fileloadertask.go

@@ -1,7 +1,11 @@
 package compiler
 
 import (
+	"math"
+	"sync"
+
 	"github.com/microsoft/typescript-go/internal/ast"
+	"github.com/microsoft/typescript-go/internal/collections"
 	"github.com/microsoft/typescript-go/internal/core"
 	"github.com/microsoft/typescript-go/internal/module"
 	"github.com/microsoft/typescript-go/internal/tsoptions"
@@ -120,30 +124,99 @@ func (t *parseTask) addSubTask(ref resolvedRef, isLib bool) {
 	t.subTasks = append(t.subTasks, subTask)
 }
 
-func (t *parseTask) getSubTasks() []*parseTask {
-	return t.subTasks
+type filesParser struct {
+	wg              core.WorkGroup
+	tasksByFileName collections.SyncMap[string, *queuedParseTask]
+	maxDepth        int
 }
 
-func (t *parseTask) shouldIncreaseDepth() bool {
-	return t.increaseDepth
+type queuedParseTask struct {
+	task                *parseTask
+	mu                  sync.Mutex
+	lowestDepth         int
+	fromExternalLibrary bool
 }
 
-func (t *parseTask) shouldElideOnDepth() bool {
-	return t.elideOnDepth
+func (w *filesParser) parse(loader *fileLoader, tasks []*parseTask) {
+	w.start(loader, tasks, 0, false)
+	w.wg.RunAndWait()
 }
 
-func (t *parseTask) isLoaded() bool {
-	return t.loaded
-}
+func (w *filesParser) start(loader *fileLoader, tasks []*parseTask, depth int, isFromExternalLibrary bool) {
+	for i, task := range tasks {
+		taskIsFromExternalLibrary := isFromExternalLibrary || task.fromExternalLibrary
+		newTask := &queuedParseTask{task: task, lowestDepth: math.MaxInt}
+		loadedTask, loaded := w.tasksByFileName.LoadOrStore(task.FileName(), newTask)
+		task = loadedTask.task
+		if loaded {
+			tasks[i] = task
+			// Add in the loaded task's external-ness.
+			taskIsFromExternalLibrary = taskIsFromExternalLibrary || task.fromExternalLibrary
+		}
+
+		w.wg.Queue(func() {
+			loadedTask.mu.Lock()
+			defer loadedTask.mu.Unlock()
+
+			startSubtasks := false
+
+			currentDepth := depth
+			if task.increaseDepth {
+				currentDepth++
+			}
+			if currentDepth < loadedTask.lowestDepth {
+				// If we're seeing this task at a lower depth than before,
+				// reprocess its subtasks to ensure they are loaded.
+				loadedTask.lowestDepth = currentDepth
+				startSubtasks = true
+			}
+
+			if !task.root && taskIsFromExternalLibrary && !loadedTask.fromExternalLibrary {
+				// If we're seeing this task now as an external library,
+				// reprocess its subtasks to ensure they are also marked as external.
+				loadedTask.fromExternalLibrary = true
+				startSubtasks = true
+			}
 
-func (t *parseTask) isRoot() bool {
-	return t.root
+			if task.elideOnDepth && currentDepth > w.maxDepth {
+				return
+			}
+
+			if !task.loaded {
+				task.load(loader)
+			}
+
+			if startSubtasks {
+				w.start(loader, task.subTasks, loadedTask.lowestDepth, loadedTask.fromExternalLibrary)
+			}
+		})
+	}
 }
 
-func (t *parseTask) isFromExternalLibrary() bool {
-	return t.fromExternalLibrary
+func (w *filesParser) collect(loader *fileLoader, tasks []*parseTask, iterate func(*parseTask)) []tspath.Path {
+	// Mark all tasks we saw as external after the fact.
+	w.tasksByFileName.Range(func(key string, value *queuedParseTask) bool {
+		if value.fromExternalLibrary {
+			value.task.fromExternalLibrary = true
+		}
+		return true
+	})
+	return w.collectWorker(loader, tasks, iterate, collections.Set[*parseTask]{})
 }
 
-func (t *parseTask) markFromExternalLibrary() {
-	t.fromExternalLibrary = true
+func (w *filesParser) collectWorker(loader *fileLoader, tasks []*parseTask, iterate func(*parseTask), seen collections.Set[*parseTask]) []tspath.Path {
+	var results []tspath.Path
+	for _, task := range tasks {
+		// ensure we only walk each task once
+		if !task.loaded || seen.Has(task) {
+			continue
+		}
+		seen.Add(task)
+		if subTasks := task.subTasks; len(subTasks) > 0 {
+			w.collectWorker(loader, subTasks, iterate, seen)
+		}
+		iterate(task)
+		results = append(results, loader.toPath(task.FileName()))
+	}
+	return results
 }

internal/compiler/parsetask.go renamed to internal/compiler/filesparser.go

@@ -25,44 +25,6 @@ type projectReferenceFileMapper struct {
 	realpathDtsToSource collections.SyncMap[tspath.Path, *tsoptions.SourceAndProjectReference]
 }
 
-func (mapper *projectReferenceFileMapper) init(loader *fileLoader, rootTasks []*projectReferenceParseTask) {
-	totalReferences := loader.projectReferenceParseTasks.tasksByFileName.Size() + 1
-	mapper.loader = loader
-	mapper.configToProjectReference = make(map[tspath.Path]*tsoptions.ParsedCommandLine, totalReferences)
-	mapper.referencesInConfigFile = make(map[tspath.Path][]tspath.Path, totalReferences)
-	mapper.sourceToOutput = make(map[tspath.Path]*tsoptions.OutputDtsAndProjectReference)
-	mapper.outputDtsToSource = make(map[tspath.Path]*tsoptions.SourceAndProjectReference)
-	mapper.referencesInConfigFile[mapper.opts.Config.ConfigFile.SourceFile.Path()] = loader.projectReferenceParseTasks.collect(
-		loader,
-		rootTasks,
-		func(task *projectReferenceParseTask, referencesInConfig []tspath.Path) {
-			path := loader.toPath(task.configName)
-			mapper.configToProjectReference[path] = task.resolved
-			if task.resolved == nil || mapper.opts.Config.ConfigFile == task.resolved.ConfigFile {
-				return
-			}
-			mapper.referencesInConfigFile[path] = referencesInConfig
-			for key, value := range task.resolved.SourceToOutput() {
-				mapper.sourceToOutput[key] = value
-			}
-			for key, value := range task.resolved.OutputDtsToSource() {
-				mapper.outputDtsToSource[key] = value
-			}
-			if mapper.opts.canUseProjectReferenceSource() {
-				declDir := task.resolved.CompilerOptions().DeclarationDir
-				if declDir == "" {
-					declDir = task.resolved.CompilerOptions().OutDir
-				}
-				if declDir != "" {
-					loader.dtsDirectories.Add(loader.toPath(declDir))
-				}
-			}
-		})
-	if mapper.opts.canUseProjectReferenceSource() && len(loader.projectReferenceFileMapper.outputDtsToSource) != 0 {
-		mapper.host = newProjectReferenceDtsFakingHost(loader)
-	}
-}
-
 func (mapper *projectReferenceFileMapper) getParseFileRedirect(file ast.HasFileName) string {
 	if mapper.opts.canUseProjectReferenceSource() {
 		// Map to source file from project reference