Add locks on concurrent alias following checker accesses under incremental mode (#2051)

Author: weswigham

internal/compiler/checkerpool.go

@@ -14,6 +14,7 @@ import (
 type CheckerPool interface {
 	GetChecker(ctx context.Context) (*checker.Checker, func())
 	GetCheckerForFile(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func())
+	GetCheckerForFileExclusive(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func())
 	GetAllCheckers(ctx context.Context) ([]*checker.Checker, func())
 	Files(checker *checker.Checker) iter.Seq[*ast.SourceFile]
 }
@@ -24,6 +25,7 @@ type checkerPool struct {
 
 	createCheckersOnce sync.Once
 	checkers           []*checker.Checker
+	locks              []sync.Mutex
 	fileAssociations   map[*ast.SourceFile]*checker.Checker
 }
 
@@ -34,6 +36,7 @@ func newCheckerPool(checkerCount int, program *Program) *checkerPool {
 		program:      program,
 		checkerCount: checkerCount,
 		checkers:     make([]*checker.Checker, checkerCount),
+		locks:        make([]sync.Mutex, checkerCount),
 	}
 
 	return pool
@@ -45,6 +48,16 @@ func (p *checkerPool) GetCheckerForFile(ctx context.Context, file *ast.SourceFil
 	return checker, noop
 }
 
+func (p *checkerPool) GetCheckerForFileExclusive(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) {
+	c, done := p.GetCheckerForFile(ctx, file)
+	idx := slices.Index(p.checkers, c)
+	p.locks[idx].Lock()
+	return c, sync.OnceFunc(func() {
+		p.locks[idx].Unlock()
+		done()
+	})
+}
+
 func (p *checkerPool) GetChecker(ctx context.Context) (*checker.Checker, func()) {
 	p.createCheckers()
 	checker := p.checkers[0]

internal/compiler/program.go

@@ -383,6 +383,12 @@ func (p *Program) GetTypeCheckerForFile(ctx context.Context, file *ast.SourceFil
 	return p.checkerPool.GetCheckerForFile(ctx, file)
 }
 
+// Return a checker for the given file, locked to the current thread to prevent data races from multiple threads
+// accessing the same checker. The lock will be released when the `done` function is called.
+func (p *Program) GetTypeCheckerForFileExclusive(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) {
+	return p.checkerPool.GetCheckerForFileExclusive(ctx, file)
+}
+
 func (p *Program) GetResolvedModule(file ast.HasFileName, moduleReference string, mode core.ResolutionMode) *module.ResolvedModule {
 	if resolutions, ok := p.resolvedModules[file.Path()]; ok {
 		if resolved, ok := resolutions[module.ModeAwareCacheKey{Name: moduleReference, Mode: mode}]; ok {
@@ -1273,6 +1279,10 @@ func (p *Program) InstantiationCount() int {
 	return count
 }
 
+func (p *Program) Program() *Program {
+	return p
+}
+
 func (p *Program) GetSourceFileMetaData(path tspath.Path) ast.SourceFileMetaData {
 	return p.sourceFileMetaDatas[path]
 }
@@ -1437,6 +1447,7 @@ func CombineEmitResults(results []*EmitResult) *EmitResult {
 
 type ProgramLike interface {
 	Options() *core.CompilerOptions
+	GetSourceFile(path string) *ast.SourceFile
 	GetSourceFiles() []*ast.SourceFile
 	GetConfigFileParsingDiagnostics() []*ast.Diagnostic
 	GetSyntacticDiagnostics(ctx context.Context, file *ast.SourceFile) []*ast.Diagnostic
@@ -1446,7 +1457,11 @@ type ProgramLike interface {
 	GetGlobalDiagnostics(ctx context.Context) []*ast.Diagnostic
 	GetSemanticDiagnostics(ctx context.Context, file *ast.SourceFile) []*ast.Diagnostic
 	GetDeclarationDiagnostics(ctx context.Context, file *ast.SourceFile) []*ast.Diagnostic
+	GetSuggestionDiagnostics(ctx context.Context, file *ast.SourceFile) []*ast.Diagnostic
 	Emit(ctx context.Context, options EmitOptions) *EmitResult
+	CommonSourceDirectory() string
+	IsSourceFileDefaultLibrary(path tspath.Path) bool
+	Program() *Program
 }
 
 func HandleNoEmitOnError(ctx context.Context, program ProgramLike, file *ast.SourceFile) *EmitResult {

internal/execute/incremental/affectedfileshandler.go

@@ -229,7 +229,7 @@ func (h *affectedFilesHandler) handleDtsMayChangeOfAffectedFile(dtsMayChange dts
 				break
 			}
 			if typeChecker == nil {
-				typeChecker, done = h.program.program.GetTypeCheckerForFile(h.ctx, affectedFile)
+				typeChecker, done = h.program.program.GetTypeCheckerForFileExclusive(h.ctx, affectedFile)
 			}
 			aliased := checker.SkipAlias(exported, typeChecker)
 			if aliased == exported {

internal/execute/incremental/program.go

@@ -84,12 +84,36 @@ func (p *Program) Options() *core.CompilerOptions {
 	return p.snapshot.options
 }
 
+// CommonSourceDirectory implements compiler.AnyProgram interface.
+func (p *Program) CommonSourceDirectory() string {
+	p.panicIfNoProgram("CommonSourceDirectory")
+	return p.program.CommonSourceDirectory()
+}
+
+// Program implements compiler.AnyProgram interface.
+func (p *Program) Program() *compiler.Program {
+	p.panicIfNoProgram("Program")
+	return p.program
+}
+
+// IsSourceFileDefaultLibrary implements compiler.AnyProgram interface.
+func (p *Program) IsSourceFileDefaultLibrary(path tspath.Path) bool {
+	p.panicIfNoProgram("IsSourceFileDefaultLibrary")
+	return p.program.IsSourceFileDefaultLibrary(path)
+}
+
 // GetSourceFiles implements compiler.AnyProgram interface.
 func (p *Program) GetSourceFiles() []*ast.SourceFile {
 	p.panicIfNoProgram("GetSourceFiles")
 	return p.program.GetSourceFiles()
 }
 
+// GetSourceFile implements compiler.AnyProgram interface.
+func (p *Program) GetSourceFile(path string) *ast.SourceFile {
+	p.panicIfNoProgram("GetSourceFile")
+	return p.program.GetSourceFile(path)
+}
+
 // GetConfigFileParsingDiagnostics implements compiler.AnyProgram interface.
 func (p *Program) GetConfigFileParsingDiagnostics() []*ast.Diagnostic {
 	p.panicIfNoProgram("GetConfigFileParsingDiagnostics")
@@ -172,6 +196,12 @@ func (p *Program) GetDeclarationDiagnostics(ctx context.Context, file *ast.Sourc
 	return nil
 }
 
+// GetSuggestionDiagnostics implements compiler.AnyProgram interface.
+func (p *Program) GetSuggestionDiagnostics(ctx context.Context, file *ast.SourceFile) []*ast.Diagnostic {
+	p.panicIfNoProgram("GetSuggestionDiagnostics")
+	return p.program.GetSuggestionDiagnostics(ctx, file) // TODO: incremental suggestion diagnostics (only relevant in editor incremental builder?)
+}
+
 // GetModeForUsageLocation implements compiler.AnyProgram interface.
 func (p *Program) Emit(ctx context.Context, options compiler.EmitOptions) *compiler.EmitResult {
 	p.panicIfNoProgram("Emit")

internal/execute/incremental/programtosnapshot.go

@@ -263,7 +263,7 @@ func getReferencedFiles(program *compiler.Program, file *ast.SourceFile) *collec
 	// We need to use a set here since the code can contain the same import twice,
 	// but that will only be one dependency.
 	// To avoid invernal conversion, the key of the referencedFiles map must be of type Path
-	checker, done := program.GetTypeCheckerForFile(context.TODO(), file)
+	checker, done := program.GetTypeCheckerForFileExclusive(context.TODO(), file)
 	defer done()
 	for _, importName := range file.Imports() {
 		addReferencedFilesFromImportLiteral(file, &referencedFiles, checker, importName)

internal/project/checkerpool.go

@@ -20,6 +20,7 @@ type CheckerPool struct {
 	cond                *sync.Cond
 	createCheckersOnce  sync.Once
 	checkers            []*checker.Checker
+	locks               []sync.Mutex
 	inUse               map[*checker.Checker]bool
 	fileAssociations    map[*ast.SourceFile]int
 	requestAssociations map[string]int
@@ -33,6 +34,7 @@ func newCheckerPool(maxCheckers int, program *compiler.Program, log func(msg str
 		program:             program,
 		maxCheckers:         maxCheckers,
 		checkers:            make([]*checker.Checker, maxCheckers),
+		locks:               make([]sync.Mutex, maxCheckers),
 		inUse:               make(map[*checker.Checker]bool),
 		requestAssociations: make(map[string]int),
 		log:                 log,
@@ -75,6 +77,12 @@ func (p *CheckerPool) GetCheckerForFile(ctx context.Context, file *ast.SourceFil
 	return checker, p.createRelease(requestID, index, checker)
 }
 
+// GetCheckerForFileExclusive is the same as GetCheckerForFile but also locks a mutex associated with the checker.
+// Call `done` to free the lock.
+func (p *CheckerPool) GetCheckerForFileExclusive(ctx context.Context, file *ast.SourceFile) (*checker.Checker, func()) {
+	panic("unimplemented") // implement if used by LS
+}
+
 func (p *CheckerPool) GetChecker(ctx context.Context) (*checker.Checker, func()) {
 	p.mu.Lock()
 	defer p.mu.Unlock()

internal/testrunner/compiler_runner.go

@@ -528,7 +528,8 @@ func createHarnessTestFile(unit *testUnit, currentDirectory string) *harnessutil
 
 func (c *compilerTest) verifyUnionOrdering(t *testing.T) {
 	t.Run("union ordering", func(t *testing.T) {
-		checkers, done := c.result.Program.GetTypeCheckers(t.Context())
+		p := c.result.Program.Program()
+		checkers, done := p.GetTypeCheckers(t.Context())
 		defer done()
 		for _, c := range checkers {
 			for union := range c.UnionTypes() {

internal/testutil/harnessutil/harnessutil.go

@@ -21,6 +21,7 @@ import (
 	"github.com/microsoft/typescript-go/internal/collections"
 	"github.com/microsoft/typescript-go/internal/compiler"
 	"github.com/microsoft/typescript-go/internal/core"
+	"github.com/microsoft/typescript-go/internal/execute/incremental"
 	"github.com/microsoft/typescript-go/internal/outputpaths"
 	"github.com/microsoft/typescript-go/internal/parser"
 	"github.com/microsoft/typescript-go/internal/repo"
@@ -687,13 +688,13 @@ func compileFilesWithHost(
 	}
 	emitResult := program.Emit(ctx, compiler.EmitOptions{})
 
-	return newCompilationResult(config.CompilerOptions(), program, emitResult, diagnostics, harnessOptions)
+	return newCompilationResult(host, config.CompilerOptions(), program, emitResult, diagnostics, harnessOptions)
 }
 
 type CompilationResult struct {
 	Diagnostics      []*ast.Diagnostic
 	Result           *compiler.EmitResult
-	Program          *compiler.Program
+	Program          compiler.ProgramLike
 	Options          *core.CompilerOptions
 	HarnessOptions   *HarnessOptions
 	JS               collections.OrderedMap[string, *TestFile]
@@ -705,6 +706,7 @@ type CompilationResult struct {
 	inputs           []*TestFile
 	inputsAndOutputs collections.OrderedMap[string, *CompilationOutput]
 	Trace            string
+	Host             compiler.CompilerHost
 }
 
 type CompilationOutput struct {
@@ -715,8 +717,9 @@ type CompilationOutput struct {
 }
 
 func newCompilationResult(
+	host compiler.CompilerHost,
 	options *core.CompilerOptions,
-	program *compiler.Program,
+	program compiler.ProgramLike,
 	result *compiler.EmitResult,
 	diagnostics []*ast.Diagnostic,
 	harnessOptions *HarnessOptions,
@@ -731,9 +734,10 @@ func newCompilationResult(
 		Program:        program,
 		Options:        options,
 		HarnessOptions: harnessOptions,
+		Host:           host,
 	}
 
-	fs := program.Host().FS().(*OutputRecorderFS)
+	fs := host.FS().(*OutputRecorderFS)
 	if fs != nil && program != nil {
 		// Corsa, unlike Strada, can use multiple threads for emit. As a result, the order of outputs is non-deterministic.
 		// To make the order deterministic, we sort the outputs by the order of the inputs.
@@ -803,7 +807,7 @@ func compareTestFiles(a *TestFile, b *TestFile) int {
 }
 
 func (c *CompilationResult) getOutputPath(path string, ext string) string {
-	path = tspath.ResolvePath(c.Program.GetCurrentDirectory(), path)
+	path = tspath.ResolvePath(c.Host.GetCurrentDirectory(), path)
 	var outDir string
 	if ext == ".d.ts" || ext == ".d.mts" || ext == ".d.cts" || (strings.HasSuffix(ext, ".ts") && strings.Contains(ext, ".d.")) {
 		outDir = c.Options.DeclarationDir
@@ -817,17 +821,17 @@ func (c *CompilationResult) getOutputPath(path string, ext string) string {
 		common := c.Program.CommonSourceDirectory()
 		if common != "" {
 			path = tspath.GetRelativePathFromDirectory(common, path, tspath.ComparePathsOptions{
-				UseCaseSensitiveFileNames: c.Program.UseCaseSensitiveFileNames(),
-				CurrentDirectory:          c.Program.GetCurrentDirectory(),
+				UseCaseSensitiveFileNames: c.Host.FS().UseCaseSensitiveFileNames(),
+				CurrentDirectory:          c.Host.GetCurrentDirectory(),
 			})
-			path = tspath.CombinePaths(tspath.ResolvePath(c.Program.GetCurrentDirectory(), c.Options.OutDir), path)
+			path = tspath.CombinePaths(tspath.ResolvePath(c.Host.GetCurrentDirectory(), c.Options.OutDir), path)
 		}
 	}
 	return tspath.ChangeExtension(path, ext)
 }
 
 func (r *CompilationResult) FS() vfs.FS {
-	return r.Program.Host().FS()
+	return r.Host.FS()
 }
 
 func (r *CompilationResult) GetNumberOfJSFiles(includeJson bool) int {
@@ -852,7 +856,7 @@ func (c *CompilationResult) Outputs() []*TestFile {
 }
 
 func (c *CompilationResult) GetInputsAndOutputsForFile(path string) *CompilationOutput {
-	return c.inputsAndOutputs.GetOrZero(tspath.ResolvePath(c.Program.GetCurrentDirectory(), path))
+	return c.inputsAndOutputs.GetOrZero(tspath.ResolvePath(c.Host.GetCurrentDirectory(), path))
 }
 
 func (c *CompilationResult) GetInputsForFile(path string) []*TestFile {
@@ -915,7 +919,24 @@ func (c *CompilationResult) GetSourceMapRecord() string {
 	return sourceMapRecorder.String()
 }
 
-func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLine) *compiler.Program {
+type testBuildInfoReader struct {
+	inner incremental.BuildInfoReader
+}
+
+func (t *testBuildInfoReader) ReadBuildInfo(config *tsoptions.ParsedCommandLine) *incremental.BuildInfo {
+	r := t.inner.ReadBuildInfo(config)
+	if r == nil {
+		return nil
+	}
+	r.Version = core.Version()
+	return r
+}
+
+func getTestBuildInfoReader(host compiler.CompilerHost) *testBuildInfoReader {
+	return &testBuildInfoReader{inner: incremental.NewBuildInfoReader(host)}
+}
+
+func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLine) compiler.ProgramLike {
 	var singleThreaded core.Tristate
 	if testutil.TestProgramIsSingleThreaded() {
 		singleThreaded = core.TSTrue
@@ -927,6 +948,11 @@ func createProgram(host compiler.CompilerHost, config *tsoptions.ParsedCommandLi
 		SingleThreaded: singleThreaded,
 	}
 	program := compiler.NewProgram(programOptions)
+	if config.CompilerOptions().Incremental.IsTrue() {
+		oldProgram := incremental.ReadBuildInfoProgram(config, getTestBuildInfoReader(host), host)
+		incrementalProgram := incremental.NewProgram(program, oldProgram, incremental.CreateHost(host), false)
+		return incrementalProgram
+	}
 	return program
 }
 

internal/testutil/tsbaseline/js_emit_baseline.go

@@ -201,7 +201,7 @@ func prepareDeclarationCompilationContext(
 		var sourceFileName string
 
 		if len(options.OutDir) != 0 {
-			sourceFilePath := tspath.GetNormalizedAbsolutePath(sourceFile.FileName(), result.Program.GetCurrentDirectory())
+			sourceFilePath := tspath.GetNormalizedAbsolutePath(sourceFile.FileName(), result.Host.GetCurrentDirectory())
 			sourceFilePath = strings.Replace(sourceFilePath, result.Program.CommonSourceDirectory(), "", 1)
 			sourceFileName = tspath.CombinePaths(options.OutDir, sourceFilePath)
 		} else {

internal/testutil/tsbaseline/type_symbol_baseline.go

@@ -31,7 +31,7 @@ func DoTypeAndSymbolBaseline(
 	t *testing.T,
 	baselinePath string,
 	header string,
-	program *compiler.Program,
+	program compiler.ProgramLike,
 	allFiles []*harnessutil.TestFile,
 	opts baseline.Options,
 	skipTypeBaselines bool,
@@ -259,13 +259,13 @@ func iterateBaseline(allFiles []*harnessutil.TestFile, fullWalker *typeWriterWal
 }
 
 type typeWriterWalker struct {
-	program              *compiler.Program
+	program              compiler.ProgramLike
 	hadErrorBaseline     bool
 	currentSourceFile    *ast.SourceFile
 	declarationTextCache map[*ast.Node]string
 }
 
-func newTypeWriterWalker(program *compiler.Program, hadErrorBaseline bool) *typeWriterWalker {
+func newTypeWriterWalker(program compiler.ProgramLike, hadErrorBaseline bool) *typeWriterWalker {
 	return &typeWriterWalker{
 		program:              program,
 		hadErrorBaseline:     hadErrorBaseline,
@@ -276,7 +276,7 @@ func newTypeWriterWalker(program *compiler.Program, hadErrorBaseline bool) *type
 func (walker *typeWriterWalker) getTypeCheckerForCurrentFile() (*checker.Checker, func()) {
 	// If we don't use the right checker for the file, its contents won't be up to date
 	// since the types/symbols baselines appear to depend on files having been checked.
-	return walker.program.GetTypeCheckerForFile(context.Background(), walker.currentSourceFile)
+	return walker.program.Program().GetTypeCheckerForFile(context.Background(), walker.currentSourceFile)
 }
 
 type typeWriterResult struct {