Merge pull request #25144 from Microsoft/gulpfileFixes

Fix runtests-parallel, tsserverlibrary, up-to-date checks in gulp builds

Author: rbuckton

Gulpfile.js

@@ -14,7 +14,6 @@ const browserify = require("browserify");
 const through2 = require("through2");
 const fold = require("travis-fold");
 const rename = require("gulp-rename");
-const concat = require("gulp-concat");
 const convertMap = require("convert-source-map");
 const sorcery = require("sorcery");
 const Vinyl = require("vinyl");
@@ -155,7 +154,7 @@ gulp.task(typescriptServicesProject, /*help*/ false, () => {
 const typescriptServicesJs = "built/local/typescriptServices.js";
 const typescriptServicesDts = "built/local/typescriptServices.d.ts";
 gulp.task(typescriptServicesJs, /*help*/ false, ["lib", "generate-diagnostics", typescriptServicesProject], () => 
-    project.compile(typescriptServicesProject, { dts: convertConstEnums() }), 
+    project.compile(typescriptServicesProject, { dts: files => files.pipe(convertConstEnums()) }),
     { aliases: [typescriptServicesDts] });
 
 const typescriptJs = "built/local/typescript.js";
@@ -182,7 +181,7 @@ gulp.task(typescriptStandaloneDts, /*help*/ false, [typescriptServicesDts], () =
         .pipe(gulp.dest("built/local")));
 
 // build all 'typescriptServices'-related outputs
-gulp.task("typescriptServices", /*help*/ false, [typescriptServicesJs, typescriptServicesDts, typescriptJs, typescriptDts, typescriptStandaloneDts]);
+gulp.task("services", /*help*/ false, [typescriptServicesJs, typescriptServicesDts, typescriptJs, typescriptDts, typescriptStandaloneDts]);
 
 const tscProject = "src/tsc/tsconfig.json";
 const tscJs = "built/local/tsc.js";
@@ -211,13 +210,28 @@ gulp.task(typesMapJson, /*help*/ false, [], () =>
         .pipe(insert.transform(contents => (JSON.parse(contents), contents)))
         .pipe(gulp.dest("built/local")));
 
+const tsserverlibraryProject = "built/local/tsserverlibrary.tsconfig.json";
+gulp.task(tsserverlibraryProject, /*help*/ false, () => {
+    // NOTE: flatten tsserverlibrary so that we can properly strip @internal
+    project.flatten("src/tsserver/tsconfig.json", tsserverlibraryProject, {
+        exclude: ["src/tsserver/server.ts"],
+        compilerOptions: {
+            "removeComments": true,
+            "stripInternal": true,
+            "outFile": "tsserverlibrary.js"
+        }
+    });
+});
+
+const tsserverlibraryJs = "built/local/tsserverlibrary.js";
 const tsserverlibraryDts = "built/local/tsserverlibrary.d.ts";
-gulp.task(tsserverlibraryDts, /*help*/ false, [tsserverJs], () =>
-    gulp.src(["built/local/compiler.d.ts", "built/local/jsTyping.d.ts", "built/local/services.d.ts", "built/local/server.d.ts"], { base: "built/local" })
-        .pipe(convertConstEnums())
-        .pipe(concat("tsserverlibrary.d.ts", { newLine: "\n" }))
-        .pipe(append("\nexport = ts;\nexport as namespace ts;"))
-        .pipe(gulp.dest("built/local")));
+gulp.task(tsserverlibraryJs, /*help*/ false, [typescriptServicesJs, tsserverlibraryProject], () =>
+    project.compile(tsserverlibraryProject, { 
+        dts: files => files
+            .pipe(convertConstEnums())
+            .pipe(append("\nexport = ts;\nexport as namespace ts;")),
+        typescript: "built"
+    }), { aliases: [tsserverlibraryDts] });
 
 gulp.task(
     "lssl",
@@ -227,7 +241,7 @@ gulp.task(
 gulp.task(
     "local",
     "Builds the full compiler and services",
-    [tscJs, "typescriptServices", tsserverJs, builtGeneratedDiagnosticMessagesJson, tsserverlibraryDts, "localize"]);
+    [tscJs, "services", tsserverJs, builtGeneratedDiagnosticMessagesJson, tsserverlibraryDts, "localize"]);
 
 gulp.task(
     "tsc",

scripts/build/gulp-typescript-oop.js

@@ -24,7 +24,10 @@ function createProject(tsConfigFileName, settings, options) {
 
     const project = tsConfigFileName === undefined ? tsc.createProject(localSettings) : tsc.createProject(tsConfigFileName, localSettings);
     const wrappedProject = /** @type {tsc.Project} */(() => {
-        const proc = child_process.fork(require.resolve("./main.js"));
+        const proc = child_process.fork(require.resolve("./main.js"), [], { 
+            // Prevent errors when debugging gulpfile due to the same debug port being passed to forked children.
+            execArgv: [] 
+        });
         /** @type {Duplex & { js?: Readable, dts?: Readable }} */
         const compileStream = new Duplex({
             objectMode: true,

scripts/build/project.js

@@ -13,36 +13,41 @@ const del = require("del");
 const needsUpdate = require("./needsUpdate");
 const mkdirp = require("./mkdirp");
 const { reportDiagnostics } = require("./diagnostics");
-const { PassThrough } = require("stream");
 
 class CompilationGulp extends gulp.Gulp {
     /**
-     * @param {import("gulp-help").GulpHelp | import("gulp").Gulp} gulp
+     * @param {boolean} [verbose] 
      */
-    constructor(gulp) {
-        super();
-        // forward notifications to the outer gulp.
-        this.on("task_start", e => gulp.emit("task_start", e));
-        this.on("task_stop", e => gulp.emit("task_stop", e));
-        this.on("task_err", e => gulp.emit("task_err", e));
-        this.on("task_not_found", e => gulp.emit("task_not_found", e));
-        this.on("task_recursion", e => gulp.emit("task_recursion", e));
-        this.on("err", e => gulp.emit("err", e));
+    fork(verbose) {
+        const child = new ForkedGulp(this.tasks);
+        if (verbose) {
+            this.on("task_start", e => gulp.emit("task_start", e));
+            this.on("task_stop", e => gulp.emit("task_stop", e));
+            this.on("task_err", e => gulp.emit("task_err", e));
+            this.on("task_not_found", e => gulp.emit("task_not_found", e));
+            this.on("task_recursion", e => gulp.emit("task_recursion", e));
+        }
+        return child;
     }
+}
 
-    dispose() {
-        this.removeAllListeners();
-        this.reset();
+class ForkedGulp extends gulp.Gulp {
+    /**
+     * @param {gulp.Gulp["tasks"]} tasks
+     */
+    constructor(tasks) {
+        super();
+        this.tasks = tasks;
     }
-    
-    // Do not reset tasks when `gulp.start()` is called
+
+    // Do not reset tasks
     _resetAllTasks() {}
     _resetSpecificTasks() {}
     _resetTask() {}
 }
 
 // internal `Gulp` instance for compilation artifacts.
-const compilationGulp = new CompilationGulp(gulp);
+const compilationGulp = new CompilationGulp();
 
 /** @type {Map<ResolvedProjectSpec, ProjectGraph>} */
 const projectGraphCache = new Map();
@@ -60,7 +65,9 @@ function createCompiler(projectSpec, options) {
     const resolvedOptions = resolveProjectOptions(options);
     const resolvedProjectSpec = resolveProjectSpec(projectSpec, resolvedOptions.paths, /*referrer*/ undefined);
     const taskName = compileTaskName(ensureCompileTask(getOrCreateProjectGraph(resolvedProjectSpec, resolvedOptions.paths), resolvedOptions), resolvedOptions.typescript);
-    return () => new Promise((resolve, reject) => compilationGulp.start(taskName, err => err ? reject(err) : resolve(err)));
+    return () => new Promise((resolve, reject) => compilationGulp
+        .fork(resolvedOptions.verbose)
+        .start(taskName, err => err ? reject(err) : resolve()));
 }
 exports.createCompiler = createCompiler;
 
@@ -74,13 +81,13 @@ exports.createCompiler = createCompiler;
  * @property {string} [cwd] The path to use for the current working directory. Defaults to `process.cwd()`.
  * @property {string} [base] The path to use as the base for relative paths. Defaults to `cwd`.
  * @property {string} [typescript] A module specifier or path (relative to gulpfile.js) to the version of TypeScript to use.
- * @property {Hook} [js] Pipeline hook for .js file outputs. For multiple steps, use `stream-combiner`.
- * @property {Hook} [dts] Pipeline hook for .d.ts file outputs. For multiple steps, use `stream-combiner`.
+ * @property {Hook} [js] Pipeline hook for .js file outputs.
+ * @property {Hook} [dts] Pipeline hook for .d.ts file outputs.
  * @property {boolean} [verbose] Indicates whether verbose logging is enabled.
  * @property {boolean} [force] Force recompilation (no up-to-date check).
  * @property {boolean} [inProcess] Indicates whether to run gulp-typescript in-process or out-of-process (default).
  * 
- * @typedef {NodeJS.ReadWriteStream | (() => NodeJS.ReadWriteStream)} Hook
+ * @typedef {(stream: NodeJS.ReadableStream) => NodeJS.ReadWriteStream} Hook
  */
 function compile(projectSpec, options) {
     const compiler = createCompiler(projectSpec, options);
@@ -97,7 +104,9 @@ function createCleaner(projectSpec, options) {
     const paths = resolvePathOptions(options);
     const resolvedProjectSpec = resolveProjectSpec(projectSpec, paths, /*referrer*/ undefined);
     const taskName = cleanTaskName(ensureCleanTask(getOrCreateProjectGraph(resolvedProjectSpec, paths)));
-    return () => new Promise((resolve, reject) => compilationGulp.start(taskName, err => err ? reject(err) : resolve(err)));
+    return () => new Promise((resolve, reject) => compilationGulp
+        .fork()
+        .start(taskName, err => err ? reject(err) : resolve()));
 }
 exports.createCleaner = createCleaner;
 
@@ -134,6 +143,7 @@ exports.addTypeScript = addTypeScript;
  * @property {string} [cwd] The path to use for the current working directory. Defaults to `process.cwd()`.
  * @property {CompilerOptions} [compilerOptions] Compiler option overrides.
  * @property {boolean} [force] Forces creation of the output project.
+ * @property {string[]} [exclude] Files to exclude (relative to `cwd`)
  */
 function flatten(projectSpec, flattenedProjectSpec, options = {}) {
     const paths = resolvePathOptions(options);
@@ -142,15 +152,16 @@ function flatten(projectSpec, flattenedProjectSpec, options = {}) {
     const resolvedOutputDirectory = path.dirname(resolvedOutputSpec);
     const resolvedProjectSpec = resolveProjectSpec(projectSpec, paths, /*referrer*/ undefined);
     const projectGraph = getOrCreateProjectGraph(resolvedProjectSpec, paths);
+    const skipProjects = /**@type {Set<ProjectGraph>}*/(new Set());
+    const skipFiles = new Set(options && options.exclude && options.exclude.map(file => path.resolve(paths.cwd, file)));
     recur(projectGraph);
 
-    const config = {
-        extends: normalizeSlashes(path.relative(resolvedOutputDirectory, resolvedProjectSpec)),
-        compilerOptions: options.compilerOptions || {},
-        files
-    };
-
     if (options.force || needsUpdate(files, resolvedOutputSpec)) {
+        const config = {
+            extends: normalizeSlashes(path.relative(resolvedOutputDirectory, resolvedProjectSpec)),
+            compilerOptions: options.compilerOptions || {},
+            files: files.map(file => normalizeSlashes(path.relative(resolvedOutputDirectory, file)))
+        };
         mkdirp.sync(resolvedOutputDirectory);
         fs.writeFileSync(resolvedOutputSpec, JSON.stringify(config, undefined, 2), "utf8");
     }
@@ -159,11 +170,16 @@ function flatten(projectSpec, flattenedProjectSpec, options = {}) {
      * @param {ProjectGraph} projectGraph 
      */
     function recur(projectGraph) {
+        if (skipProjects.has(projectGraph)) return;
+        skipProjects.add(projectGraph);
         for (const ref of projectGraph.references) {
             recur(ref.target);
         }
-        for (const file of projectGraph.project.fileNames) {
-            files.push(normalizeSlashes(path.relative(resolvedOutputDirectory, path.resolve(projectGraph.projectDirectory, file))));
+        for (let file of projectGraph.project.fileNames) {
+            file = path.resolve(projectGraph.projectDirectory, file);
+            if (skipFiles.has(file)) continue;
+            skipFiles.add(file);
+            files.push(file);
         }
     }
 }
@@ -257,14 +273,6 @@ function resolveProjectOptions(options = {}) {
     };
 }
 
-/**
- * @param {Hook} hook 
- * @returns {NodeJS.ReadWriteStream}
- */
-function evaluateHook(hook) {
-    return (typeof hook === "function" ? hook() : hook) || new PassThrough({ objectMode: true });
-}
-
 /**
  * @param {ResolvedProjectOptions} left
  * @param {ResolvedProjectOptions} right
@@ -448,31 +456,29 @@ function resolveDestPath(projectGraph, paths) {
 
 /**
  * @param {ProjectGraph} projectGraph
- * @param {ResolvedProjectOptions} resolvedOptions
+ * @param {ResolvedProjectOptions} options
  */
-function ensureCompileTask(projectGraph, resolvedOptions) {
-    const projectGraphConfig = getOrCreateProjectGraphConfiguration(projectGraph, resolvedOptions);
-    projectGraphConfig.resolvedOptions = resolvedOptions = mergeProjectOptions(resolvedOptions, resolvedOptions);
+function ensureCompileTask(projectGraph, options) {
+    const projectGraphConfig = getOrCreateProjectGraphConfiguration(projectGraph, options);
+    projectGraphConfig.resolvedOptions = options = mergeProjectOptions(options, options);
     if (!projectGraphConfig.compileTaskCreated) {
-        const deps = makeProjectReferenceCompileTasks(projectGraph, resolvedOptions.typescript, resolvedOptions.paths);
-        compilationGulp.task(compileTaskName(projectGraph, resolvedOptions.typescript), deps, () => {
-            const destPath = resolveDestPath(projectGraph, resolvedOptions.paths);
+        const deps = makeProjectReferenceCompileTasks(projectGraph, options.typescript, options.paths);
+        compilationGulp.task(compileTaskName(projectGraph, options.typescript), deps, () => {
+            const destPath = resolveDestPath(projectGraph, options.paths);
             const { sourceMap, inlineSourceMap, inlineSources = false, sourceRoot, declarationMap } = projectGraph.project.options;
             const configFilePath = projectGraph.project.options.configFilePath;
             const sourceMapPath = inlineSourceMap ? undefined : ".";
             const sourceMapOptions = { includeContent: inlineSources, sourceRoot, destPath };
-            const project = resolvedOptions.inProcess
-                ? tsc.createProject(configFilePath, { typescript: require(resolvedOptions.typescript.typescript) })
-                : tsc_oop.createProject(configFilePath, {}, { typescript: resolvedOptions.typescript.typescript });
+            const project = options.inProcess
+                ? tsc.createProject(configFilePath, { typescript: require(options.typescript.typescript) })
+                : tsc_oop.createProject(configFilePath, {}, { typescript: options.typescript.typescript });
             const stream = project.src()
-                .pipe(gulpif(!resolvedOptions.force, upToDate(projectGraph.project, { verbose: resolvedOptions.verbose })))
+                .pipe(gulpif(!options.force, upToDate(projectGraph.project, { verbose: options.verbose })))
                 .pipe(gulpif(sourceMap || inlineSourceMap, sourcemaps.init()))
                 .pipe(project());
-            const js = stream.js
-                .pipe(evaluateHook(resolvedOptions.js))
+            const js = (options.js ? options.js(stream.js) : stream.js)
                 .pipe(gulpif(sourceMap || inlineSourceMap, sourcemaps.write(sourceMapPath, sourceMapOptions)));
-            const dts = stream.dts
-                .pipe(evaluateHook(resolvedOptions.dts))
+            const dts = (options.dts ? options.dts(stream.dts) : stream.dts)
                 .pipe(gulpif(declarationMap, sourcemaps.write(sourceMapPath, sourceMapOptions)));
             return merge2([js, dts])
                 .pipe(gulp.dest(destPath));

scripts/build/upToDate.js

@@ -1,5 +1,6 @@
 // @ts-check
 const path = require("path");
+const fs = require("fs");
 const log = require("fancy-log"); // was `require("gulp-util").log (see https://github.com/gulpjs/gulp-util)
 const ts = require("../../lib/typescript");
 const { Duplex } = require("stream");
@@ -10,20 +11,26 @@ const Vinyl = require("vinyl");
  * Creates a stream that passes through its inputs only if the project outputs are not up to date
  * with respect to the inputs. 
  * @param {ParsedCommandLine} parsedProject
- * @param {{verbose?: boolean}} [options]
+ * @param {UpToDateOptions} [options]
+ * 
+ * @typedef UpToDateOptions
+ * @property {boolean} [verbose]
  */
 function upToDate(parsedProject, options) {
     /** @type {File[]} */
     const inputs = [];
     /** @type {Map<string, File>} */
     const inputMap = new Map();
+    /** @type {Map<string, fs.Stats>} */
+    const statCache = new Map();
     /** @type {UpToDateHost} */
     const upToDateHost = {
         fileExists(fileName) {
-            return inputMap.has(path.resolve(fileName));
+            const stats = getStat(fileName);
+            return stats ? stats.isFile() : false;
         },
         getModifiedTime(fileName) {
-            return inputMap.get(path.resolve(fileName)).stat.mtime;
+            return getStat(fileName).mtime;
         }
     };
     const duplex = new Duplex({
@@ -34,7 +41,7 @@ function upToDate(parsedProject, options) {
         write(file, _, cb) {
             if (Vinyl.isVinyl(file)) {
                 inputs.push(file);
-                inputMap.set(file.path, file);
+                inputMap.set(path.resolve(file.path), file);
             }
             cb();
         },
@@ -45,12 +52,27 @@ function upToDate(parsedProject, options) {
                 for (const input of inputs) duplex.push(input);
             }
             duplex.push(null);
+            inputMap.clear();
+            statCache.clear();
             cb();
         },
         read() {
         }
     });
     return duplex;
+
+    function getStat(fileName) {
+        fileName = path.resolve(fileName);
+        const inputFile = inputMap.get(fileName);
+        if (inputFile && inputFile.stat) return inputFile.stat;
+        
+        let stats = statCache.get(fileName);
+        if (!stats && fs.existsSync(fileName)) {
+            stats = fs.statSync(fileName);
+            statCache.set(fileName, stats);
+        }
+        return stats;
+    }
 }
 module.exports = exports = upToDate;