From bd66d46e2626f683e6c220f6d9a08681bad42f7f Mon Sep 17 00:00:00 2001 From: Lindsay Levine Date: Thu, 7 Jan 2021 21:00:50 -0500 Subject: [PATCH 1/2] track NoN files when configured dirs are used and clean before running NoN --- index.js | 8 +- lib/steps/prepareFolders.js | 77 +++++++++++++++++- tests/configurableDirs.test.js | 141 +++++++++++++++++++++++++++++++++ tests/helpers/buildNextApp.js | 35 +++++++- 4 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 tests/configurableDirs.test.js diff --git a/index.js b/index.js index f4b464c..2293e23 100644 --- a/index.js +++ b/index.js @@ -16,11 +16,15 @@ const { * publishDir: string to path * } */ + const nextOnNetlify = (options = {}) => { const functionsPath = options.functionsDir || NETLIFY_FUNCTIONS_PATH; const publishPath = options.publishDir || NETLIFY_PUBLISH_PATH; - prepareFolders({ functionsPath, publishPath }); + const trackNextOnNetlifyFiles = prepareFolders({ + functionsPath, + publishPath, + }); copyPublicFiles(publishPath); @@ -33,6 +37,8 @@ const nextOnNetlify = (options = {}) => { setupRedirects(publishPath); setupHeaders(publishPath); + + trackNextOnNetlifyFiles(); }; module.exports = nextOnNetlify; diff --git a/lib/steps/prepareFolders.js b/lib/steps/prepareFolders.js index af497ac..cff9e32 100644 --- a/lib/steps/prepareFolders.js +++ b/lib/steps/prepareFolders.js @@ -1,8 +1,19 @@ -const { emptyDirSync } = require("fs-extra"); +const { join } = require("path"); +const { + emptyDirSync, + existsSync, + readdirSync, + readFileSync, + writeFileSync, + removeSync, +} = require("fs-extra"); +const findCacheDir = require("find-cache-dir"); const { logTitle, log } = require("../helpers/logger"); const { NETLIFY_PUBLISH_PATH, NETLIFY_FUNCTIONS_PATH } = require("../config"); -// Empty existing publish and functions folders +const TRACKING_FILE_SEPARATOR = "---"; + +// Clean existing publish and functions folders const prepareFolders = ({ functionsPath, publishPath }) => { logTitle("🚀 Next on Netlify 🚀"); @@ -19,8 +30,66 @@ const prepareFolders = ({ functionsPath, publishPath }) => { log(" ", "Make sure these are set in your netlify.toml file."); } - if (publishPath === NETLIFY_PUBLISH_PATH) emptyDirSync(publishPath); - if (functionsPath === NETLIFY_FUNCTIONS_PATH) emptyDirSync(functionsPath); + const cacheDir = findCacheDir({ name: "next-on-netlify", create: true }); + const trackingFilePath = join(cacheDir, ".nonfiletracking"); + const trackingFile = existsSync(trackingFilePath) + ? readFileSync(trackingFilePath, "utf8") + : "---"; + + const [trackedFunctions, trackedPublish] = trackingFile.split("---"); + const isConfiguredPublishDir = publishPath !== NETLIFY_PUBLISH_PATH; + const isConfiguredFunctionsDir = functionsPath !== NETLIFY_FUNCTIONS_PATH; + + if (isConfiguredPublishDir) { + trackedPublish + .trim() + .split("\n") + .forEach((file) => { + const filePath = join(publishPath, file); + if (existsSync(filePath) && file !== "") { + removeSync(filePath); + } + }); + } else { + emptyDirSync(publishPath); + } + if (isConfiguredFunctionsDir) { + trackedFunctions + .trim() + .split("\n") + .forEach((file) => { + const filePath = join(functionsPath, file); + if (existsSync(filePath) && file !== "") { + removeSync(filePath); + } + }); + } else { + emptyDirSync(functionsPath); + } + + const functionsBeforeRun = existsSync(functionsPath) + ? readdirSync(functionsPath) + : []; + const publishBeforeRun = existsSync(publishPath) + ? readdirSync(publishPath) + : []; + + // this callback will run at the end of nextOnNetlify() + return () => { + const functionsAfterRun = isConfiguredFunctionsDir + ? readdirSync(functionsPath) + : functionsBeforeRun; + const publishAfterRun = isConfiguredPublishDir + ? readdirSync(publishPath) + : publishBeforeRun; + const getDiff = (before, after) => + after.filter((filePath) => !before.includes(filePath)); + const functionsDiff = getDiff(functionsBeforeRun, functionsAfterRun); + const publishDiff = getDiff(publishBeforeRun, publishAfterRun); + + const totalFilesDiff = [...functionsDiff, "---", ...publishDiff]; + writeFileSync(trackingFilePath, totalFilesDiff.join("\n")); + }; }; module.exports = prepareFolders; diff --git a/tests/configurableDirs.test.js b/tests/configurableDirs.test.js new file mode 100644 index 0000000..336a63d --- /dev/null +++ b/tests/configurableDirs.test.js @@ -0,0 +1,141 @@ +// Test next-on-netlify when config is set from a function in next.config.js +// See: https://github.com/netlify/next-on-netlify/issues/25 + +const { parse, join } = require("path"); +const { existsSync, readdirSync, readFileSync } = require("fs-extra"); +const buildNextApp = require("./helpers/buildNextApp"); + +// The name of this test file (without extension) +const FILENAME = parse(__filename).name; + +// The directory which will be used for testing. +// We simulate a NextJS app within that directory, with pages, and a +// package.json file. +const PROJECT_PATH = join(__dirname, "builds", FILENAME); +const FUNCTIONS_DIR = "my-functions"; +const PUBLISH_DIR = "my-publish"; + +// Capture the output to verify successful build +let buildOutput; + +beforeAll( + async () => { + runOutput = await buildNextApp() + .forTest(__filename) + .withPages("pages") + .withNextConfig("next.config.js") + .withPackageJson("package.json") + .runWithRequire({ functionsDir: FUNCTIONS_DIR, publishDir: PUBLISH_DIR }); + }, + // time out after 180 seconds + 180 * 1000 +); + +describe("next-on-netlify", () => { + const functionsDir = join(PROJECT_PATH, FUNCTIONS_DIR); + + test("builds successfully", () => { + expect(runOutput).toMatch("Built successfully!"); + }); + + test("creates a Netlify Function for each SSR page", () => { + expect(existsSync(join(functionsDir, "next_index", "next_index.js"))).toBe( + true + ); + expect( + existsSync(join(functionsDir, "next_shows_id", "next_shows_id.js")) + ).toBe(true); + expect( + existsSync( + join(functionsDir, "next_shows_params", "next_shows_params.js") + ) + ).toBe(true); + expect( + existsSync( + join( + functionsDir, + "next_getServerSideProps_static", + "next_getServerSideProps_static.js" + ) + ) + ).toBe(true); + expect( + existsSync( + join( + functionsDir, + "next_getServerSideProps_id", + "next_getServerSideProps_id.js" + ) + ) + ).toBe(true); + }); + + test("copies static pages to output directory", () => { + const OUTPUT_PATH = join(PROJECT_PATH, PUBLISH_DIR); + + expect(existsSync(join(OUTPUT_PATH, "static.html"))).toBe(true); + expect(existsSync(join(OUTPUT_PATH, "static/[id].html"))).toBe(true); + }); + + test("copies static assets to out_publish/_next/ directory", () => { + const dirs = readdirSync( + join(PROJECT_PATH, PUBLISH_DIR, "_next", "static") + ); + + expect(dirs.length).toBe(2); + expect(dirs).toContain("chunks"); + }); +}); + +describe("clean up of NoN files", () => { + test("creates a .nonfiletracking to audit NoN-specific files between builds", () => { + const cacheDir = join(PROJECT_PATH, "/node_modules/.cache/next-on-netlify"); + const dirs = readdirSync(cacheDir); + expect(dirs[0]).toEqual(".nonfiletracking"); + }); + + test(".nonfiletracking contains NoN-specific files", () => { + const cacheDir = join(PROJECT_PATH, "/node_modules/.cache/next-on-netlify"); + const fileList = readFileSync(join(cacheDir, ".nonfiletracking"), "utf8"); + // had to test equality this way because of windows :) + const isSameList = (arr1, arr2) => + arr1.reduce((isSame, func) => { + if (arr2.includes(func)) { + isSame = true; + } else { + isSame = false; + } + return isSame; + }, true); + const nextFunctions = [ + "next_api_shows_id", + "next_api_shows_params", + "next_api_static", + "next_getServerSideProps_all_slug", + "next_getServerSideProps_id", + "next_getServerSideProps_static", + "next_getStaticProps_id", + "next_getStaticProps_static", + "next_getStaticProps_withFallback_id", + "next_getStaticProps_withFallback_slug", + "next_getStaticProps_withRevalidate_id", + "next_getStaticProps_withRevalidate_withFallback_id", + "next_getStaticProps_withrevalidate", + "next_index", + "next_shows_id", + "next_shows_params", + ]; + const fileListFunctions = fileList.split("---")[0].trim().split("\n"); + expect(isSameList(nextFunctions, fileListFunctions)).toBe(true); + const publishFiles = [ + "404.html", + "_next", + "_redirects", + "getStaticProps", + "static", + "static.html", + ]; + const fileListPublish = fileList.split("---")[1].trim().split("\n"); + expect(isSameList(publishFiles, fileListPublish)).toBe(true); + }); +}); diff --git a/tests/helpers/buildNextApp.js b/tests/helpers/buildNextApp.js index eaee609..4bfd9f1 100644 --- a/tests/helpers/buildNextApp.js +++ b/tests/helpers/buildNextApp.js @@ -63,7 +63,7 @@ class NextAppBuilder { // Build the application with next build async build() { - // Generate a cach hash ID from the current contents of the staging folder. + // Generate a cache hash ID from the current contents of the staging folder. const { hash: cacheHash } = await hashElement(this.__stagingPath, { encoding: "hex", }); @@ -88,6 +88,39 @@ class NextAppBuilder { return stdout; } + async runWithRequire(options) { + // Generate a cach hash ID from the current contents of the staging folder. + const { hash: cacheHash } = await hashElement(this.__stagingPath, { + encoding: "hex", + }); + this.__cacheHash = cacheHash; + + // If we have no cached build for this NextJS app, let's run next build and + // cache the result + if (!existsSync(this.__cachePath)) { + // Build the nextJS app + await npmRun("next-build", this.__stagingPath); + + // Cache the build + copySync(this.__stagingPath, this.__cachePath); + } + + // Copy the built NextJS app from the cache to the app folder, where we will + // run next-on-netlify + copySync(this.__cachePath, this.__appPath); + + process.chdir(this.__appPath); + const nextOnNetlify = require("../.."); + nextOnNetlify({ + functionsDir: join(this.__appPath, options.functionsDir), + publishDir: join(this.__appPath, options.publishDir), + }); + return "Built successfully!"; + } + + // TO-DO: when I try to split out the shared logic between build & runWithRequire into its own + // function on NextBuilder, everything breaks; not sure why + /***************************************************************************** * Private functions ****************************************************************************/ From 9d78777471d1f352f60a8c71e72b703f9a7f2f9d Mon Sep 17 00:00:00 2001 From: Lindsay Levine Date: Wed, 13 Jan 2021 03:58:20 -0500 Subject: [PATCH 2/2] some cleanup and extra test case --- lib/helpers/handleFileTracking.js | 80 ++++++++++++++++++ lib/steps/prepareFolders.js | 84 +++---------------- tests/configurableDirs.test.js | 10 ++- .../fixtures/my-functions/someTestFunction.js | 0 tests/helpers/buildNextApp.js | 40 ++++----- 5 files changed, 115 insertions(+), 99 deletions(-) create mode 100644 lib/helpers/handleFileTracking.js create mode 100644 tests/fixtures/my-functions/someTestFunction.js diff --git a/lib/helpers/handleFileTracking.js b/lib/helpers/handleFileTracking.js new file mode 100644 index 0000000..736d9f5 --- /dev/null +++ b/lib/helpers/handleFileTracking.js @@ -0,0 +1,80 @@ +const { join } = require("path"); +const { + existsSync, + readdirSync, + readFileSync, + writeFileSync, + removeSync, +} = require("fs-extra"); +const findCacheDir = require("find-cache-dir"); +const { NETLIFY_PUBLISH_PATH, NETLIFY_FUNCTIONS_PATH } = require("../config"); + +const TRACKING_FILE_SEPARATOR = "---"; + +// Clean configured publish and functions folders and track next-on-netlify files +// for future cleans +const handleFileTracking = ({ functionsPath, publishPath }) => { + const isConfiguredFunctionsDir = functionsPath !== NETLIFY_FUNCTIONS_PATH; + const isConfiguredPublishDir = publishPath !== NETLIFY_PUBLISH_PATH; + + const cacheDir = findCacheDir({ name: "next-on-netlify", create: true }); + const trackingFilePath = join(cacheDir, ".nonfiletracking"); + + if (existsSync(trackingFilePath)) { + const trackingFile = readFileSync(trackingFilePath, "utf8"); + const [trackedFunctions, trackedPublish] = trackingFile.split( + TRACKING_FILE_SEPARATOR + ); + + const cleanConfiguredFiles = (trackedFiles) => { + trackedFiles.forEach((file) => { + const filePath = join(publishPath, file); + if (file !== "" && existsSync(filePath)) { + removeSync(filePath); + } + }); + }; + + if (isConfiguredPublishDir) { + cleanConfiguredFiles(trackedPublish.split("\n")); + } + if (isConfiguredFunctionsDir) { + cleanConfiguredFiles(trackedFunctions.split("\n")); + } + } + + const functionsBeforeRun = existsSync(functionsPath) + ? readdirSync(functionsPath) + : []; + const publishBeforeRun = existsSync(publishPath) + ? readdirSync(publishPath) + : []; + + // this callback will run at the end of nextOnNetlify() + const trackNewFiles = () => { + const functionsAfterRun = isConfiguredFunctionsDir + ? readdirSync(functionsPath) + : functionsBeforeRun; + const publishAfterRun = isConfiguredPublishDir + ? readdirSync(publishPath) + : publishBeforeRun; + const getDifference = (before, after) => + after.filter((filePath) => !before.includes(filePath)); + const newFunctionsFiles = getDifference( + functionsBeforeRun, + functionsAfterRun + ); + const newPublishFiles = getDifference(publishBeforeRun, publishAfterRun); + + const allTrackedFiles = [ + ...newFunctionsFiles, + TRACKING_FILE_SEPARATOR, + ...newPublishFiles, + ]; + writeFileSync(trackingFilePath, allTrackedFiles.join("\n")); + }; + + return trackNewFiles; +}; + +module.exports = handleFileTracking; diff --git a/lib/steps/prepareFolders.js b/lib/steps/prepareFolders.js index cff9e32..cb65598 100644 --- a/lib/steps/prepareFolders.js +++ b/lib/steps/prepareFolders.js @@ -1,95 +1,37 @@ const { join } = require("path"); -const { - emptyDirSync, - existsSync, - readdirSync, - readFileSync, - writeFileSync, - removeSync, -} = require("fs-extra"); +const { emptyDirSync } = require("fs-extra"); const findCacheDir = require("find-cache-dir"); const { logTitle, log } = require("../helpers/logger"); const { NETLIFY_PUBLISH_PATH, NETLIFY_FUNCTIONS_PATH } = require("../config"); - -const TRACKING_FILE_SEPARATOR = "---"; +const handleFileTracking = require("../helpers/handleFileTracking"); // Clean existing publish and functions folders const prepareFolders = ({ functionsPath, publishPath }) => { logTitle("🚀 Next on Netlify 🚀"); - if (functionsPath === NETLIFY_FUNCTIONS_PATH) { + const isNotConfiguredFunctionsDir = functionsPath === NETLIFY_FUNCTIONS_PATH; + const isNotConfiguredPublishDir = publishPath === NETLIFY_PUBLISH_PATH; + + if (isNotConfiguredFunctionsDir) { log(" ", "Functions directory: ", functionsPath); } - if (publishPath === NETLIFY_PUBLISH_PATH) { + if (isNotConfiguredPublishDir) { log(" ", "Publish directory: ", publishPath); } - if ( - functionsPath === NETLIFY_FUNCTIONS_PATH || - publishPath === NETLIFY_PUBLISH_PATH - ) { + if (isNotConfiguredFunctionsDir || isNotConfiguredPublishDir) { log(" ", "Make sure these are set in your netlify.toml file."); } - const cacheDir = findCacheDir({ name: "next-on-netlify", create: true }); - const trackingFilePath = join(cacheDir, ".nonfiletracking"); - const trackingFile = existsSync(trackingFilePath) - ? readFileSync(trackingFilePath, "utf8") - : "---"; - - const [trackedFunctions, trackedPublish] = trackingFile.split("---"); - const isConfiguredPublishDir = publishPath !== NETLIFY_PUBLISH_PATH; - const isConfiguredFunctionsDir = functionsPath !== NETLIFY_FUNCTIONS_PATH; - - if (isConfiguredPublishDir) { - trackedPublish - .trim() - .split("\n") - .forEach((file) => { - const filePath = join(publishPath, file); - if (existsSync(filePath) && file !== "") { - removeSync(filePath); - } - }); - } else { + // We can empty these dirs knowing there will only be stale NoN-generated files inside + if (isNotConfiguredPublishDir) { emptyDirSync(publishPath); } - if (isConfiguredFunctionsDir) { - trackedFunctions - .trim() - .split("\n") - .forEach((file) => { - const filePath = join(functionsPath, file); - if (existsSync(filePath) && file !== "") { - removeSync(filePath); - } - }); - } else { + if (isNotConfiguredFunctionsDir) { emptyDirSync(functionsPath); } - const functionsBeforeRun = existsSync(functionsPath) - ? readdirSync(functionsPath) - : []; - const publishBeforeRun = existsSync(publishPath) - ? readdirSync(publishPath) - : []; - - // this callback will run at the end of nextOnNetlify() - return () => { - const functionsAfterRun = isConfiguredFunctionsDir - ? readdirSync(functionsPath) - : functionsBeforeRun; - const publishAfterRun = isConfiguredPublishDir - ? readdirSync(publishPath) - : publishBeforeRun; - const getDiff = (before, after) => - after.filter((filePath) => !before.includes(filePath)); - const functionsDiff = getDiff(functionsBeforeRun, functionsAfterRun); - const publishDiff = getDiff(publishBeforeRun, publishAfterRun); - - const totalFilesDiff = [...functionsDiff, "---", ...publishDiff]; - writeFileSync(trackingFilePath, totalFilesDiff.join("\n")); - }; + // This returns a function that runs as the last step of nextOnNetlify() + return handleFileTracking({ functionsPath, publishPath }); }; module.exports = prepareFolders; diff --git a/tests/configurableDirs.test.js b/tests/configurableDirs.test.js index 336a63d..aa122a7 100644 --- a/tests/configurableDirs.test.js +++ b/tests/configurableDirs.test.js @@ -25,6 +25,7 @@ beforeAll( .withPages("pages") .withNextConfig("next.config.js") .withPackageJson("package.json") + .withCustomFunctions("my-functions") .runWithRequire({ functionsDir: FUNCTIONS_DIR, publishDir: PUBLISH_DIR }); }, // time out after 180 seconds @@ -38,6 +39,10 @@ describe("next-on-netlify", () => { expect(runOutput).toMatch("Built successfully!"); }); + test("copies custom Netlify Function to configured functions directory", () => { + expect(existsSync(join(functionsDir, "someTestFunction.js"))).toBe(true); + }); + test("creates a Netlify Function for each SSR page", () => { expect(existsSync(join(functionsDir, "next_index", "next_index.js"))).toBe( true @@ -125,8 +130,9 @@ describe("clean up of NoN files", () => { "next_shows_id", "next_shows_params", ]; - const fileListFunctions = fileList.split("---")[0].trim().split("\n"); + const fileListFunctions = fileList.split("---")[0].split("\n"); expect(isSameList(nextFunctions, fileListFunctions)).toBe(true); + expect(fileListFunctions.includes("someTestFunction.js")).toBe(false); const publishFiles = [ "404.html", "_next", @@ -135,7 +141,7 @@ describe("clean up of NoN files", () => { "static", "static.html", ]; - const fileListPublish = fileList.split("---")[1].trim().split("\n"); + const fileListPublish = fileList.split("---")[1].split("\n"); expect(isSameList(publishFiles, fileListPublish)).toBe(true); }); }); diff --git a/tests/fixtures/my-functions/someTestFunction.js b/tests/fixtures/my-functions/someTestFunction.js new file mode 100644 index 0000000..e69de29 diff --git a/tests/helpers/buildNextApp.js b/tests/helpers/buildNextApp.js index 4bfd9f1..8073da6 100644 --- a/tests/helpers/buildNextApp.js +++ b/tests/helpers/buildNextApp.js @@ -46,6 +46,10 @@ class NextAppBuilder { return this.withFile(packageJsonFile, "package.json"); } + withCustomFunctions(functionsDir) { + return this.withFile(functionsDir); + } + // Copy a file from the fixtures folder to the app's staging folder withFile(fixture, target = null) { // If no target file name is given, use the same name as the fixture @@ -61,8 +65,7 @@ class NextAppBuilder { return this; } - // Build the application with next build - async build() { + async buildNextApp() { // Generate a cache hash ID from the current contents of the staging folder. const { hash: cacheHash } = await hashElement(this.__stagingPath, { encoding: "hex", @@ -83,33 +86,21 @@ class NextAppBuilder { // run next-on-netlify copySync(this.__cachePath, this.__appPath); - // Run next-on-netlify + process.chdir(this.__appPath); + } + + async build() { + await this.buildNextApp(); + + // Run next-on-netlify as postbuild script const { stdout } = await npmRun("next-on-netlify", this.__appPath); return stdout; } async runWithRequire(options) { - // Generate a cach hash ID from the current contents of the staging folder. - const { hash: cacheHash } = await hashElement(this.__stagingPath, { - encoding: "hex", - }); - this.__cacheHash = cacheHash; - - // If we have no cached build for this NextJS app, let's run next build and - // cache the result - if (!existsSync(this.__cachePath)) { - // Build the nextJS app - await npmRun("next-build", this.__stagingPath); - - // Cache the build - copySync(this.__stagingPath, this.__cachePath); - } + await this.buildNextApp(); - // Copy the built NextJS app from the cache to the app folder, where we will - // run next-on-netlify - copySync(this.__cachePath, this.__appPath); - - process.chdir(this.__appPath); + // Run next-on-netlify as an imported module const nextOnNetlify = require("../.."); nextOnNetlify({ functionsDir: join(this.__appPath, options.functionsDir), @@ -118,9 +109,6 @@ class NextAppBuilder { return "Built successfully!"; } - // TO-DO: when I try to split out the shared logic between build & runWithRequire into its own - // function on NextBuilder, everything breaks; not sure why - /***************************************************************************** * Private functions ****************************************************************************/