Skip to content

Commit 72d34ce

Browse files
ascorbicascorbic
authored
Merge pull request #334 from netlify/mk/page-function-worker
perf: parallelize page setup
2 parents 6107137 + b27f30d commit 72d34ce

File tree

13 files changed

+11034
-317
lines changed

13 files changed

+11034
-317
lines changed

‎package-lock.json‎

Lines changed: 10907 additions & 219 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"commander": "^7.1.0",
6363
"debounce-fn": "^4.0.0",
6464
"execa": "^5.0.0",
65+
"fastq": "^1.11.0",
6566
"find-cache-dir": "^3.3.1",
6667
"find-up": "^5.0.0",
6768
"fs-extra": "^9.1.0",

‎src/lib/helpers/copyDynamicImportChunks.js‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,27 +12,30 @@ const copyDynamicImportChunks = async (functionPath) => {
1212
const filesWP4 = readdirSync(chunksPathWebpack4)
1313
const chunkRegexWP4 = new RegExp(/^(\.?[-$~\w]+)+\.js$/g)
1414
const excludeFiles = new Set(['init-server.js.js', 'on-error-server.js.js'])
15+
const copyPathWP4 = join(functionPath, 'nextPage')
16+
if (filesWP4.length !== 0) {
17+
logTitle('💼 Copying WP4 dynamic import chunks to', copyPathWP4)
18+
}
1519
filesWP4.forEach((file) => {
1620
if (!excludeFiles.has(file) && chunkRegexWP4.test(file)) {
17-
// WP4 files are looked for one level up (../) in runtime
18-
// This is a hack to make the file one level up i.e. with
19-
// nextPage/nextPage/index.js, the chunk is moved to the inner nextPage
20-
const copyPath = join(functionPath, 'nextPage')
21-
logTitle('💼 Copying WP4 dynamic import chunks to', copyPath)
22-
copySync(join(chunksPathWebpack4, file), join(copyPath, file), {
21+
copySync(join(chunksPathWebpack4, file), join(copyPathWP4, file), {
2322
overwrite: false,
2423
errorOnExist: true,
2524
})
2625
}
2726
})
27+
28+
// Chunks are copied into the nextPage directory, as a sibling to "pages" or "api".
29+
// This matches the Next output, so that imports work correctly
2830
const chunksPathWebpack5 = join(nextDistDir, 'serverless', 'chunks')
2931
const filesWP5 = existsSync(chunksPathWebpack5) ? readdirSync(chunksPathWebpack5) : []
32+
const copyPathWP5 = join(functionPath, 'nextPage', 'chunks')
33+
if (filesWP5.length !== 0) {
34+
logTitle('💼 Copying WB5 dynamic import chunks to', copyPathWP5)
35+
}
36+
3037
filesWP5.forEach((file) => {
31-
// Chunks are copied into the nextPage directory, as a sibling to pages or api.
32-
// This matches the Next output, so that imports work correctly
33-
const copyPath = join(functionPath, 'nextPage', 'chunks')
34-
logTitle('💼 Copying WB5 dynamic import chunks to', copyPath)
35-
copySync(join(chunksPathWebpack5, file), join(copyPath, file), {
38+
copySync(join(chunksPathWebpack5, file), join(copyPathWP5, file), {
3639
overwrite: false,
3740
errorOnExist: true,
3841
})

‎src/lib/helpers/runJobsQueue.js‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
const os = require('os')
2+
3+
const fastq = require('fastq')
4+
5+
const { processPage } = require('../pages/worker')
6+
7+
// TODO: benchmark a large site to find a good value for this
8+
const PAGE_CONCURRENCY = 4
9+
10+
const runJobsQueue = (jobs) => {
11+
console.log(`Building ${jobs.length} pages`)
12+
13+
const queue = fastq.promise(processPage, PAGE_CONCURRENCY)
14+
15+
return Promise.all(jobs.map((job) => queue.push(job)))
16+
}
17+
18+
module.exports = runJobsQueue

‎src/lib/pages/api/setup.js‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -10,11 +8,8 @@ const setup = async (functionsPath) => {
108

119
const pages = await getPages()
1210

13-
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isApiPage: true })
17-
})
11+
// Create Netlify Function job for every page
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath, isApiPage: true }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getInitialProps/setup.js‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getServerSideProps/setup.js‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
const asyncForEach = require('../../helpers/asyncForEach')
2-
const { logTitle, logItem } = require('../../helpers/logger')
3-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
1+
const { logTitle } = require('../../helpers/logger')
42

53
const getPages = require('./pages')
64

@@ -11,10 +9,7 @@ const setup = async (functionsPath) => {
119
const pages = await getPages()
1210

1311
// Create Netlify Function for every page
14-
await asyncForEach(pages, async ({ filePath }) => {
15-
logItem(filePath)
16-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
17-
})
12+
return pages.map(({ filePath }) => ({ type: 'function', filePath, functionsPath }))
1813
}
1914

2015
module.exports = setup

‎src/lib/pages/getStaticProps/setup.js‎

Lines changed: 39 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,51 +1,56 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
54
const isRouteWithFallback = require('../../helpers/isRouteWithFallback')
6-
const { logTitle, logItem } = require('../../helpers/logger')
7-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
8-
const setupStaticFileForPage = require('../../helpers/setupStaticFileForPage')
5+
const { logTitle } = require('../../helpers/logger')
96

107
const getPages = require('./pages')
118

129
// Copy pre-rendered SSG pages
1310
const setup = async ({ functionsPath, publishPath }) => {
1411
logTitle('🔥 Copying pre-rendered pages with getStaticProps and JSON data to', publishPath)
15-
1612
// Keep track of the functions that have been set up, so that we do not set up
1713
// a function for the same file path twice
18-
const filePathsDone = []
19-
14+
const filePathsDone = new Set()
2015
const pages = await getPages()
2116

22-
await asyncForEach(pages, async ({ route, dataRoute, srcRoute }) => {
23-
logItem(route)
24-
25-
// Copy pre-rendered HTML page
26-
const htmlPath = getFilePathForRoute(route, 'html')
27-
await setupStaticFileForPage({ inputPath: htmlPath, publishPath })
28-
29-
// Copy page's JSON data
30-
const jsonPath = getFilePathForRoute(route, 'json')
31-
await setupStaticFileForPage({
32-
inputPath: jsonPath,
33-
outputPath: dataRoute,
34-
publishPath,
35-
})
36-
37-
// Set up the Netlify function (this is ONLY for preview mode)
38-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
39-
const filePath = join('pages', relativePath)
40-
41-
// Skip if we have already set up a function for this file
42-
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
43-
if (filePathsDone.includes(filePath) || (await isRouteWithFallback(srcRoute))) return
44-
45-
logItem(filePath)
46-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
47-
filePathsDone.push(filePath)
48-
})
17+
const jobs = []
18+
19+
await Promise.all(
20+
pages.map(async ({ route, dataRoute, srcRoute }) => {
21+
// Copy pre-rendered HTML page
22+
const htmlPath = getFilePathForRoute(route, 'html')
23+
24+
jobs.push({ type: 'static', inputPath: htmlPath, publishPath })
25+
26+
// Copy page's JSON data
27+
const jsonPath = getFilePathForRoute(route, 'json')
28+
jobs.push({
29+
type: 'static',
30+
inputPath: jsonPath,
31+
outputPath: dataRoute,
32+
publishPath,
33+
})
34+
35+
// Set up the Netlify function (this is ONLY for preview mode)
36+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
37+
const filePath = join('pages', relativePath)
38+
39+
// Skip if we have already set up a function for this file
40+
41+
if (filePathsDone.has(filePath)) {
42+
return
43+
}
44+
filePathsDone.add(filePath)
45+
46+
// or if the source route has a fallback (handled by getStaticPropsWithFallback)
47+
if (await isRouteWithFallback(srcRoute)) {
48+
return
49+
}
50+
jobs.push({ type: 'function', filePath, functionsPath })
51+
}),
52+
)
53+
return jobs
4954
}
5055

5156
module.exports = setup

‎src/lib/pages/getStaticPropsWithFallback/setup.js‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -14,11 +12,10 @@ const setup = async (functionsPath) => {
1412
const pages = await getPages()
1513

1614
// Create Netlify Function for every page
17-
await asyncForEach(pages, async ({ route }) => {
15+
return pages.map(({ route }) => {
1816
const relativePath = getFilePathForRoute(route, 'js')
1917
const filePath = join('pages', relativePath)
20-
logItem(filePath)
21-
await setupNetlifyFunctionForPage({ filePath, functionsPath, isISR: true })
18+
return { type: 'function', filePath, functionsPath, isISR: true }
2219
})
2320
}
2421

‎src/lib/pages/getStaticPropsWithRevalidate/setup.js‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
const { join } = require('path')
22

3-
const asyncForEach = require('../../helpers/asyncForEach')
43
const getFilePathForRoute = require('../../helpers/getFilePathForRoute')
5-
const { logTitle, logItem } = require('../../helpers/logger')
6-
const setupNetlifyFunctionForPage = require('../../helpers/setupNetlifyFunctionForPage')
4+
const { logTitle } = require('../../helpers/logger')
75

86
const getPages = require('./pages')
97

@@ -17,23 +15,24 @@ const setup = async (functionsPath) => {
1715

1816
// Keep track of the functions that have been set up, so that we do not set up
1917
// a function for the same file path twice
20-
const filePathsDone = []
18+
const filePathsDone = new Set()
2119

2220
const pages = await getPages()
2321

2422
// Create Netlify Function for every page
25-
await asyncForEach(pages, async ({ route, srcRoute }) => {
26-
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
27-
const filePath = join('pages', relativePath)
2823

29-
// Skip if we have already set up a function for this file
30-
if (filePathsDone.includes(filePath)) return
24+
const jobs = []
3125

32-
// Set up the function
33-
logItem(filePath)
34-
await setupNetlifyFunctionForPage({ filePath, functionsPath })
35-
filePathsDone.push(filePath)
26+
pages.forEach(({ route, srcRoute }) => {
27+
const relativePath = getFilePathForRoute(srcRoute || route, 'js')
28+
const filePath = join('pages', relativePath)
29+
if (filePathsDone.has(filePath)) {
30+
return
31+
}
32+
filePathsDone.add(filePath)
33+
jobs.push({ type: 'function', filePath, functionsPath })
3634
})
35+
return jobs
3736
}
3837

3938
module.exports = setup

0 commit comments

Comments
 (0)