mirror of
https://github.com/terribleplan/next.js.git
synced 2024-01-19 02:48:18 +00:00
multi-threaded export with nice progress indication (#5870)
This PR will - allow nextjs export to use all available CPU cores for rendering & writing pages by using child_process - make use of async-sema to allow each thread to concurrently write multiple paths - show a fancy progress bar while processing pages (with non-TTY fallback for CI web consoles) The performance gain for my MacBook with 4 CPU cores went from ~25 pages per second to ~75 pages per second. Beefy CI machines with lots of cores should profit even more.
This commit is contained in:
parent
71d1d363ad
commit
e6c3686629
|
@ -54,6 +54,8 @@ if (!existsSync(join(dir, 'pages'))) {
|
||||||
|
|
||||||
const options = {
|
const options = {
|
||||||
silent: argv.silent,
|
silent: argv.silent,
|
||||||
|
threads: argv.threads,
|
||||||
|
concurrency: argv.concurrency,
|
||||||
outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
|
outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,13 +1,15 @@
|
||||||
import del from 'del'
|
import del from 'del'
|
||||||
|
import { cpus } from 'os'
|
||||||
|
import { fork } from 'child_process'
|
||||||
import cp from 'recursive-copy'
|
import cp from 'recursive-copy'
|
||||||
import mkdirp from 'mkdirp-then'
|
import mkdirp from 'mkdirp-then'
|
||||||
import { extname, resolve, join, dirname, sep } from 'path'
|
import { resolve, join } from 'path'
|
||||||
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
import { existsSync, readFileSync } from 'fs'
|
||||||
import loadConfig from 'next-server/next-config'
|
import loadConfig from 'next-server/next-config'
|
||||||
import {PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH} from 'next-server/constants'
|
import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants'
|
||||||
import { renderToHTML } from 'next-server/dist/server/render'
|
|
||||||
import { setAssetPrefix } from 'next-server/asset'
|
import { setAssetPrefix } from 'next-server/asset'
|
||||||
import * as envConfig from 'next-server/config'
|
import * as envConfig from 'next-server/config'
|
||||||
|
import createProgress from 'tty-aware-progress'
|
||||||
|
|
||||||
export default async function (dir, options, configuration) {
|
export default async function (dir, options, configuration) {
|
||||||
function log (message) {
|
function log (message) {
|
||||||
|
@ -17,6 +19,8 @@ export default async function (dir, options, configuration) {
|
||||||
|
|
||||||
dir = resolve(dir)
|
dir = resolve(dir)
|
||||||
const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir)
|
const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir)
|
||||||
|
const concurrency = options.concurrency || 10
|
||||||
|
const threads = options.threads || Math.max(cpus().length - 1, 1)
|
||||||
const distDir = join(dir, nextConfig.distDir)
|
const distDir = join(dir, nextConfig.distDir)
|
||||||
|
|
||||||
log(`> using build directory: ${distDir}`)
|
log(`> using build directory: ${distDir}`)
|
||||||
|
@ -108,35 +112,48 @@ export default async function (dir, options, configuration) {
|
||||||
nextExport: true
|
nextExport: true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log(` launching ${threads} threads with concurrency of ${concurrency} per thread`)
|
||||||
const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {dev: false, dir, outDir, distDir, buildId})
|
const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {dev: false, dir, outDir, distDir, buildId})
|
||||||
const exportPaths = Object.keys(exportPathMap)
|
const exportPaths = Object.keys(exportPathMap)
|
||||||
|
|
||||||
for (const path of exportPaths) {
|
const progress = !options.silent && createProgress(exportPaths.length)
|
||||||
log(`> exporting path: ${path}`)
|
|
||||||
if (!path.startsWith('/')) {
|
const chunks = exportPaths.reduce((result, route, i) => {
|
||||||
throw new Error(`path "${path}" doesn't start with a backslash`)
|
const worker = i % threads
|
||||||
|
if (!result[worker]) {
|
||||||
|
result[worker] = { paths: [], pathMap: {} }
|
||||||
}
|
}
|
||||||
|
result[worker].pathMap[route] = exportPathMap[route]
|
||||||
|
result[worker].paths.push(route)
|
||||||
|
return result
|
||||||
|
}, [])
|
||||||
|
|
||||||
const { page, query = {} } = exportPathMap[path]
|
await Promise.all(
|
||||||
const req = { url: path }
|
chunks.map(
|
||||||
const res = {}
|
chunk =>
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
let htmlFilename = `${path}${sep}index.html`
|
const worker = fork(require.resolve('./worker'), [], {
|
||||||
if (extname(path) !== '') {
|
env: process.env
|
||||||
// If the path has an extension, use that as the filename instead
|
})
|
||||||
htmlFilename = path
|
worker.send({
|
||||||
} else if (path === '/') {
|
exportPaths: chunk.paths,
|
||||||
// If the path is the root, just use index.html
|
exportPathMap: chunk.pathMap,
|
||||||
htmlFilename = 'index.html'
|
outDir,
|
||||||
}
|
renderOpts,
|
||||||
const baseDir = join(outDir, dirname(htmlFilename))
|
concurrency
|
||||||
const htmlFilepath = join(outDir, htmlFilename)
|
})
|
||||||
|
worker.on('message', ({ type, payload }) => {
|
||||||
await mkdirp(baseDir)
|
if (type === 'progress' && progress) {
|
||||||
|
progress()
|
||||||
const html = await renderToHTML(req, res, page, query, renderOpts)
|
} else if (type === 'error') {
|
||||||
writeFileSync(htmlFilepath, html, 'utf8')
|
reject(payload)
|
||||||
}
|
} else if (type === 'done') {
|
||||||
|
resolve()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
// Add an empty line to the console for the better readability.
|
// Add an empty line to the console for the better readability.
|
||||||
log('')
|
log('')
|
||||||
|
|
59
packages/next/export/worker.js
Normal file
59
packages/next/export/worker.js
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
global.__NEXT_DATA__ = {
|
||||||
|
nextExport: true
|
||||||
|
}
|
||||||
|
|
||||||
|
const { extname, join, dirname, sep } = require('path')
|
||||||
|
const mkdirp = require('mkdirp-then')
|
||||||
|
const { renderToHTML } = require('next-server/dist/server/render')
|
||||||
|
const { writeFile } = require('fs')
|
||||||
|
const Sema = require('async-sema')
|
||||||
|
|
||||||
|
process.on(
|
||||||
|
'message',
|
||||||
|
async ({
|
||||||
|
exportPaths,
|
||||||
|
exportPathMap,
|
||||||
|
outDir,
|
||||||
|
renderOpts,
|
||||||
|
concurrency
|
||||||
|
}) => {
|
||||||
|
const sema = new Sema(concurrency, { capacity: exportPaths.length })
|
||||||
|
try {
|
||||||
|
const work = async path => {
|
||||||
|
await sema.acquire()
|
||||||
|
const { page, query = {} } = exportPathMap[path]
|
||||||
|
const req = { url: path }
|
||||||
|
const res = {}
|
||||||
|
|
||||||
|
let htmlFilename = `${path}${sep}index.html`
|
||||||
|
if (extname(path) !== '') {
|
||||||
|
// If the path has an extension, use that as the filename instead
|
||||||
|
htmlFilename = path
|
||||||
|
} else if (path === '/') {
|
||||||
|
// If the path is the root, just use index.html
|
||||||
|
htmlFilename = 'index.html'
|
||||||
|
}
|
||||||
|
const baseDir = join(outDir, dirname(htmlFilename))
|
||||||
|
const htmlFilepath = join(outDir, htmlFilename)
|
||||||
|
|
||||||
|
await mkdirp(baseDir)
|
||||||
|
const html = await renderToHTML(req, res, page, query, renderOpts)
|
||||||
|
await new Promise((resolve, reject) =>
|
||||||
|
writeFile(
|
||||||
|
htmlFilepath,
|
||||||
|
html,
|
||||||
|
'utf8',
|
||||||
|
err => (err ? reject(err) : resolve())
|
||||||
|
)
|
||||||
|
)
|
||||||
|
process.send({ type: 'progress' })
|
||||||
|
sema.release()
|
||||||
|
}
|
||||||
|
await Promise.all(exportPaths.map(work))
|
||||||
|
process.send({ type: 'done' })
|
||||||
|
} catch (err) {
|
||||||
|
console.error(err)
|
||||||
|
process.send({ type: 'error', payload: err })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
|
@ -49,6 +49,7 @@
|
||||||
"@babel/runtime-corejs2": "7.1.2",
|
"@babel/runtime-corejs2": "7.1.2",
|
||||||
"@babel/template": "7.1.2",
|
"@babel/template": "7.1.2",
|
||||||
"ansi-html": "0.0.7",
|
"ansi-html": "0.0.7",
|
||||||
|
"async-sema": "^2.1.4",
|
||||||
"autodll-webpack-plugin": "0.4.2",
|
"autodll-webpack-plugin": "0.4.2",
|
||||||
"babel-core": "7.0.0-bridge.0",
|
"babel-core": "7.0.0-bridge.0",
|
||||||
"babel-loader": "8.0.2",
|
"babel-loader": "8.0.2",
|
||||||
|
@ -79,6 +80,7 @@
|
||||||
"strip-ansi": "3.0.1",
|
"strip-ansi": "3.0.1",
|
||||||
"styled-jsx": "3.1.1",
|
"styled-jsx": "3.1.1",
|
||||||
"terser-webpack-plugin": "1.1.0",
|
"terser-webpack-plugin": "1.1.0",
|
||||||
|
"tty-aware-progress": "1.0.3",
|
||||||
"unfetch": "3.0.0",
|
"unfetch": "3.0.0",
|
||||||
"url": "0.11.0",
|
"url": "0.11.0",
|
||||||
"webpack": "4.26.0",
|
"webpack": "4.26.0",
|
||||||
|
|
Loading…
Reference in a new issue