mirror of
https://github.com/terribleplan/next.js.git
synced 2024-01-19 02:48:18 +00:00
multi-threaded export with nice progress indication (#5870)
This PR will - allow nextjs export to use all available CPU cores for rendering & writing pages by using child_process - make use of async-sema to allow each thread to concurrently write multiple paths - show a fancy progress bar while processing pages (with non-TTY fallback for CI web consoles) The performance gain for my MacBook with 4 CPU cores went from ~25 pages per second to ~75 pages per second. Beefy CI machines with lots of cores should profit even more.
This commit is contained in:
parent
71d1d363ad
commit
e6c3686629
|
@ -54,6 +54,8 @@ if (!existsSync(join(dir, 'pages'))) {
|
|||
|
||||
const options = {
|
||||
silent: argv.silent,
|
||||
threads: argv.threads,
|
||||
concurrency: argv.concurrency,
|
||||
outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
|
||||
}
|
||||
|
||||
|
|
|
@ -1,13 +1,15 @@
|
|||
import del from 'del'
|
||||
import { cpus } from 'os'
|
||||
import { fork } from 'child_process'
|
||||
import cp from 'recursive-copy'
|
||||
import mkdirp from 'mkdirp-then'
|
||||
import { extname, resolve, join, dirname, sep } from 'path'
|
||||
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
||||
import { resolve, join } from 'path'
|
||||
import { existsSync, readFileSync } from 'fs'
|
||||
import loadConfig from 'next-server/next-config'
|
||||
import {PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH} from 'next-server/constants'
|
||||
import { renderToHTML } from 'next-server/dist/server/render'
|
||||
import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants'
|
||||
import { setAssetPrefix } from 'next-server/asset'
|
||||
import * as envConfig from 'next-server/config'
|
||||
import createProgress from 'tty-aware-progress'
|
||||
|
||||
export default async function (dir, options, configuration) {
|
||||
function log (message) {
|
||||
|
@ -17,6 +19,8 @@ export default async function (dir, options, configuration) {
|
|||
|
||||
dir = resolve(dir)
|
||||
const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir)
|
||||
const concurrency = options.concurrency || 10
|
||||
const threads = options.threads || Math.max(cpus().length - 1, 1)
|
||||
const distDir = join(dir, nextConfig.distDir)
|
||||
|
||||
log(`> using build directory: ${distDir}`)
|
||||
|
@ -108,35 +112,48 @@ export default async function (dir, options, configuration) {
|
|||
nextExport: true
|
||||
}
|
||||
|
||||
log(` launching ${threads} threads with concurrency of ${concurrency} per thread`)
|
||||
const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {dev: false, dir, outDir, distDir, buildId})
|
||||
const exportPaths = Object.keys(exportPathMap)
|
||||
|
||||
for (const path of exportPaths) {
|
||||
log(`> exporting path: ${path}`)
|
||||
if (!path.startsWith('/')) {
|
||||
throw new Error(`path "${path}" doesn't start with a backslash`)
|
||||
const progress = !options.silent && createProgress(exportPaths.length)
|
||||
|
||||
const chunks = exportPaths.reduce((result, route, i) => {
|
||||
const worker = i % threads
|
||||
if (!result[worker]) {
|
||||
result[worker] = { paths: [], pathMap: {} }
|
||||
}
|
||||
result[worker].pathMap[route] = exportPathMap[route]
|
||||
result[worker].paths.push(route)
|
||||
return result
|
||||
}, [])
|
||||
|
||||
const { page, query = {} } = exportPathMap[path]
|
||||
const req = { url: path }
|
||||
const res = {}
|
||||
|
||||
let htmlFilename = `${path}${sep}index.html`
|
||||
if (extname(path) !== '') {
|
||||
// If the path has an extension, use that as the filename instead
|
||||
htmlFilename = path
|
||||
} else if (path === '/') {
|
||||
// If the path is the root, just use index.html
|
||||
htmlFilename = 'index.html'
|
||||
}
|
||||
const baseDir = join(outDir, dirname(htmlFilename))
|
||||
const htmlFilepath = join(outDir, htmlFilename)
|
||||
|
||||
await mkdirp(baseDir)
|
||||
|
||||
const html = await renderToHTML(req, res, page, query, renderOpts)
|
||||
writeFileSync(htmlFilepath, html, 'utf8')
|
||||
}
|
||||
await Promise.all(
|
||||
chunks.map(
|
||||
chunk =>
|
||||
new Promise((resolve, reject) => {
|
||||
const worker = fork(require.resolve('./worker'), [], {
|
||||
env: process.env
|
||||
})
|
||||
worker.send({
|
||||
exportPaths: chunk.paths,
|
||||
exportPathMap: chunk.pathMap,
|
||||
outDir,
|
||||
renderOpts,
|
||||
concurrency
|
||||
})
|
||||
worker.on('message', ({ type, payload }) => {
|
||||
if (type === 'progress' && progress) {
|
||||
progress()
|
||||
} else if (type === 'error') {
|
||||
reject(payload)
|
||||
} else if (type === 'done') {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
// Add an empty line to the console for the better readability.
|
||||
log('')
|
||||
|
|
59
packages/next/export/worker.js
Normal file
59
packages/next/export/worker.js
Normal file
|
@ -0,0 +1,59 @@
|
|||
global.__NEXT_DATA__ = {
|
||||
nextExport: true
|
||||
}
|
||||
|
||||
const { extname, join, dirname, sep } = require('path')
|
||||
const mkdirp = require('mkdirp-then')
|
||||
const { renderToHTML } = require('next-server/dist/server/render')
|
||||
const { writeFile } = require('fs')
|
||||
const Sema = require('async-sema')
|
||||
|
||||
process.on(
|
||||
'message',
|
||||
async ({
|
||||
exportPaths,
|
||||
exportPathMap,
|
||||
outDir,
|
||||
renderOpts,
|
||||
concurrency
|
||||
}) => {
|
||||
const sema = new Sema(concurrency, { capacity: exportPaths.length })
|
||||
try {
|
||||
const work = async path => {
|
||||
await sema.acquire()
|
||||
const { page, query = {} } = exportPathMap[path]
|
||||
const req = { url: path }
|
||||
const res = {}
|
||||
|
||||
let htmlFilename = `${path}${sep}index.html`
|
||||
if (extname(path) !== '') {
|
||||
// If the path has an extension, use that as the filename instead
|
||||
htmlFilename = path
|
||||
} else if (path === '/') {
|
||||
// If the path is the root, just use index.html
|
||||
htmlFilename = 'index.html'
|
||||
}
|
||||
const baseDir = join(outDir, dirname(htmlFilename))
|
||||
const htmlFilepath = join(outDir, htmlFilename)
|
||||
|
||||
await mkdirp(baseDir)
|
||||
const html = await renderToHTML(req, res, page, query, renderOpts)
|
||||
await new Promise((resolve, reject) =>
|
||||
writeFile(
|
||||
htmlFilepath,
|
||||
html,
|
||||
'utf8',
|
||||
err => (err ? reject(err) : resolve())
|
||||
)
|
||||
)
|
||||
process.send({ type: 'progress' })
|
||||
sema.release()
|
||||
}
|
||||
await Promise.all(exportPaths.map(work))
|
||||
process.send({ type: 'done' })
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
process.send({ type: 'error', payload: err })
|
||||
}
|
||||
}
|
||||
)
|
|
@ -49,6 +49,7 @@
|
|||
"@babel/runtime-corejs2": "7.1.2",
|
||||
"@babel/template": "7.1.2",
|
||||
"ansi-html": "0.0.7",
|
||||
"async-sema": "^2.1.4",
|
||||
"autodll-webpack-plugin": "0.4.2",
|
||||
"babel-core": "7.0.0-bridge.0",
|
||||
"babel-loader": "8.0.2",
|
||||
|
@ -79,6 +80,7 @@
|
|||
"strip-ansi": "3.0.1",
|
||||
"styled-jsx": "3.1.1",
|
||||
"terser-webpack-plugin": "1.1.0",
|
||||
"tty-aware-progress": "1.0.3",
|
||||
"unfetch": "3.0.0",
|
||||
"url": "0.11.0",
|
||||
"webpack": "4.26.0",
|
||||
|
|
Loading…
Reference in a new issue