From e6c36866294a9e32ec644dd6698f6df3bc78f29d Mon Sep 17 00:00:00 2001 From: Benjamin Kniffler Date: Wed, 12 Dec 2018 13:59:11 +0100 Subject: [PATCH] multi-threaded export with nice progress indication (#5870) This PR will - allow nextjs export to use all available CPU cores for rendering & writing pages by using child_process - make use of async-sema to allow each thread to concurrently write multiple paths - show a fancy progress bar while processing pages (with non-TTY fallback for CI web consoles) The performance gain for my MacBook with 4 CPU cores went from ~25 pages per second to ~75 pages per second. Beefy CI machines with lots of cores should profit even more. --- packages/next/bin/next-export | 2 + packages/next/export/index.js | 73 +++++++++++++++++++++------------- packages/next/export/worker.js | 59 +++++++++++++++++++++++++++ packages/next/package.json | 2 + 4 files changed, 108 insertions(+), 28 deletions(-) create mode 100644 packages/next/export/worker.js diff --git a/packages/next/bin/next-export b/packages/next/bin/next-export index a012be4d..ddb93020 100755 --- a/packages/next/bin/next-export +++ b/packages/next/bin/next-export @@ -54,6 +54,8 @@ if (!existsSync(join(dir, 'pages'))) { const options = { silent: argv.silent, + threads: argv.threads, + concurrency: argv.concurrency, outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out') } diff --git a/packages/next/export/index.js b/packages/next/export/index.js index c5bd29d2..49f5cf51 100644 --- a/packages/next/export/index.js +++ b/packages/next/export/index.js @@ -1,13 +1,15 @@ import del from 'del' +import { cpus } from 'os' +import { fork } from 'child_process' import cp from 'recursive-copy' import mkdirp from 'mkdirp-then' -import { extname, resolve, join, dirname, sep } from 'path' -import { existsSync, readFileSync, writeFileSync } from 'fs' +import { resolve, join } from 'path' +import { existsSync, readFileSync } from 'fs' import loadConfig from 'next-server/next-config' -import {PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH} from 'next-server/constants' -import { renderToHTML } from 'next-server/dist/server/render' +import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants' import { setAssetPrefix } from 'next-server/asset' import * as envConfig from 'next-server/config' +import createProgress from 'tty-aware-progress' export default async function (dir, options, configuration) { function log (message) { @@ -17,6 +19,8 @@ export default async function (dir, options, configuration) { dir = resolve(dir) const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir) + const concurrency = options.concurrency || 10 + const threads = options.threads || Math.max(cpus().length - 1, 1) const distDir = join(dir, nextConfig.distDir) log(`> using build directory: ${distDir}`) @@ -108,35 +112,48 @@ export default async function (dir, options, configuration) { nextExport: true } + log(` launching ${threads} threads with concurrency of ${concurrency} per thread`) const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {dev: false, dir, outDir, distDir, buildId}) const exportPaths = Object.keys(exportPathMap) - for (const path of exportPaths) { - log(`> exporting path: ${path}`) - if (!path.startsWith('/')) { - throw new Error(`path "${path}" doesn't start with a backslash`) + const progress = !options.silent && createProgress(exportPaths.length) + + const chunks = exportPaths.reduce((result, route, i) => { + const worker = i % threads + if (!result[worker]) { + result[worker] = { paths: [], pathMap: {} } } + result[worker].pathMap[route] = exportPathMap[route] + result[worker].paths.push(route) + return result + }, []) - const { page, query = {} } = exportPathMap[path] - const req = { url: path } - const res = {} - - let htmlFilename = `${path}${sep}index.html` - if (extname(path) !== '') { - // If the path has an extension, use that as the filename instead - htmlFilename = path - } else if (path === '/') { - // If the path is the root, just use index.html - htmlFilename = 'index.html' - } - const baseDir = join(outDir, dirname(htmlFilename)) - const htmlFilepath = join(outDir, htmlFilename) - - await mkdirp(baseDir) - - const html = await renderToHTML(req, res, page, query, renderOpts) - writeFileSync(htmlFilepath, html, 'utf8') - } + await Promise.all( + chunks.map( + chunk => + new Promise((resolve, reject) => { + const worker = fork(require.resolve('./worker'), [], { + env: process.env + }) + worker.send({ + exportPaths: chunk.paths, + exportPathMap: chunk.pathMap, + outDir, + renderOpts, + concurrency + }) + worker.on('message', ({ type, payload }) => { + if (type === 'progress' && progress) { + progress() + } else if (type === 'error') { + reject(payload) + } else if (type === 'done') { + resolve() + } + }) + }) + ) + ) // Add an empty line to the console for the better readability. log('') diff --git a/packages/next/export/worker.js b/packages/next/export/worker.js new file mode 100644 index 00000000..432ac387 --- /dev/null +++ b/packages/next/export/worker.js @@ -0,0 +1,59 @@ +global.__NEXT_DATA__ = { + nextExport: true +} + +const { extname, join, dirname, sep } = require('path') +const mkdirp = require('mkdirp-then') +const { renderToHTML } = require('next-server/dist/server/render') +const { writeFile } = require('fs') +const Sema = require('async-sema') + +process.on( + 'message', + async ({ + exportPaths, + exportPathMap, + outDir, + renderOpts, + concurrency + }) => { + const sema = new Sema(concurrency, { capacity: exportPaths.length }) + try { + const work = async path => { + await sema.acquire() + const { page, query = {} } = exportPathMap[path] + const req = { url: path } + const res = {} + + let htmlFilename = `${path}${sep}index.html` + if (extname(path) !== '') { + // If the path has an extension, use that as the filename instead + htmlFilename = path + } else if (path === '/') { + // If the path is the root, just use index.html + htmlFilename = 'index.html' + } + const baseDir = join(outDir, dirname(htmlFilename)) + const htmlFilepath = join(outDir, htmlFilename) + + await mkdirp(baseDir) + const html = await renderToHTML(req, res, page, query, renderOpts) + await new Promise((resolve, reject) => + writeFile( + htmlFilepath, + html, + 'utf8', + err => (err ? reject(err) : resolve()) + ) + ) + process.send({ type: 'progress' }) + sema.release() + } + await Promise.all(exportPaths.map(work)) + process.send({ type: 'done' }) + } catch (err) { + console.error(err) + process.send({ type: 'error', payload: err }) + } + } +) diff --git a/packages/next/package.json b/packages/next/package.json index fd81ddf1..122006bd 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -49,6 +49,7 @@ "@babel/runtime-corejs2": "7.1.2", "@babel/template": "7.1.2", "ansi-html": "0.0.7", + "async-sema": "^2.1.4", "autodll-webpack-plugin": "0.4.2", "babel-core": "7.0.0-bridge.0", "babel-loader": "8.0.2", @@ -79,6 +80,7 @@ "strip-ansi": "3.0.1", "styled-jsx": "3.1.1", "terser-webpack-plugin": "1.1.0", + "tty-aware-progress": "1.0.3", "unfetch": "3.0.0", "url": "0.11.0", "webpack": "4.26.0",