multi-threaded export with nice progress indication (#5870)

This PR will

- allow nextjs export to use all available CPU cores for rendering & writing pages by using child_process
- make use of async-sema to allow each thread to concurrently write multiple paths
- show a fancy progress bar while processing pages (with non-TTY fallback for CI web consoles)

The performance gain for my MacBook with 4 CPU cores went from ~25 pages per second to ~75 pages per second. Beefy CI machines with lots of cores should profit even more.

packages/next/bin/next-export

@@ -54,6 +54,8 @@ if (!existsSync(join(dir, 'pages'))) {
 
 const options = {
   silent: argv.silent,
+  threads: argv.threads,
+  concurrency: argv.concurrency,
   outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
 }
 

packages/next/export/index.js

@@ -1,13 +1,15 @@
 import del from 'del'
+import { cpus } from 'os'
+import { fork } from 'child_process'
 import cp from 'recursive-copy'
 import mkdirp from 'mkdirp-then'
-import { extname, resolve, join, dirname, sep } from 'path'
-import { existsSync, readFileSync, writeFileSync } from 'fs'
+import { resolve, join } from 'path'
+import { existsSync, readFileSync } from 'fs'
 import loadConfig from 'next-server/next-config'
-import {PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH} from 'next-server/constants'
-import { renderToHTML } from 'next-server/dist/server/render'
+import { PHASE_EXPORT, SERVER_DIRECTORY, PAGES_MANIFEST, CONFIG_FILE, BUILD_ID_FILE, CLIENT_STATIC_FILES_PATH } from 'next-server/constants'
 import { setAssetPrefix } from 'next-server/asset'
 import * as envConfig from 'next-server/config'
+import createProgress from 'tty-aware-progress'
 
 export default async function (dir, options, configuration) {
   function log (message) {
@@ -17,6 +19,8 @@ export default async function (dir, options, configuration) {
 
   dir = resolve(dir)
   const nextConfig = configuration || loadConfig(PHASE_EXPORT, dir)
+  const concurrency = options.concurrency || 10
+  const threads = options.threads || Math.max(cpus().length - 1, 1)
   const distDir = join(dir, nextConfig.distDir)
 
   log(`> using build directory: ${distDir}`)
@@ -108,35 +112,48 @@ export default async function (dir, options, configuration) {
     nextExport: true
   }
 
+  log(`  launching ${threads} threads with concurrency of ${concurrency} per thread`)
   const exportPathMap = await nextConfig.exportPathMap(defaultPathMap, {dev: false, dir, outDir, distDir, buildId})
   const exportPaths = Object.keys(exportPathMap)
 
-  for (const path of exportPaths) {
-    log(`> exporting path: ${path}`)
-    if (!path.startsWith('/')) {
-      throw new Error(`path "${path}" doesn't start with a backslash`)
+  const progress = !options.silent && createProgress(exportPaths.length)
+
+  const chunks = exportPaths.reduce((result, route, i) => {
+    const worker = i % threads
+    if (!result[worker]) {
+      result[worker] = { paths: [], pathMap: {} }
     }
+    result[worker].pathMap[route] = exportPathMap[route]
+    result[worker].paths.push(route)
+    return result
+  }, [])
 
-    const { page, query = {} } = exportPathMap[path]
-    const req = { url: path }
-    const res = {}
-
-    let htmlFilename = `${path}${sep}index.html`
-    if (extname(path) !== '') {
-      // If the path has an extension, use that as the filename instead
-      htmlFilename = path
-    } else if (path === '/') {
-      // If the path is the root, just use index.html
-      htmlFilename = 'index.html'
-    }
-    const baseDir = join(outDir, dirname(htmlFilename))
-    const htmlFilepath = join(outDir, htmlFilename)
-
-    await mkdirp(baseDir)
-
-    const html = await renderToHTML(req, res, page, query, renderOpts)
-    writeFileSync(htmlFilepath, html, 'utf8')
-  }
+  await Promise.all(
+    chunks.map(
+      chunk =>
+        new Promise((resolve, reject) => {
+          const worker = fork(require.resolve('./worker'), [], {
+            env: process.env
+          })
+          worker.send({
+            exportPaths: chunk.paths,
+            exportPathMap: chunk.pathMap,
+            outDir,
+            renderOpts,
+            concurrency
+          })
+          worker.on('message', ({ type, payload }) => {
+            if (type === 'progress' && progress) {
+              progress()
+            } else if (type === 'error') {
+              reject(payload)
+            } else if (type === 'done') {
+              resolve()
+            }
+          })
+        })
+    )
+  )
 
   // Add an empty line to the console for the better readability.
   log('')

packages/next/export/worker.js

@@ -0,0 +1,59 @@
+global.__NEXT_DATA__ = {
+  nextExport: true
+}
+
+const { extname, join, dirname, sep } = require('path')
+const mkdirp = require('mkdirp-then')
+const { renderToHTML } = require('next-server/dist/server/render')
+const { writeFile } = require('fs')
+const Sema = require('async-sema')
+
+process.on(
+  'message',
+  async ({
+    exportPaths,
+    exportPathMap,
+    outDir,
+    renderOpts,
+    concurrency
+  }) => {
+    const sema = new Sema(concurrency, { capacity: exportPaths.length })
+    try {
+      const work = async path => {
+        await sema.acquire()
+        const { page, query = {} } = exportPathMap[path]
+        const req = { url: path }
+        const res = {}
+
+        let htmlFilename = `${path}${sep}index.html`
+        if (extname(path) !== '') {
+          // If the path has an extension, use that as the filename instead
+          htmlFilename = path
+        } else if (path === '/') {
+          // If the path is the root, just use index.html
+          htmlFilename = 'index.html'
+        }
+        const baseDir = join(outDir, dirname(htmlFilename))
+        const htmlFilepath = join(outDir, htmlFilename)
+
+        await mkdirp(baseDir)
+        const html = await renderToHTML(req, res, page, query, renderOpts)
+        await new Promise((resolve, reject) =>
+          writeFile(
+            htmlFilepath,
+            html,
+            'utf8',
+            err => (err ? reject(err) : resolve())
+          )
+        )
+        process.send({ type: 'progress' })
+        sema.release()
+      }
+      await Promise.all(exportPaths.map(work))
+      process.send({ type: 'done' })
+    } catch (err) {
+      console.error(err)
+      process.send({ type: 'error', payload: err })
+    }
+  }
+)

packages/next/package.json

@@ -49,6 +49,7 @@
     "@babel/runtime-corejs2": "7.1.2",
     "@babel/template": "7.1.2",
     "ansi-html": "0.0.7",
+    "async-sema": "^2.1.4",
     "autodll-webpack-plugin": "0.4.2",
     "babel-core": "7.0.0-bridge.0",
     "babel-loader": "8.0.2",
@@ -79,6 +80,7 @@
     "strip-ansi": "3.0.1",
     "styled-jsx": "3.1.1",
     "terser-webpack-plugin": "1.1.0",
+    "tty-aware-progress": "1.0.3",
     "unfetch": "3.0.0",
     "url": "0.11.0",
     "webpack": "4.26.0",