Implement "on demand entries" (#1111)

* Add a plan for dynamic entry middleware.

* Use dynamic pages middleware to load pages in dev.

* Add the first version of middleware but not tested.

* Integrated.

* Disable prefetching in development.
Otherwise it'll discard the use of dynamic-entries.

* Build custom document and error always.

* Refactor code base.

* Change branding as on-demand entries.

* Fix tests.

* Add a client side pinger for on-demand-entries.

* Dispose inactive entries.

* Add proper logs.

* Update grammer changes.

* Add integration tests for ondemand entries.

* Improve ondemand entry disposing logic.

* Try to improve testing.

*  Make sure entries are not getting disposed in basic integration tests.

* Resolve conflicts.

* Fix tests.

* Fix issue when running Router.onRouteChangeComplete

* Simplify state management.

* Make sure we don't dispose the last active page.

* Reload invalid pages detected with the client side ping.

* Improve the pinger code.

* Touch the first page to speed up the future rebuild times.

* Add Websockets based pinger.

* Revert "Add Websockets based pinger."

This reverts commit f706a49a3d886d0231259b7a1fded750ced2e48f.

* Do not send requests per every route change.

* Make sure we are completing the middleware request always.

* Make sure test pages are prebuilt.

client/on-demand-entries-client.js

@@ -0,0 +1,29 @@
+/* global location */
+
+import Router from '../lib/router'
+import fetch from 'unfetch'
+
+async function ping () {
+  try {
+    const url = `/on-demand-entries-ping?page=${Router.pathname}`
+    const res = await fetch(url)
+    const payload = await res.json()
+    if (payload.invalid) {
+      location.reload()
+    }
+  } catch (err) {
+    console.error(`Error with on-demand-entries-ping: ${err.message}`)
+  }
+}
+
+async function runPinger () {
+  while (true) {
+    await new Promise((resolve) => setTimeout(resolve, 5000))
+    await ping()
+  }
+}
+
+runPinger()
+  .catch((err) => {
+    console.error(err)
+  })

lib/router/router.js

@@ -224,6 +224,10 @@ export default class Router extends EventEmitter {
   }
 
   async prefetch (url) {
+    // We don't add support for prefetch in the development mode.
+    // If we do that, our on-demand-entries optimization won't performs better
+    if (process.env.NODE_ENV === 'development') return
+
     const { pathname } = parse(url)
     const route = toRoute(pathname)
     return this.prefetchQueue.add(() => this.fetchRoute(route))

package.json

@@ -78,6 +78,7 @@
     "source-map-support": "0.4.11",
     "strip-ansi": "3.0.1",
     "styled-jsx": "0.5.7",
+    "touch": "1.0.0",
     "unfetch": "2.1.0",
     "url": "0.11.0",
     "uuid": "3.0.1",

server/build/webpack.js

@@ -6,7 +6,6 @@ import WriteFilePlugin from 'write-file-webpack-plugin'
 import FriendlyErrorsWebpackPlugin from 'friendly-errors-webpack-plugin'
 import CaseSensitivePathPlugin from 'case-sensitive-paths-webpack-plugin'
 import UnlinkFilePlugin from './plugins/unlink-file-plugin'
-import WatchPagesPlugin from './plugins/watch-pages-plugin'
 import JsonPagesPlugin from './plugins/json-pages-plugin'
 import getConfig from '../config'
 import * as babelCore from 'babel-core'
@@ -40,9 +39,19 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     const entries = { 'main.js': mainJS }
 
     const pages = await glob('pages/**/*.js', { cwd: dir })
+    const devPages = pages.filter((p) => p === 'pages/_document.js' || p === 'pages/_error.js')
+
+    // In the dev environment, on-demand-entry-handler will take care of
+    // managing pages.
+    if (dev) {
+      for (const p of devPages) {
+        entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
+      }
+    } else {
       for (const p of pages) {
         entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
       }
+    }
 
     for (const p of defaultPages) {
       const entryName = join('bundles', 'pages', p)
@@ -81,6 +90,9 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
         return count >= minChunks
       }
     }),
+    new webpack.DefinePlugin({
+      'process.env.NODE_ENV': JSON.stringify(dev ? 'development' : 'production')
+    }),
     new JsonPagesPlugin(),
     new CaseSensitivePathPlugin()
   ]
@@ -89,17 +101,13 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     plugins.push(
       new webpack.HotModuleReplacementPlugin(),
       new webpack.NoEmitOnErrorsPlugin(),
-      new UnlinkFilePlugin(),
+      new UnlinkFilePlugin()
-      new WatchPagesPlugin(dir)
     )
     if (!quiet) {
       plugins.push(new FriendlyErrorsWebpackPlugin())
     }
   } else {
     plugins.push(
-      new webpack.DefinePlugin({
-        'process.env.NODE_ENV': JSON.stringify('production')
-      }),
       new webpack.optimize.UglifyJsPlugin({
         compress: { warnings: false },
         sourceMap: false

server/hot-reloader.js

@@ -1,10 +1,12 @@
 import { join, relative, sep } from 'path'
 import webpackDevMiddleware from 'webpack-dev-middleware'
 import webpackHotMiddleware from 'webpack-hot-middleware'
+import onDemandEntryHandler from './on-demand-entry-handler'
 import isWindowsBash from 'is-windows-bash'
 import webpack from './build/webpack'
 import clean from './build/clean'
 import readPage from './read-page'
+import getConfig from './config'
 
 export default class HotReloader {
   constructor (dir, { quiet } = {}) {
@@ -20,6 +22,8 @@ export default class HotReloader {
     this.prevChunkNames = null
     this.prevFailedChunkNames = null
     this.prevChunkHashes = null
+
+    this.config = getConfig(dir)
   }
 
   async run (req, res) {
@@ -145,10 +149,16 @@ export default class HotReloader {
     })
 
     this.webpackHotMiddleware = webpackHotMiddleware(compiler, { log: false })
+    this.onDemandEntries = onDemandEntryHandler(this.webpackDevMiddleware, compiler, {
+      dir: this.dir,
+      dev: true,
+      ...this.config.onDemandEntries
+    })
 
     this.middlewares = [
       this.webpackDevMiddleware,
-      this.webpackHotMiddleware
+      this.webpackHotMiddleware,
+      this.onDemandEntries.middleware()
     ]
   }
 
@@ -184,6 +194,10 @@ export default class HotReloader {
   send (action, ...args) {
     this.webpackHotMiddleware.publish({ action, data: args })
   }
+
+  ensurePage (page) {
+    return this.onDemandEntries.ensurePage(page)
+  }
 }
 
 function deleteCache (path) {

server/index.js

@@ -22,9 +22,9 @@ export default class Server {
     this.dir = resolve(dir)
     this.dev = dev
     this.quiet = quiet
-    this.renderOpts = { dir: this.dir, dev, staticMarkup }
     this.router = new Router()
     this.hotReloader = dev ? new HotReloader(this.dir, { quiet }) : null
+    this.renderOpts = { dir: this.dir, dev, staticMarkup, hotReloader: this.hotReloader }
     this.http = null
     this.config = getConfig(this.dir)
 
@@ -107,6 +107,7 @@ export default class Server {
 
         const paths = params.path || ['index']
         const pathname = `/${paths.join('/')}`
+
         await this.renderJSON(req, res, pathname)
       },
 

server/on-demand-entry-handler.js

@@ -0,0 +1,184 @@
+import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
+import { EventEmitter } from 'events'
+import { join } from 'path'
+import { parse } from 'url'
+import resolvePath from './resolve'
+import touch from 'touch'
+
+const ADDED = Symbol()
+const BUILDING = Symbol()
+const BUILT = Symbol()
+
+export default function onDemandEntryHandler (devMiddleware, compiler, {
+  dir,
+  dev,
+  maxInactiveAge = 1000 * 25
+}) {
+  const entries = {}
+  const lastAccessPages = ['']
+  const doneCallbacks = new EventEmitter()
+  let touchedAPage = false
+
+  compiler.plugin('make', function (compilation, done) {
+    const allEntries = Object.keys(entries).map((page) => {
+      const { name, entry } = entries[page]
+      entries[page].status = BUILDING
+      return addEntry(compilation, this.context, name, entry)
+    })
+
+    Promise.all(allEntries)
+      .then(() => done())
+      .catch(done)
+  })
+
+  compiler.plugin('done', function (stats) {
+    // Call all the doneCallbacks
+    Object.keys(entries).forEach((page) => {
+      const entryInfo = entries[page]
+      if (entryInfo.status !== BUILDING) return
+
+      // With this, we are triggering a filesystem based watch trigger
+      // It'll memorize some timestamp related info related to common files used
+      // in the page
+      // That'll reduce the page building time significantly.
+      if (!touchedAPage) {
+        setTimeout(() => {
+          touch.sync(entryInfo.pathname)
+        }, 0)
+        touchedAPage = true
+      }
+
+      entryInfo.status = BUILT
+      entries[page].lastActiveTime = Date.now()
+      doneCallbacks.emit(page)
+    })
+  })
+
+  setInterval(function () {
+    disposeInactiveEntries(devMiddleware, entries, lastAccessPages, maxInactiveAge)
+  }, 5000)
+
+  return {
+    async ensurePage (page) {
+      page = normalizePage(page)
+
+      const pagePath = join(dir, 'pages', page)
+      const pathname = await resolvePath(pagePath)
+      const name = join('bundles', pathname.substring(dir.length))
+
+      const entry = [
+        join(__dirname, '..', 'client/webpack-hot-middleware-client'),
+        join(__dirname, '..', 'client', 'on-demand-entries-client'),
+        `${pathname}?entry`
+      ]
+
+      await new Promise((resolve, reject) => {
+        const entryInfo = entries[page]
+
+        if (entryInfo) {
+          if (entryInfo.status === BUILT) {
+            resolve()
+            return
+          }
+
+          if (entryInfo.status === BUILDING) {
+            doneCallbacks.on(page, processCallback)
+            return
+          }
+        }
+
+        console.log(`> Building page: ${page}`)
+
+        entries[page] = { name, entry, pathname, status: ADDED }
+        doneCallbacks.on(page, processCallback)
+
+        devMiddleware.invalidate()
+
+        function processCallback (err) {
+          if (err) return reject(err)
+          resolve()
+        }
+      })
+    },
+
+    middleware () {
+      return function (req, res, next) {
+        if (!/^\/on-demand-entries-ping/.test(req.url)) return next()
+
+        const { query } = parse(req.url, true)
+        const page = normalizePage(query.page)
+        const entryInfo = entries[page]
+
+        // If there's no entry.
+        // Then it seems like an weird issue.
+        if (!entryInfo) {
+          const message = `Client pings, but there's no entry for page: ${page}`
+          console.error(message)
+          sendJson(res, { invalid: true })
+          return
+        }
+
+        sendJson(res, { success: true })
+
+        // We don't need to maintain active state of anything other than BUILT entries
+        if (entryInfo.status !== BUILT) return
+
+        // If there's an entryInfo
+        lastAccessPages.pop()
+        lastAccessPages.unshift(page)
+        entryInfo.lastActiveTime = Date.now()
+      }
+    }
+  }
+}
+
+function addEntry (compilation, context, name, entry) {
+  return new Promise((resolve, reject) => {
+    const dep = DynamicEntryPlugin.createDependency(entry, name)
+    compilation.addEntry(context, dep, name, (err) => {
+      if (err) return reject(err)
+      resolve()
+    })
+  })
+}
+
+function disposeInactiveEntries (devMiddleware, entries, lastAccessPages, maxInactiveAge) {
+  const disposingPages = []
+
+  Object.keys(entries).forEach((page) => {
+    const { lastActiveTime, status } = entries[page]
+
+    // This means this entry is currently building or just added
+    // We don't need to dispose those entries.
+    if (status !== BUILT) return
+
+    // We should not build the last accessed page even we didn't get any pings
+    // Sometimes, it's possible our XHR ping to wait before completing other requests.
+    // In that case, we should not dispose the current viewing page
+    if (lastAccessPages[0] === page) return
+
+    if (Date.now() - lastActiveTime > maxInactiveAge) {
+      disposingPages.push(page)
+    }
+  })
+
+  if (disposingPages.length > 0) {
+    disposingPages.forEach((page) => {
+      delete entries[page]
+    })
+    console.log(`> Disposing inactive page(s): ${disposingPages.join(', ')}`)
+    devMiddleware.invalidate()
+  }
+}
+
+// /index and / is the same. So, we need to identify both pages as the same.
+// This also applies to sub pages as well.
+function normalizePage (page) {
+  return page.replace(/\/index$/, '/')
+}
+
+function sendJson (res, payload) {
+  res.setHeader('Content-Type', 'application/json')
+  res.status = 200
+  res.end(JSON.stringify(payload))
+}

server/render.js

@@ -32,11 +32,15 @@ async function doRender (req, res, pathname, query, {
   err,
   page,
   buildId,
+  hotReloader,
   dir = process.cwd(),
   dev = false,
   staticMarkup = false
 } = {}) {
   page = page || pathname
+
+  await ensurePage(page, { dir, hotReloader })
+
   let [Component, Document] = await Promise.all([
     requireModule(join(dir, '.next', 'dist', 'pages', page)),
     requireModule(join(dir, '.next', 'dist', 'pages', '_document'))
@@ -100,7 +104,8 @@ async function doRender (req, res, pathname, query, {
   return '<!DOCTYPE html>' + renderToStaticMarkup(doc)
 }
 
-export async function renderJSON (req, res, page, { dir = process.cwd() } = {}) {
+export async function renderJSON (req, res, page, { dir = process.cwd(), hotReloader } = {}) {
+  await ensurePage(page, { dir, hotReloader })
   const pagePath = await resolvePath(join(dir, '.next', 'bundles', 'pages', page))
   return serveStatic(req, res, pagePath)
 }
@@ -158,3 +163,10 @@ export function serveStatic (req, res, path) {
     .on('finish', resolve)
   })
 }
+
+async function ensurePage (page, { dir, hotReloader }) {
+  if (!hotReloader) return
+  if (page === '_error' || page === '_document') return
+
+  await hotReloader.ensurePage(page)
+}

test/integration/basic/next.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  onDemandEntries: {
+    // Make sure entries are not getting disposed.
+    maxInactiveAge: 1000 * 60 * 60
+  }
+}

test/integration/basic/test/client-navigation.js

@@ -2,7 +2,7 @@
 
 import webdriver from 'next-webdriver'
 
-export default (context) => {
+export default (context, render) => {
   describe('Client Navigation', () => {
     describe('with <Link/>', () => {
       it('should navigate the page', async () => {

test/integration/basic/test/index.test.js

@@ -28,6 +28,25 @@ describe('Basic Features', () => {
   beforeAll(async () => {
     context.server = await startApp(context.app)
     context.appPort = context.server.address().port
+
+    // pre-build all pages at the start
+    await Promise.all([
+      renderViaHTTP(context.appPort, '/async-props'),
+      renderViaHTTP(context.appPort, '/empty-get-initial-props'),
+      renderViaHTTP(context.appPort, '/error'),
+      renderViaHTTP(context.appPort, '/finish-response'),
+      renderViaHTTP(context.appPort, '/head'),
+      renderViaHTTP(context.appPort, '/json'),
+      renderViaHTTP(context.appPort, '/link'),
+      renderViaHTTP(context.appPort, '/stateful'),
+      renderViaHTTP(context.appPort, '/stateless'),
+      renderViaHTTP(context.appPort, '/styled-jsx'),
+
+      renderViaHTTP(context.appPort, '/nav'),
+      renderViaHTTP(context.appPort, '/nav/about'),
+      renderViaHTTP(context.appPort, '/nav/querystring'),
+      renderViaHTTP(context.appPort, '/nav/self-reload')
+    ])
   })
   afterAll(() => stopApp(context.server))
 
@@ -35,5 +54,5 @@ describe('Basic Features', () => {
   rendering(context, 'Rendering via HTTP', (p, q) => renderViaHTTP(context.appPort, p, q))
   xPoweredBy(context)
   misc(context)
-  clientNavigation(context)
+  clientNavigation(context, (p, q) => renderViaHTTP(context.appPort, p, q))
 })

test/integration/ondemand/next.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  onDemandEntries: {
+    maxInactiveAge: 1000 * 5
+  }
+}

test/integration/ondemand/pages/about.js

@@ -0,0 +1,5 @@
+export default () => (
+  <div>
+    <p>About Page</p>
+  </div>
+)

test/integration/ondemand/pages/index.js

@@ -0,0 +1,8 @@
+import Link from 'next/link'
+
+export default () => (
+  <div>
+    <p>Index Page</p>
+    <Link href='/about'><a id='about-link'>About Page</a></Link>
+  </div>
+)

test/integration/ondemand/test/index.test.js

@@ -0,0 +1,50 @@
+/* global jasmine, describe, beforeAll, afterAll, it, expect */
+import { join, resolve } from 'path'
+import { existsSync } from 'fs'
+import {
+  nextServer,
+  renderViaHTTP,
+  startApp,
+  stopApp,
+  waitFor
+} from 'next-test-utils'
+
+const context = {}
+context.app = nextServer({
+  dir: join(__dirname, '../'),
+  dev: true,
+  quiet: true
+})
+
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 40000
+
+describe('On Demand Entries', () => {
+  beforeAll(async () => {
+    context.server = await startApp(context.app)
+    context.appPort = context.server.address().port
+  })
+  afterAll(() => stopApp(context.server))
+
+  it('should compile pages for SSR', async () => {
+    const pageContent = await renderViaHTTP(context.appPort, '/')
+    expect(pageContent.includes('Index Page')).toBeTruthy()
+  })
+
+  it('should compile pages for JSON page requests', async () => {
+    const pageContent = await renderViaHTTP(context.appPort, '/_next/-/pages/about')
+    expect(pageContent.includes('About Page')).toBeTruthy()
+  })
+
+  it('should dispose inactive pages', async () => {
+    await renderViaHTTP(context.appPort, '/_next/-/pages/about')
+    const aboutPagePath = resolve(__dirname, '../.next/bundles/pages/about.json')
+    expect(existsSync(aboutPagePath)).toBeTruthy()
+
+    // Wait maximum of jasmine.DEFAULT_TIMEOUT_INTERVAL checking
+    // for disposing /about
+    while (true) {
+      await waitFor(1000 * 1)
+      if (!existsSync(aboutPagePath)) return
+    }
+  })
+})

test/lib/next-test-utils.js

@@ -47,3 +47,7 @@ function promiseCall (obj, method, ...args) {
     obj[method](...newArgs)
   })
 }
+
+export function waitFor (millis) {
+  return new Promise((resolve) => setTimeout(resolve, millis))
+}