Make pageExtensions configurable (#3787)

* Make page require faster

* Add windows search/replace

* Use normalize instead of resolve

* Add remaining tests

* Use sep instead of /

* Add test files

* Make component require faster

* Add console.error

* Make pageExtensions configurable

* Remove resolve.js

* Add test for `.jsx`

* Also resolve `/nav/index` and the likes

* Normalize page paths

* Use config passed off by webpack

server/build/webpack.js

@@ -115,7 +115,7 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
     externals: externalsConfig(dir, isServer),
     context: dir,
     entry: async () => {
-      const pages = await getPages(dir, {dev, isServer})
+      const pages = await getPages(dir, {dev, isServer, pageExtensions: config.pageExtensions.join('|')})
       totalPages = Object.keys(pages).length
       const mainJS = require.resolve(`../../client/next${dev ? '-dev' : ''}`)
       const clientConfig = !isServer ? {

server/build/webpack/utils.js

@@ -3,26 +3,28 @@ import glob from 'glob-promise'
 
 const nextPagesDir = path.join(__dirname, '..', '..', '..', 'pages')
 
-export async function getPages (dir, {dev, isServer}) {
-  const pageFiles = await getPagePaths(dir, {dev, isServer})
+export async function getPages (dir, {dev, isServer, pageExtensions}) {
+  const pageFiles = await getPagePaths(dir, {dev, isServer, pageExtensions})
 
-  return getPageEntries(pageFiles, {isServer})
+  return getPageEntries(pageFiles, {isServer, pageExtensions})
 }
 
-async function getPagePaths (dir, {dev, isServer}) {
+async function getPagePaths (dir, {dev, isServer, pageExtensions}) {
   let pages
 
   if (dev) {
-    pages = await glob(isServer ? 'pages/+(_document|_error).+(js|jsx|ts|tsx)' : 'pages/_error.+(js|jsx|ts|tsx)', { cwd: dir })
+    // In development we only compile _document.js and _error.js when starting, since they're always needed. All other pages are compiled with on demand entries
+    pages = await glob(isServer ? `pages/+(_document|_error).+(${pageExtensions})` : `pages/_error.+(${pageExtensions})`, { cwd: dir })
   } else {
-    pages = await glob(isServer ? 'pages/**/*.+(js|jsx|ts|tsx)' : 'pages/**/!(_document)*.+(js|jsx|ts|tsx)', { cwd: dir })
+    // In production get all pages from the pages directory
+    pages = await glob(isServer ? `pages/**/*.+(${pageExtensions})` : `pages/**/!(_document)*.+(${pageExtensions})`, { cwd: dir })
   }
 
   return pages
 }
 
 // Convert page path into single entry
-export function createEntry (filePath, name) {
+export function createEntry (filePath, {name, pageExtensions} = {}) {
   const parsedPath = path.parse(filePath)
   let entryName = name || filePath
 
@@ -33,7 +35,9 @@ export function createEntry (filePath, name) {
   }
 
   // Makes sure supported extensions are stripped off. The outputted file should always be `.js`
-  entryName = entryName.replace(/\.+(jsx|tsx|ts)/, '.js')
+  if (pageExtensions) {
+    entryName = entryName.replace(new RegExp(`\\.+(${pageExtensions})`), '.js')
+  }
 
   return {
     name: path.join('bundles', entryName),
@@ -42,23 +46,23 @@ export function createEntry (filePath, name) {
 }
 
 // Convert page paths into entries
-export function getPageEntries (pagePaths, {isServer}) {
+export function getPageEntries (pagePaths, {isServer = false, pageExtensions} = {}) {
   const entries = {}
 
   for (const filePath of pagePaths) {
-    const entry = createEntry(filePath)
+    const entry = createEntry(filePath, {pageExtensions})
     entries[entry.name] = entry.files
   }
 
   const errorPagePath = path.join(nextPagesDir, '_error.js')
-  const errorPageEntry = createEntry(errorPagePath, 'pages/_error.js') // default error.js
+  const errorPageEntry = createEntry(errorPagePath, {name: 'pages/_error.js'}) // default error.js
   if (!entries[errorPageEntry.name]) {
     entries[errorPageEntry.name] = errorPageEntry.files
   }
 
   if (isServer) {
     const documentPagePath = path.join(nextPagesDir, '_document.js')
-    const documentPageEntry = createEntry(documentPagePath, 'pages/_document.js')
+    const documentPageEntry = createEntry(documentPagePath, {name: 'pages/_document.js'}) // default _document.js
     if (!entries[documentPageEntry.name]) {
       entries[documentPageEntry.name] = documentPageEntry.files
     }

server/config.js

@@ -8,7 +8,8 @@ const defaultConfig = {
   distDir: '.next',
   assetPrefix: '',
   configOrigin: 'default',
-  useFileSystemPublicRoutes: true
+  useFileSystemPublicRoutes: true,
+  pageExtensions: ['jsx', 'js'] // jsx before js because otherwise regex matching will match js first
 }
 
 export default function getConfig (dir, customConfig) {

server/hot-reloader.js

@@ -228,6 +228,7 @@ export default class HotReloader {
       dir: this.dir,
       dev: true,
       reload: this.reload.bind(this),
+      pageExtensions: this.config.pageExtensions,
       ...this.config.onDemandEntries
     })
 

server/on-demand-entry-handler.js

@@ -1,9 +1,10 @@
 import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
 import { EventEmitter } from 'events'
-import { join, relative } from 'path'
+import { join } from 'path'
 import { parse } from 'url'
 import touch from 'touch'
-import resolvePath from './resolve'
+import glob from 'glob-promise'
+import {normalizePagePath, pageNotFoundError} from './require'
 import {createEntry} from './build/webpack/utils'
 import { MATCH_ROUTE_NAME, IS_BUNDLED_PAGE } from './utils'
 
@@ -15,6 +16,7 @@ export default function onDemandEntryHandler (devMiddleware, compilers, {
   dir,
   dev,
   reload,
+  pageExtensions,
   maxInactiveAge = 1000 * 60,
   pagesBufferLength = 2
 }) {
@@ -139,10 +141,26 @@ export default function onDemandEntryHandler (devMiddleware, compilers, {
     async ensurePage (page) {
       await this.waitUntilReloaded()
       page = normalizePage(page)
+      let normalizedPagePath
+      try {
+        normalizedPagePath = normalizePagePath(page)
+      } catch (err) {
+        console.error(err)
+        throw pageNotFoundError(normalizedPagePath)
+      }
 
-      const pagePath = join(dir, 'pages', page)
-      const pathname = await resolvePath(pagePath)
-      const {name, files} = createEntry(relative(dir, pathname))
+      const extensions = pageExtensions.join('|')
+      const paths = await glob(`pages/{${normalizedPagePath}/index,${normalizedPagePath}}.+(${extensions})`, {cwd: dir})
+
+      if (paths.length === 0) {
+        throw pageNotFoundError(normalizedPagePath)
+      }
+
+      const relativePathToPage = paths[0]
+
+      const pathname = join(dir, relativePathToPage)
+
+      const {name, files} = createEntry(relativePathToPage, {pageExtensions: extensions})
 
       await new Promise((resolve, reject) => {
         const entryInfo = entries[page]

server/resolve.js

@@ -1,96 +0,0 @@
-import { join, sep, parse } from 'path'
-import fs from 'mz/fs'
-import glob from 'glob-promise'
-
-export default async function resolve (id) {
-  const paths = getPaths(id)
-  for (const p of paths) {
-    if (await isFile(p)) {
-      return p
-    }
-  }
-
-  const err = new Error(`Cannot find module ${id}`)
-  err.code = 'ENOENT'
-  throw err
-}
-
-export function resolveFromList (id, files) {
-  const paths = getPaths(id)
-  const set = new Set(files)
-  for (const p of paths) {
-    if (set.has(p)) return p
-  }
-}
-
-function getPaths (id) {
-  const i = sep === '/' ? id : id.replace(/\//g, sep)
-
-  if (i.slice(-3) === '.js') return [i]
-  if (i.slice(-4) === '.jsx') return [i]
-  if (i.slice(-4) === '.tsx') return [i]
-  if (i.slice(-3) === '.ts') return [i]
-  if (i.slice(-5) === '.json') return [i]
-
-  if (i[i.length - 1] === sep) {
-    return [
-      i + 'index.js',
-      i + 'index.jsx',
-      i + 'index.ts',
-      i + 'index.tsx',
-      i + 'index.json'
-    ]
-  }
-
-  return [
-    i + '.js',
-    join(i, 'index.js'),
-    i + '.jsx',
-    join(i, 'index.jsx'),
-    i + '.tsx',
-    join(i, 'index.tsx'),
-    i + '.ts',
-    join(i, 'index.ts'),
-    i + '.json',
-    join(i, 'index.json')
-  ]
-}
-
-async function isFile (p) {
-  let stat
-  try {
-    stat = await fs.stat(p)
-  } catch (err) {
-    if (err.code === 'ENOENT') return false
-    throw err
-  }
-
-  // We need the path to be case sensitive
-  const realpath = await getTrueFilePath(p)
-  if (p !== realpath) return false
-
-  return stat.isFile() || stat.isFIFO()
-}
-
-// This is based on the stackoverflow answer: http://stackoverflow.com/a/33139702/457224
-// We assume we'll get properly normalized path names as p
-async function getTrueFilePath (p) {
-  let fsPathNormalized = p
-  // OSX: HFS+ stores filenames in NFD (decomposed normal form) Unicode format,
-  // so we must ensure that the input path is in that format first.
-  if (process.platform === 'darwin') fsPathNormalized = fsPathNormalized.normalize('NFD')
-
-  // !! Windows: Curiously, the drive component mustn't be part of a glob,
-  // !! otherwise glob.sync() will invariably match nothing.
-  // !! Thus, we remove the drive component and instead pass it in as the 'cwd'
-  // !! (working dir.) property below.
-  var pathRoot = parse(fsPathNormalized).root
-  var noDrivePath = fsPathNormalized.slice(Math.max(pathRoot.length - 1, 0))
-
-  // Perform case-insensitive globbing (on Windows, relative to the drive /
-  // network share) and return the 1st match, if any.
-  // Fortunately, glob() with nocase case-corrects the input even if it is
-  // a *literal* path.
-  const result = await glob(noDrivePath, { nocase: true, cwd: pathRoot })
-  return result[0]
-}

test/isolated/resolve.test.js

@@ -1,52 +0,0 @@
-/* global describe, it, expect */
-
-import { join } from 'path'
-import resolve from '../../dist/server/resolve'
-
-const dataPath = join(__dirname, '_resolvedata')
-
-describe('Resolve', () => {
-  it('should resolve a .js path', async () => {
-    const p = await resolve(join(dataPath, 'one.js'))
-    expect(p).toBe(join(dataPath, 'one.js'))
-  })
-
-  it('should resolve a .json path', async () => {
-    const p = await resolve(join(dataPath, 'two.json'))
-    expect(p).toBe(join(dataPath, 'two.json'))
-  })
-
-  it('should resolve a module without an extension', async () => {
-    const p = await resolve(join(dataPath, 'one'))
-    expect(p).toBe(join(dataPath, 'one.js'))
-  })
-
-  it('should resolve a .js module in a directory without /', async () => {
-    const p = await resolve(join(dataPath, 'aa'))
-    expect(p).toBe(join(dataPath, 'aa', 'index.js'))
-  })
-
-  it('should resolve a .js module in a directory with /', async () => {
-    const p = await resolve(join(dataPath, 'aa/'))
-    expect(p).toBe(join(dataPath, 'aa', 'index.js'))
-  })
-
-  it('should resolve a .json module in a directory', async () => {
-    const p = await resolve(join(dataPath, 'bb'))
-    expect(p).toBe(join(dataPath, 'bb', 'index.json'))
-  })
-
-  it('should resolve give priority to index.js over index.json', async () => {
-    const p = await resolve(join(dataPath, 'cc'))
-    expect(p).toBe(join(dataPath, 'cc', 'index.js'))
-  })
-
-  it('should throw an error for non existing paths', async () => {
-    try {
-      await resolve(join(dataPath, 'aaa.js'))
-      throw new Error('Should not run this line.')
-    } catch (ex) {
-      expect(ex.message).toMatch(/Cannot find module/)
-    }
-  })
-})

test/isolated/webpack-utils.test.js

@@ -0,0 +1,68 @@
+/* global describe, it, expect */
+
+import {normalize} from 'path'
+import {getPageEntries, createEntry} from '../../dist/server/build/webpack/utils'
+
+describe('createEntry', () => {
+  it('Should turn a path into a page entry', () => {
+    const entry = createEntry('pages/index.js')
+    expect(entry.name).toBe(normalize('bundles/pages/index.js'))
+    expect(entry.files[0]).toBe('./pages/index.js')
+  })
+
+  it('Should have a custom name', () => {
+    const entry = createEntry('pages/index.js', {name: 'something-else.js'})
+    expect(entry.name).toBe(normalize('bundles/something-else.js'))
+    expect(entry.files[0]).toBe('./pages/index.js')
+  })
+
+  it('Should allow custom extension like .ts to be turned into .js', () => {
+    const entry = createEntry('pages/index.ts', {pageExtensions: ['js', 'ts'].join('|')})
+    expect(entry.name).toBe(normalize('bundles/pages/index.js'))
+    expect(entry.files[0]).toBe('./pages/index.ts')
+  })
+
+  it('Should allow custom extension like .jsx to be turned into .js', () => {
+    const entry = createEntry('pages/index.jsx', {pageExtensions: ['jsx', 'js'].join('|')})
+    expect(entry.name).toBe(normalize('bundles/pages/index.js'))
+    expect(entry.files[0]).toBe('./pages/index.jsx')
+  })
+
+  it('Should turn pages/blog/index.js into pages/blog.js', () => {
+    const entry = createEntry('pages/blog/index.js')
+    expect(entry.name).toBe(normalize('bundles/pages/blog.js'))
+    expect(entry.files[0]).toBe('./pages/blog/index.js')
+  })
+})
+
+describe('getPageEntries', () => {
+  it('Should return paths', () => {
+    const pagePaths = ['pages/index.js']
+    const pageEntries = getPageEntries(pagePaths)
+    expect(pageEntries[normalize('bundles/pages/index.js')][0]).toBe('./pages/index.js')
+  })
+
+  it('Should include default _error', () => {
+    const pagePaths = ['pages/index.js']
+    const pageEntries = getPageEntries(pagePaths)
+    expect(pageEntries[normalize('bundles/pages/_error.js')][0]).toMatch(/dist[/\\]pages[/\\]_error\.js/)
+  })
+
+  it('Should not include default _error when _error.js is inside the pages directory', () => {
+    const pagePaths = ['pages/index.js', 'pages/_error.js']
+    const pageEntries = getPageEntries(pagePaths)
+    expect(pageEntries[normalize('bundles/pages/_error.js')][0]).toBe('./pages/_error.js')
+  })
+
+  it('Should include default _document when isServer is true', () => {
+    const pagePaths = ['pages/index.js']
+    const pageEntries = getPageEntries(pagePaths, {isServer: true})
+    expect(pageEntries[normalize('bundles/pages/_document.js')][0]).toMatch(/dist[/\\]pages[/\\]_document\.js/)
+  })
+
+  it('Should not include default _document when _document.js is inside the pages directory', () => {
+    const pagePaths = ['pages/index.js', 'pages/_document.js']
+    const pageEntries = getPageEntries(pagePaths, {isServer: true})
+    expect(pageEntries[normalize('bundles/pages/_document.js')][0]).toBe('./pages/_document.js')
+  })
+})