From 14c7b25fcc711e4f942f1ec18ae665864e92f75f Mon Sep 17 00:00:00 2001 From: Don Alvarez Date: Mon, 3 Sep 2018 07:43:19 -0700 Subject: [PATCH] Add with-sitemap-and-robots-express-server-typescript example (#5076) Pulling out a few core points from the readme... This example builds from /src into /dist, managing the different expectations of express.js (es5, commonjs) and next.js (es6) by using a pair of tsconfig.json files, both of which are run by `npm run build-ts` or any of the other npm targets. Hot module reloading is largely but not completely wired up (nodemon is watching /dist but tsc isn't set up to watch /src and transpile changes in /src to /dist automatically (that's mainly because I wasn't sure how to start both nodemon and a pair of tsc watchers and be confident all would get shut down when the user killed dev mode). The readme suggests running `npm run build-ts` manually in another window to push changes from /src into /dev and on into the browser. tslint is also wired up via `npm run tslint` --- .../.babelrc | 6 ++ .../.gitignore | 16 ++++ .../.npmignore | 10 +++ .../README.md | 74 +++++++++++++++++++ .../now.json | 12 +++ .../package.json | 34 +++++++++ .../src/components/RobotsLink.tsx | 9 +++ .../src/components/SitemapLink.tsx | 9 +++ .../src/pages/index.tsx | 22 ++++++ .../src/server/app.ts | 29 ++++++++ .../src/server/posts.ts | 16 ++++ .../src/server/sitemapAndRobots.ts | 50 +++++++++++++ .../src/static/robots.txt | 10 +++ .../tsconfig.express.json | 31 ++++++++ .../tsconfig.next.json | 32 ++++++++ .../tslint.json | 51 +++++++++++++ 16 files changed, 411 insertions(+) create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/.babelrc create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/.gitignore create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/.npmignore create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/README.md create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/now.json create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/package.json create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/components/RobotsLink.tsx create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/components/SitemapLink.tsx create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/pages/index.tsx create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/server/app.ts create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/server/posts.ts create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/server/sitemapAndRobots.ts create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/src/static/robots.txt create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/tsconfig.express.json create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/tsconfig.next.json create mode 100644 examples/with-sitemap-and-robots-express-server-typescript/tslint.json diff --git a/examples/with-sitemap-and-robots-express-server-typescript/.babelrc b/examples/with-sitemap-and-robots-express-server-typescript/.babelrc new file mode 100644 index 00000000..4a76d038 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/.babelrc @@ -0,0 +1,6 @@ +{ + "presets": [ + "next/babel", + "@zeit/next-typescript/babel" + ] +} diff --git a/examples/with-sitemap-and-robots-express-server-typescript/.gitignore b/examples/with-sitemap-and-robots-express-server-typescript/.gitignore new file mode 100644 index 00000000..ff7cfb44 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/.gitignore @@ -0,0 +1,16 @@ +*~ +*.swp +tmp/ +npm-debug.log +.DS_Store + + +.build/* +.next +.vscode/ +node_modules/ +.coverage +.env +.next + +yarn-error.log \ No newline at end of file diff --git a/examples/with-sitemap-and-robots-express-server-typescript/.npmignore b/examples/with-sitemap-and-robots-express-server-typescript/.npmignore new file mode 100644 index 00000000..6feb99f5 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/.npmignore @@ -0,0 +1,10 @@ +.build/ +.next/ +.vscode/ +node_modules/ + +README.md + +.env +.eslintrc.js +.gitignore \ No newline at end of file diff --git a/examples/with-sitemap-and-robots-express-server-typescript/README.md b/examples/with-sitemap-and-robots-express-server-typescript/README.md new file mode 100644 index 00000000..f87f33e1 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/README.md @@ -0,0 +1,74 @@ +[![Deploy to now](https://deploy.now.sh/static/button.svg)](https://deploy.now.sh/?repo=https://github.com/zeit/next.js/tree/master/examples/with-sitemap-and-robots-typescript) + +# Example with sitemap.xml and robots.txt using Express server and typescript + +## How to use + +### Using `create-next-app` + +Execute [`create-next-app`](https://github.com/segmentio/create-next-app) with [Yarn](https://yarnpkg.com/lang/en/docs/cli/create/) or [npx](https://github.com/zkat/npx#readme) to bootstrap the example: + +```bash +npx create-next-app --example with-sitemap-and-robots-express-server-typescript with-sitemap-and-robots-express-server-typescript-app +# or +yarn create next-app --example with-sitemap-and-robots-express-server-typescript with-sitemap-and-robots-express-server-typescript-app +``` + +### Download manually + +Download the example: + +```bash +curl https://codeload.github.com/zeit/next.js/tar.gz/canary | tar -xz --strip=2 next.js-canary/examples/with-sitemap-and-robots-expres-server-typescript +cd with-sitemap-and-robots-express-server-typescript +``` + +Install it and run: + +```bash +npm install +npm run dev +# or +yarn +yarn dev +``` + +Deploy it to the cloud with [now](https://zeit.co/now) ([download](https://zeit.co/download)) + +```bash +now +``` + +## The idea behind the example + +This example app shows you how to set up sitemap.xml and robots.txt files for proper indexing by search engine bots. + +The app is deployed at: https://sitemap-robots.now.sh. Open the page and click the links to see sitemap.xml and robots.txt. Here is a snapshot of these files, with sitemap.xml on the left and robots.txt on the right: +![sitemap-robots](https://user-images.githubusercontent.com/26158226/38786210-4d0c3f70-40db-11e8-8e44-b2c90cfd1b74.png) + +Notes: +- routes `/a` and `/b` are added to sitemap manually +- routes that start with `/posts` are added automatically to sitemap; in a real application, you will get post slugs from a database + +When you start this example locally: +- your app with run at https://localhost:8000 +- sitemap.xml will be located at http://localhost:8000/sitemap.xml +- robots.txt will be located at http://localhost:8000/robots.txt + +In case you want to deploy this example, replace the URL in the following locations with your own domain: +- `hostname` in `src/server/sitemapAndRobots.ts` +- `ROOT_URL` in `src/server/app.ts` +- `Sitemap` at the bottom of `src/server/robots.txt` +- `alias` in `now.json` + +Deploy with `now` or with `yarn now` if you specified `alias` in `now.json` + +## Typescript notes + +express.js and next.js require slighty different forms of javascript (es5 vs. es6, es6 modules vs commonjs modules, that sort of thing) so there are two tsconfig files that are used to generate the appropriate *.js and *.jsx files in dist from the *.ts and *.tsx files in src (`tsconfig.next.json` and `tsconfig.server.json`). Any files under /src/server are assumed to be for express, any other files under /src are assumed to be for next. To keep things simple the two typescript transpiles using these two config files are run directly from npm commands in package.json so all you need to run the example are simple commands like + +`npm run start` +or +`npm run dev` + +This example has most of the hot module reload plumbing setup for both express- and next-related source files when you use `npm run dev` but it's currently only watching the *.js and *.jsx files in /dist for changes. You will need to manually `npm run build-ts` in another window to transpile your modified typescript files in `src` into the watched javascript files in `dist` (it should be possible to have tsc watch the src folder and auto compile on save but that isn't wired up yet in this example). This example does also have tslint support though `npm run tslint`. \ No newline at end of file diff --git a/examples/with-sitemap-and-robots-express-server-typescript/now.json b/examples/with-sitemap-and-robots-express-server-typescript/now.json new file mode 100644 index 00000000..42b80ae7 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/now.json @@ -0,0 +1,12 @@ +{ + "env": { + "NODE_ENV": "production" + }, + "scale": { + "sfo1": { + "min": 1, + "max": 1 + } + }, + "alias": "https://sitemap-robots-typescript.now.sh" +} \ No newline at end of file diff --git a/examples/with-sitemap-and-robots-express-server-typescript/package.json b/examples/with-sitemap-and-robots-express-server-typescript/package.json new file mode 100644 index 00000000..af9a17c8 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/package.json @@ -0,0 +1,34 @@ +{ + "name": "with-sitemap-and-robots-express-server-typescript", + "version": "1.0.0", + "license": "MIT", + "scripts": { + "build-ts": "tsc -p tsconfig.next.json && tsc -p tsconfig.express.json", + "build-static": "npx copyfiles -u 1 \"src/static/**/*\" dist", + "build-next": "next build ./dist", + "build": "npm run build-ts && npm run build-static && npm run build-next", + "start-node": "cd ./dist && node server/app.js", + "start-nodemon": "cd ./dist && nodemon server/app.js --watch server", + "start": "npm run build-ts && npm run build-static && npm run start-node", + "dev": "npm run build-ts && npm run build-static && npm run start-nodemon", + "now": "npm run build && now ./dist && now alias", + "tslint": "tslint -c tslint.json -p tsconfig.next.json && tslint -c tslint.json -p tsconfig.server.json" + }, + "dependencies": { + "express": "^4.16.3", + "next": "latest", + "react": "^16.2.0", + "react-dom": "^16.2.0", + "sitemap": "^1.13.0" + }, + "devDependencies": { + "@types/express": "^4.16.0", + "@zeit/next-typescript": "1.1.0", + "nodemon": "^1.14.11", + "tslint": "^5.9.1", + "tslint-config-standard": "^7.0.0", + "tslint-loader": "^3.5.3", + "tslint-react": "^3.4.0", + "typescript": "^3.0.1" + } +} diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/components/RobotsLink.tsx b/examples/with-sitemap-and-robots-express-server-typescript/src/components/RobotsLink.tsx new file mode 100644 index 00000000..225a4b4d --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/components/RobotsLink.tsx @@ -0,0 +1,9 @@ +const RobotsLink = () => { + return ( + + Robots + + ); +}; + +export { RobotsLink }; diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/components/SitemapLink.tsx b/examples/with-sitemap-and-robots-express-server-typescript/src/components/SitemapLink.tsx new file mode 100644 index 00000000..2be4e042 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/components/SitemapLink.tsx @@ -0,0 +1,9 @@ +const SitemapLink = () => { + return ( + + Sitemap + + ); +}; + +export { SitemapLink }; diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/pages/index.tsx b/examples/with-sitemap-and-robots-express-server-typescript/src/pages/index.tsx new file mode 100644 index 00000000..b7cf0f97 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/pages/index.tsx @@ -0,0 +1,22 @@ +import Head from "next/head"; +import React from "react"; +import { RobotsLink } from "../components/RobotsLink"; +import { SitemapLink } from "../components/SitemapLink"; + +function Index() { + return ( +
+ + Index page + + +

+ +
+ +

+
+ ); +} + +export default Index; diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/server/app.ts b/examples/with-sitemap-and-robots-express-server-typescript/src/server/app.ts new file mode 100644 index 00000000..fa63031d --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/server/app.ts @@ -0,0 +1,29 @@ +import * as express from "express"; +import * as next from "next"; +import { sitemapAndRobots } from "./sitemapAndRobots"; + +const dev = process.env.NODE_ENV !== "production"; +const port = process.env.PORT || 8000; +const ROOT_URL = dev + ? `http://localhost:${port}` + : "https://sitemap-robots-typescript.now.sh"; + +const app = next({ dev }); +const handle = app.getRequestHandler(); + +// Nextjs's server prepared +app.prepare().then(() => { + const server = express(); + + sitemapAndRobots({ server }); + + server.get("*", (req, res) => { handle(req, res); }); + + // starting express server + server.listen(port, (err) => { + if (err) { + throw err; + } + console.log(`> Ready on ${ROOT_URL}`); + }); +}); diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/server/posts.ts b/examples/with-sitemap-and-robots-express-server-typescript/src/server/posts.ts new file mode 100644 index 00000000..1fc4469c --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/server/posts.ts @@ -0,0 +1,16 @@ +interface IPost { + name:string; + slug:string; +} + +const posts = () => { + const arrayOfPosts:IPost[] = []; + const n = 5; + + for (let i = 1; i < n + 1; i += 1) { + arrayOfPosts.push({ name: `Post ${i}`, slug: `post-${i}` }); + } + return arrayOfPosts; +}; + +export { IPost, posts }; diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/server/sitemapAndRobots.ts b/examples/with-sitemap-and-robots-express-server-typescript/src/server/sitemapAndRobots.ts new file mode 100644 index 00000000..562dc889 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/server/sitemapAndRobots.ts @@ -0,0 +1,50 @@ +import * as path from "path"; +import * as sm from "sitemap"; +import { posts } from "./posts"; + +const sitemap = sm.createSitemap({ + cacheTime: 600000, // 600 sec - cache purge period + hostname: "https://sitemap-robots-typescript.now.sh", +}); + +const sitemapAndRobots = ({ server }) => { + const Posts = posts(); + for (const post of Posts) { + sitemap.add({ + changefreq: "daily", + priority: 0.9, + url: `/posts/${post.slug}`, + }); + } + + sitemap.add({ + changefreq: "daily", + priority: 1, + url: "/a", + }); + + sitemap.add({ + changefreq: "daily", + priority: 1, + url: "/b", + }); + // Note {} in next line is a placeholder filling the spot where the req parameter + // would normally be listed (but isn't listed here since we aren't using it) + server.get("/sitemap.xml", ({}, res) => { + sitemap.toXML((err, xml) => { + if (err) { + res.status(500).end(); + return; + } + res.header("Content-Type", "application/xml"); + res.send(xml); + }); + }); + // Note {} in next line is a placeholder filling the spot where the req parameter + // would normally be listed (but isn't listed here since we aren't using it) + server.get("/robots.txt", ({}, res) => { + res.sendFile(path.join(__dirname, "../static", "robots.txt")); + }); +}; + +export { sitemapAndRobots }; diff --git a/examples/with-sitemap-and-robots-express-server-typescript/src/static/robots.txt b/examples/with-sitemap-and-robots-express-server-typescript/src/static/robots.txt new file mode 100644 index 00000000..749240f2 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/src/static/robots.txt @@ -0,0 +1,10 @@ +User-agent: * +Allow: /a +Allow: /b +Allow: /posts/* +Disallow: / +Disallow: /a/* +Disallow: /b/* +Disallow: /posts + +Sitemap: https://sitemap-robots.now.sh/sitemap.xml \ No newline at end of file diff --git a/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.express.json b/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.express.json new file mode 100644 index 00000000..da376a49 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.express.json @@ -0,0 +1,31 @@ +{ + "compileOnSave": true, + "compilerOptions": { + "pretty": true, + "experimentalDecorators": true, + "target": "es5", + "module": "commonjs", + "jsx": "preserve", + "allowJs": true, + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, + "noImplicitThis": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "strictNullChecks": true, + "removeComments": false, + "preserveConstEnums": true, + "sourceMap": true, + "skipLibCheck": true, + "typeRoots": [ + "./node_modules/@types" + ], + "outDir": "./dist/server", + "lib": [ + "dom", + "es2015", + "es2016" + ] + }, + "include": ["./src/server/**/*"] +} diff --git a/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.next.json b/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.next.json new file mode 100644 index 00000000..9e56e62a --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/tsconfig.next.json @@ -0,0 +1,32 @@ +{ + "compileOnSave": true, + "compilerOptions": { + "pretty": true, + "experimentalDecorators": true, + "target": "esnext", + "module": "es6", + "jsx": "preserve", + "allowJs": true, + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, + "noImplicitThis": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "strictNullChecks": true, + "removeComments": false, + "preserveConstEnums": true, + "sourceMap": true, + "skipLibCheck": true, + "typeRoots": [ + "./node_modules/@types" + ], + "outDir": "./dist", + "lib": [ + "dom", + "es2015", + "es2016" + ] + }, + "include": ["./src/**/*"], + "exclude": ["./src/server/**/*"] +} diff --git a/examples/with-sitemap-and-robots-express-server-typescript/tslint.json b/examples/with-sitemap-and-robots-express-server-typescript/tslint.json new file mode 100644 index 00000000..954d6375 --- /dev/null +++ b/examples/with-sitemap-and-robots-express-server-typescript/tslint.json @@ -0,0 +1,51 @@ +{ + "extends": [ + "tslint-config-standard", + "tslint:latest", + "tslint-react" + ], + "rules": { + "indent": [true, "spaces"], + "jsx-no-lambda": false, + "jsx-no-multiline-js": false, + "max-line-length": false, + "no-console": false, + "no-object-literal-type-assertion": false, + "no-submodule-imports": [ + true, + "next" + ], + "no-unused-variable": false, + "space-before-function-paren": false, + "ter-indent": [true, 2], + "typedef-whitespace": [ + true, + { + "call-signature": "nospace", + "index-signature": "nospace", + "parameter": "nospace", + "property-declaration": "nospace", + "variable-declaration": "nospace" + }, + { + "call-signature": "nospace", + "index-signature": "nospace", + "parameter": "nospace", + "property-declaration": "nospace", + "variable-declaration": "nospace" + } + ], + "whitespace": [ + true, + "check-branch", + "check-decl", + "check-operator", + "check-module", + "check-preblock", + "check-rest-spread", + "check-separator", + "check-typecast", + "check-type-operator" + ] + } +} \ No newline at end of file