fix: generate better slugs for non latin langs (close vuejs#45)

Author: yyx990803

docs/config/README.md

@@ -113,19 +113,26 @@ Provide config options to the used theme. The options will vary depending on the
 
 ## Markdown
 
+### markdown.slugify
+
+- Type: `Function`
+- Default: [source](https://github.com/vuejs/vuepress/blob/master/lib/markdown/slugify.js)
+
+Function for transforming header texts into slugs. This affects the ids/links generated for header anchors, table of contents and sidebar links.
+
 ### markdown.anchor
 
 - Type: `Object`
 - Default: `{ permalink: true, permalinkBefore: true, permalinkSymbol: '#' }`
 
-Options for [markdown-it-anchor](https://github.com/valeriangalliat/markdown-it-anchor).
+Options for [markdown-it-anchor](https://github.com/valeriangalliat/markdown-it-anchor). (Note: prefer `markdown.slugify` if you want to customize header ids.)
 
 ### markdown.toc
 
 - Type: `Object`
 - Default: `{ includeLevel: [2, 3] }`
 
-Options for [markdown-it-table-of-contents](https://github.com/Oktavilla/markdown-it-table-of-contents).
+Options for [markdown-it-table-of-contents](https://github.com/Oktavilla/markdown-it-table-of-contents). (Note: prefer `markdown.slugify` if you want to customize header ids.)
 
 ### markdown.config
 

lib/markdown/index.js

@@ -7,8 +7,12 @@ const containers = require('./containers')
 const emoji = require('markdown-it-emoji')
 const anchor = require('markdown-it-anchor')
 const toc = require('markdown-it-table-of-contents')
+const _slugify = require('./slugify')
+
+module.exports = ({ markdown = {}} = {}) => {
+ // allow user config slugify
+ const slugify = markdown.slugify || _slugify
 
-module.exports = ({ markdown = {}}) => {
  const md = require('markdown-it')({
  html: true,
  typographer: true,
@@ -24,11 +28,13 @@ module.exports = ({ markdown = {}}) => {
  // 3rd party plugins
  .use(emoji)
  .use(anchor, Object.assign({
+ slugify,
  permalink: true,
  permalinkBefore: true,
  permalinkSymbol: '#'
  }, markdown.anchor))
  .use(toc, Object.assign({
+ slugify,
  includeLevel: [2, 3]
  }, markdown.toc))
 
@@ -48,5 +54,8 @@ module.exports = ({ markdown = {}}) => {
  }
  }
 
+ // expose slugify
+ md.slugify = slugify
+
  return md
 }

lib/markdown/slugify.js

@@ -0,0 +1,20 @@
+// string.js slugify drops non ascii chars so we have to
+// use a custom implementation here
+const removeDiacritics = require('diacritics').remove
+// eslint-disable-next-line no-control-regex
+const rControl = /[\u0000-\u001f]/g
+const rSpecial = /[\s~`!@#$%^&*()\-_+=[\]{}|\\;:"'<>,.?/]+/g
+
+module.exports = function slugify (str) {
+ return removeDiacritics(str)
+ // Remove control characters
+ .replace(rControl, '')
+ // Replace special characters
+ .replace(rSpecial, '-')
+ // Remove continous separators
+ .replace(/\-{2,}/g, '-')
+ // Remove prefixing and trailing separtors
+ .replace(/^\-+|\-+$/g, '')
+ // lowercase
+ .toLowerCase()
+}

lib/util.js renamed to lib/util/index.js

@@ -45,6 +45,7 @@ exports.parseFrontmatter = content => {
 
 const LRU = require('lru-cache')
 const cache = LRU({ max: 1000 })
+const md = require('../markdown')()
 
 exports.extractHeaders = (content, include = []) => {
  const key = content + include.join(',')
@@ -53,8 +54,6 @@ exports.extractHeaders = (content, include = []) => {
  return hit
  }
 
- const md = require('./markdown')({})
- const S = require('string')
  const tokens = md.parse(content, {})
 
  const res = []
@@ -64,7 +63,7 @@ exports.extractHeaders = (content, include = []) => {
  res.push({
  level: parseInt(t.tag.slice(1), 10),
  title,
- slug: S(title).slugify().s
+ slug: md.slugify(title)
  })
  }
  })

package.json

@@ -50,6 +50,7 @@
  "connect-history-api-fallback": "^1.5.0",
  "copy-webpack-plugin": "^4.5.1",
  "css-loader": "^0.28.11",
+ "diacritics": "^1.3.0",
  "es6-promise": "^4.2.4",
  "escape-html": "^1.0.3",
  "file-loader": "^1.1.11",

yarn.lock

@@ -1381,6 +1381,10 @@ detect-libc@^1.0.2:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b"
 
+diacritics@^1.3.0:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/diacritics/-/diacritics-1.3.0.tgz#3efa87323ebb863e6696cebb0082d48ff3d6f7a1"
+
 diff@^3.2.0:
  version "3.5.0"
  resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"