Commit 046b7882 authored by julien's avatar julien

add hyphen lib

parent 31ebe13e
/* regular */
@font-face {
font-family: 'Minion Pro';
src: url('../fonts/MinionPro-Regular.otf') format('truetype');
font-weight: 400;
font-style: normal;
}
/* italic */
@font-face {
font-family: 'Minion Pro';
src: url('../fonts/MinionPro-It.otf') format('truetype');
font-weight: 400;
font-style: italic;
}
/* Bold */
@font-face {
font-family: 'Minion Pro';
src: url('../fonts/MinionPro-Bold.otf') format('truetype');
font-weight: 600;
font-style: normal;
}
/* bold italic */
@font-face {
font-family: 'Minion Pro';
src: url('../fonts/MinionPro-BoldIt.otf') format('truetype');
font-weight: 600;
font-style: italic;
}
\ No newline at end of file
language: node_js
cache: bundler
node_js:
- "10"
notifications:
email: false
\ No newline at end of file
# Version History
## Version 2.3.0 (Juli 26, 2018)
### Hyphenopoly_Loader.js and Hyphenopoly.js:
* Don't use template strings [#28](https://github.com/mnater/Hyphenopoly/issues/28)
* run feature test for wasm support only if necessary
### hyphenopoly.module.js:
* define node >=8.3.0 as requirement (for util.TextDecoder)
* small refactorings
## Version 2.2.0 (June 26, 2018)
* provide example.js for RunKit
* use tap instead of mocha
* [6f9e539](https://github.com/mnater/Hyphenopoly/commit/6f9e539a5dab2d1eff5bdeb0c7857c6fda9eb41e)
* bugfix: [#24](https://github.com/mnater/Hyphenopoly/issues/24): [aeefe6e](https://github.com/mnater/Hyphenopoly/commit/aeefe6e3a59e8356abc99ca490acabf6c3374d7b)
## Version 2.1.0 (Mai 27, 2018)
* Configure Travis-CI
* bugfixes
## Version 2.0.0 (Mai 27, 2018)
* Provide node module (https://github.com/mnater/Hyphenopoly/wiki/Node-Module)
* default file locations better reflect usual installations [#19](https://github.com/mnater/Hyphenopoly/issues/19)
* Add ability to store results of feature tests (optional) [#22](https://github.com/mnater/Hyphenopoly/issues/22)
* better error handling (f4bbaa7759eed24208e5cd7c744f1131262abb20, 1c7b0b67666b507d6f6b02eea38460562a5835e4)
* correct implementation of e.preventDefault (df988788db6fb7120fc0c8a1cff1c91aac5a3998)
* fix string normalization (a3229f730f79ccdd3054cbac257b2345f5c8e11a)
* Better tooling: minify, eslint, testing (mocha), compiling (https://github.com/mnater/Hyphenopoly/wiki/Usage-of-devDependencies)
## Version 1.0.1 (Mai 13, 2018)
Prevent browsers to force layout on feature test in some cases.
## Version 1.0.0 (Mai 12, 2018)
First release
This diff is collapsed.
This diff is collapsed.
The MIT License (MIT)
Copyright (c) 2018 Mathias Nater
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
# Hyphenopoly.js
[![Build Status](https://travis-ci.org/mnater/Hyphenopoly.svg?branch=master)](https://travis-ci.org/mnater/Hyphenopoly) [![Coverage Status](https://coveralls.io/repos/github/mnater/Hyphenopoly/badge.svg?branch=master)](https://coveralls.io/github/mnater/Hyphenopoly?branch=master) [![dependencies Status](https://david-dm.org/mnater/Hyphenopoly/status.svg)](https://david-dm.org/mnater/Hyphenopoly) [![devDependencies Status](https://david-dm.org/mnater/Hyphenopoly/dev-status.svg)](https://david-dm.org/mnater/Hyphenopoly?type=dev) [![npms score](https://badges.npms.io/hyphenopoly.svg)](https://npms.io/search?q=hyphenopoly)
Hyphenopoly.js is a __JavaScript-polyfill for hyphenation in HTML__: it hyphenates text if the user agent does not support CSS-hyphenation at all or not for the required languages and it is a __Node.js-module__.
The package consists of the following parts:
- _Hyphenopoly_Loader.js_ (~13KB unpacked, ~2.5KB minified and compressed): feature-checks the client and loads other resources if necessary.
- _Hyphenopoly.js_ (~30KB unpacked, ~4KB minified and compressed): does the whole DOM-foo and wraps (w)asm.
- _hyphenEngine.wasm_ (~1KB uncompressed): wasm code for creating pattern trie and finding hyphenation points.
- _hyphenEngine.asm.js_ (~7KB uncompressed, ~1KB minified and compressed): fallback for clients that don't support wasm.
- _pattern.hpb_ (sizes differ! e.g. en-us.hpb: ~29KB): space saving binary format of the hyphenation patterns (including their license).
- _hyphenopoly.module.js_: the node module
# Usage (Browser)
Place all code for Hyphenopoly at the top of the header (immediately after the `<title>` tag) to ensure resources are loaded as early as possible.
You'll have to insert two script blocks. In the first block provide the initial configurations for Hyphenopoly_Loader as inline script. In the second block load Hyphenopoly_Loader.js as external script.
Also, don't forget to enable CSS hyphenation.
[Example](http://mnater.github.io/Hyphenopoly/example1.html):
```html
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Example 1</title>
<script>
var Hyphenopoly = {
require: {
"la": "honorificabilitudinitas",
"de": "Silbentrennungsalgorithmus",
"en-us": "Supercalifragilisticexpialidocious"
},
setup: {
classnames: {
"container": {}
}
}
};
</script>
<script src="./Hyphenopoly_Loader.js"></script>
<style type="text/css">
body {
width:60%;
margin-left:20%;
}
p {
text-align: justify;
margin: 0 2em 0 0;
}
.container {
display: flex;
hyphens: auto;
-ms-hyphens: auto;
-moz-hyphens: auto;
-webkit-hyphens: auto;
}
</style>
</head>
<body>
<h1>Example 1</h1>
<div class="container">
<p lang="la">Qua de causa Helvetii quoque reliquos Gallos virtute praecedunt, quod fere cotidianis proeliis cum Germanis contendunt, cum aut suis finibus eos prohibent aut ipsi in eorum finibus bellum gerunt.</p>
<p lang="en-us">For which reason the Helvetii also surpass the rest of the Gauls in valor, as they contend with the Germans in almost daily battles, when they either repel them from their own territories, or themselves wage war on their frontiers.</p>
<p lang="de">Aus diesem Grund übertreffen auch die Helvetier die übrigen Gallier an Tapferkeit, weil sie sich in fast täglichen Gefechten mit den Germanen messen, wobei sie diese entweder von ihrem Gebiet fernhalten oder selbst in deren Gebiet kämpfen.</p>
</body>
</html>
```
Let's go through this example step by step:
## UTF-8
Make sure your page is encoded as utf-8.
## First script block – configurations
Hyphenopoly_Loader.js needs some information to run. This information is provided in a globally accessible Object called `Hyphenopoly`. Hyphenopoly_Loader.js and (if necessary) Hyphenopoly.js will add other methods and properties only to this object – there will be no other global variables/functions beyond this object.
### require
The `Hyphenopoly` object must have exactly one property called `require` which itself is an object containing at least one nameValuePair where the name is a language code string (Some patterns are region-specific. See the patterns directory for supported languages. E.g. just using `en` won't work, use either `en-us`or `en-gb`) and the value is a long word string in that language (preferably more than 12 characters long).
Hyphenator_Loader.js will feature test the client (aka browser, aka user agent) for CSS-hyphens support for the given languages with the given words respectively. In the example above it will test if the client supports CSS-hyphenation for latin. If your page contains more than just one language just add more lines.
If you want to force the usage of Hyphenopoly.js for a language (e.g. for testing purposes) write `"FORCEHYPHENOPOLY"` instead of the long word.
## Second script block – load and run Hyphenopoly_Loader.js
Hyphenopoly_Loader.js tests if the browser supports CSS hyphenation for the language(s) given in `Hyphenopoly.require`. If one of the given languages isn't supported it automatically hides the documents contents and loads Hyphenopoly.js and the necessary patterns. Hyphenopoly.js – once loaded – will hyphenate the elements according to the settings and unhide the document when it's done. If something goes wrong and Hyphenopoly.js is unable to unhide the document Hyphenopoly_Loader.js has a timeout that kicks in after some time (defaults to 1000ms) and unhides the document and writes a message to the console.
If the browser supports all required languages the script deletes the `Hyphenopoly`-object and terminates without further ado.
## enable CSS-hyphenation
Hyphenopoly by default hyphenates elements (and their children) with the classname `.hyphenate`. Don't forget to enable CSS-hyphenation for the classes eventually handled by Hyphenopoly.
# Usage (node)
[![Try hyphenopoly on RunKit](https://badge.runkitcdn.com/hyphenopoly.svg)](https://npm.runkit.com/hyphenopoly)
Install:
````
npm i hyphenopoly
````
````javascript
"use strict";
const hyphenopoly = require("hyphenopoly");
const hyphenator = hyphenopoly.config({
"require": ["de", "en-us"],
"hyphen": "•",
"exceptions": {
"en-us": "en-han-ces"
}
});
async function hyphenate_en(text) {
const hyphenateText = await hyphenator.get("en-us");
console.log(hyphenateText(text));
}
async function hyphenate_de(text) {
const hyphenateText = await hyphenator.get("de");
console.log(hyphenateText(text));
}
hyphenate_en("hyphenation enhances justification.");
hyphenate_de("Silbentrennung verbessert den Blocksatz.");
````
# Automatic hyphenation
The algorithm used for hyphenation was developed by Franklin M. Liang for TeX. It works more or less like this:
1. Load a set of precomputed language specific patterns. The patterns are stored in a structure called a trie, which is very efficient for this task.
2. Collect all patterns that are a substring of the word to be hyphenated.
3. Combine the numerical values between characters: higher values overwrite lower values.
4. Odd values are hyphenation points (except if the hyphenation point is left from leftmin and right from rightmin), replace them with a soft hyphen and drop the other values.
5. Repeat 2. - 4. for all words longer than minWordLength
Example:
````
Hyphenation
h y p h e n a t i o n
h y3p h
h e2n
h e n a4
h e n5a t
1n a
n2a t
1t i o
2i o
o2n
h0y3p0h0e2n5a4t2i0o2n
Hy-phen-ation
````
The patterns are precomputed and available for many languages on CTAN. Hyphenopoly.js uses a proprietary binary format (including pattern licence, metadata and the patterns). Patterns are computed from a large list of hyphenated words by a program called patgen. They aim to find some hyphenation points – not all – because it's better to miss a hyphenation point then to have some false hyphenation points. Most patterns are really good but none is error free.
These pattern vary in size. This is mostly due to the different linguistic characteristics of the languages.
# Hyphenopoly.js vs. Hyphenator.js
Hyphenator.js (https://github.com/mnater/Hyphenator) started 2007 and had evolved ever since.
But web browsers have evolved much more!
Almost all of them support native hyphenation (https://developer.mozilla.org/en-US/docs/Web/CSS/hyphens) for a specific set of languages (https://developer.mozilla.org/en-US/docs/Web/CSS/hyphens#Browser_compatibility). So it was time for something new!
Hyphenopoly.js is based on Hyphenator.js (they share some code) but - in favor of simplicity and speed – lacks many features of Hyphenator.js. Most of these features aren't needed in modern webdesign anymore:
- dropped support for usage as bookmarklet
- dropped support for frames
- dropped support for ancient browsers
- dropped caching of patterns in browser storage
- dropped breaking of non-textual content (urls, code etc.)
- and some more…
If you need one of those features use Hyphenator.js – or give some feedback and proof that the feature is really useful and should be implemented in Hyphenopoly.js
On the other hand Hyphenopoly has a much finer-grained configuration system that allows you to make settings based on CSS-classes.
And last but not least it is faster than Hyphenator.js
const hyphenopoly = require("hyphenopoly");
const hyphenator = hyphenopoly.config({
"require": ["de", "en-us"],
"hyphen": "•",
"exceptions": {
"en-us": "en-han-ces"
}
});
async function hyphenate_en(text) {
const hyphenateText = await hyphenator.get("en-us");
console.log(hyphenateText(text));
}
async function hyphenate_de(text) {
const hyphenateText = await hyphenator.get("de");
console.log(hyphenateText(text));
}
hyphenate_en("hyphenation enhances justification.");
hyphenate_de("Silbentrennung verbessert den Blocksatz.");
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Example 1</title>
<script>
var Hyphenopoly = {
require: {
"la": "honorificabilitudinitas",
"de": "Silbentrennungsalgorithmus",
"en-us": "Supercalifragilisticexpialidocious"
},
paths: {
patterndir: "./patterns/",
maindir: "./"
},
setup: {
classnames: {
"container": {
}
}
}
};
</script>
<script src="./Hyphenopoly_Loader.js"></script>
<style type="text/css">
body {
width:60%;
margin-left:20%;
}
p {
text-align: justify;
margin: 0 2em 0 0;
}
.container {
display: flex;
hyphens: auto;
-ms-hyphens: auto;
-moz-hyphens: auto;
-webkit-hyphens: auto;
}
</style>
</head>
<body>
<h1>Example 1</h1>
<div class="container">
<p lang="la">Qua de causa Helvetii quoque reliquos Gallos virtute praecedunt, quod fere cotidianis proeliis cum Germanis contendunt, cum aut suis finibus eos prohibent aut ipsi in eorum finibus bellum gerunt.</p>
<p lang="en-us">For which reason the Helvetii also surpass the rest of the Gauls in valor, as they contend with the Germans in almost daily battles, when they either repel them from their own territories, or themselves wage war on their frontiers.</p>
<p lang="de">Aus diesem Grund übertreffen auch die Helvetier die übrigen Gallier an Tapferkeit, weil sie sich in fast täglichen Gefechten mit den Germanen messen, wobei sie diese entweder von ihrem Gebiet fernhalten oder selbst in deren Gebiet kämpfen.</p>
</body>
</html>
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
{
"name": "hyphenopoly",
"version": "2.4.0",
"description": "Hyphenation for node and Polyfill for client-side hyphenation.",
"keywords": [
"hyphenation",
"html",
"polyfill",
"hyphens",
"hyphen",
"soft",
"hyphenate",
"JavaScript"
],
"homepage": "https://github.com/mnater/Hyphenopoly",
"bugs": "https://github.com/mnater/Hyphenopoly/issues",
"license": "MIT",
"author": {
"name": "Mathias Nater",
"email": "mathiasnater@gmail.com"
},
"files": [
"hyphenEngine.asm.js",
"hyphenEngine.wasm",
"Hyphenopoly_Loader.js",
"Hyphenopoly.js",
"patterns/",
"min/",
"example.js",
"example.html"
],
"browser": "Hyphenopoly_Loader.js",
"main": "hyphenopoly.module.js",
"engines": {
"node": ">=8.3.0"
},
"repository": {
"type": "git",
"url": "https://github.com/mnater/Hyphenopoly"
},
"dependencies": {},
"devDependencies": {
"eslint": "^5.5.0",
"tap": "^12.0.1",
"terser": "^3.8.2"
},
"eslintConfig": {
"parserOptions": {
"ecmaVersion": 8
},
"env": {
"browser": true,
"es6": true
},
"globals": {
"Hyphenopoly": true
},
"extends": "eslint:all",
"rules": {
"require-unicode-regexp": 0,
"require-jsdoc": [
"error",
{
"require": {
"FunctionDeclaration": true,
"MethodDefinition": true,
"ClassDeclaration": true,
"ArrowFunctionExpression": true,
"FunctionExpression": true
}
}
],
"no-template-curly-in-string": 2,
"prefer-template": 0,
"wrap-iife": 1,
"max-lines": 0,
"max-lines-per-function": 0,
"max-len": [
1,
{
"ignoreStrings": true,
"ignoreTemplateLiterals": true
}
],
"max-statements": 0,
"space-before-function-paren": [
1,
{
"anonymous": "always",
"named": "never",
"asyncArrow": "never"
}
],
"padded-blocks": [
1,
"never"
],
"id-length": [
1,
{
"min": 1
}
],
"one-var": 0,
"func-names": [
1,
"as-needed"
],
"func-style": [
1,
"declaration"
],
"object-shorthand": 0,
"prefer-arrow-callback": 0,
"no-param-reassign": 0,
"no-extra-parens": 0,
"array-element-newline": [
1,
{
"multiline": true
}
],
"no-magic-numbers": 0,
"function-paren-newline": [
1,
"consistent"
],
"no-ternary": 0,
"no-nested-ternary": 0,
"prefer-destructuring": 0,
"max-params": [
1,
5
],
"lines-around-comment": [
1,
{
"allowBlockStart": true
}
],
"sort-keys": [
1,
"asc",
{
"caseSensitive": false
}
]
}
},
"scripts": {
"preinstall3rdparty": "rm -fR third-party",
"install3rdparty": "git clone https://github.com/WebAssembly/binaryen.git third-party/binaryen && git clone --recursive https://github.com/WebAssembly/wabt.git third-party/wabt",
"postinstall3rdparty": "npm run compile-binaryen && npm run compile-wabt",
"compile-binaryen": "cd ./third-party/binaryen/ && cmake . && make",
"compile-wabt": "cd ./third-party/wabt/ && make",
"compilewasm": "sh ./tools/compileWASM.sh hyphenEngine",
"test": "tap test/*.js",
"testsuite": "open http://127.0.0.1/~mnater/Hyphenopoly/testsuite/ && open http://127.0.0.1/~mnater/Hyphenopoly/min/testsuite/",
"lint": "eslint Hyphenopoly_Loader.js Hyphenopoly.js hyphenopoly.module.js test/*.js",
"prepare": "npm run minify",
"minify": "sh ./tools/minify.sh"
},
"runkitExampleFilename": "example.js"
}
/* eslint-env node */
/* eslint global-require: 0, func-names: 0, no-shadow: 0 */
"use strict";
const t = require("tap");
let H9Y = null;
const H9YKey = require.resolve("../hyphenopoly.module");
t.beforeEach(function setup(done) {
H9Y = require("../hyphenopoly.module");
done();
});
t.afterEach(function tearDown(done) {
H9Y = null;
delete require.cache[H9YKey];
done();
});
t.test("run config with one language", async function (t) {
const deHyphenator = await H9Y.config({"require": ["de"]});
t.test("return a function", function (t) {
t.equal(typeof deHyphenator, "function", typeof deHyphenator);
t.end();
});
t.test("hyphenate one word", function (t) {
t.equal(deHyphenator("Silbentrennung"), "Sil\u00ADben\u00ADtren\u00ADnung", deHyphenator("Silbentrennung"));
t.end();
});
t.test("hyphenate two words", function (t) {
t.equal(deHyphenator("Silbentrennung Algorithmus"), "Sil\u00ADben\u00ADtren\u00ADnung Al\u00ADgo\u00ADrith\u00ADmus", deHyphenator("Silbentrennung Algorithmus"));
t.end();
});
t.end();
});
t.test("run config with two languages", async function (t) {
const hyphenators = await H9Y.config({"require": ["de", "en-us"]});
t.test("return a Map", function (t) {
t.type(hyphenators, Map);
t.end();
});
t.test("get the hyphenator function promise for a language", function (t) {
const deHyphenator = hyphenators.get("de");
t.type(deHyphenator, Promise);
t.end();
});
t.test("get the hyphenator function for a language", async function (t) {
const deHyphenator = await hyphenators.get("de");
t.type(deHyphenator, Function);
t.end();
});
t.test("hyphenate one word of the first language", async function (t) {
const deHyphenator = await hyphenators.get("de");
t.equal(deHyphenator("Silbentrennung"), "Sil\u00ADben\u00ADtren\u00ADnung", deHyphenator("Silbentrennung"));
t.end();
});
t.test("hyphenate two words of the first language", async function (t) {
const deHyphenator = await hyphenators.get("de");
t.equal(deHyphenator("Silbentrennung Algorithmus"), "Sil\u00ADben\u00ADtren\u00ADnung Al\u00ADgo\u00ADrith\u00ADmus", deHyphenator("Silbentrennung Algorithmus"));
t.end();
});
t.test("hyphenate one word of the second language", async function (t) {
const deHyphenator = await hyphenators.get("de");
t.equal(deHyphenator("Silbentrennung"), "Sil\u00ADben\u00ADtren\u00ADnung", deHyphenator("Silbentrennung"));
t.end();
});
t.test("hyphenate two words of the second language", async function (t) {
const deHyphenator = await hyphenators.get("de");