Merge pull request #348 from jamesramsay/get-stream

4.0-beta.1

Author: James

.eslintignore

@@ -1,2 +1,3 @@
 test/fixtures/index.js
 test/integration/_mock.js
+test/benchmark.js

.gitignore

@@ -1,7 +1,7 @@
 node_modules
 lib
+.bats
 .coveralls.yml
 coverage
 .nyc_output
 *.log
-test/modules*

README.md

@@ -27,7 +27,7 @@ Hercule is a command-line tool and library for transcluding markdown, [API Bluep
 Install Hercule using [npm](http://npmjs.org):
 
 ```bash
-npm install -g hercule
+$ npm install -g hercule
 ```
 
 ## Usage
@@ -41,7 +41,7 @@ hercule src/blueprint.md -o output.md
 Hercule supports processing input from stdin, and writing to stdout:
 
 ```
-cat src/blueprint.md | hercule | less
+cat src/blueprint.md | hercule - | less
 ```
 
 Or you can use Hercule as a library:
@@ -180,10 +180,7 @@ Each line of `snippet.c` will be indented with the whitespace preceding it.
 - [`transcludeString`](#transcludeString)
 - [`transcludeFile`](#transcludeFile)
 - [Resolvers](#resolvers)
-  - [`defaultResolvers`](#defaultResolvers)
-  - [`resolveHttpUrl`](#resolveHttpUrl)
-  - [`resolveLocalUrl`](#resolveLocalUrl)
-  - [`resolveString`](#resolveString)
+- [Custom Transclusion Syntax](#customSyntax)
 
 ---------------------------------------
 
@@ -197,7 +194,8 @@ __Arguments__
 
 1. `source` (_String_): A string used for resolving relative links and generating sourcemap.
 2. `options` (_Object_): An object of options to be applied when processing input.
-  - `resolvers` - An array of functions which are applied to resolve the URLs to content.
+  - `resolvers` (_Array[Function]_): An array of functions which are applied to resolve the URLs to content.
+  - `transclusionSyntax` (_String_): Choose transclusion link syntax. Supports 'hercule', 'aglio', 'marked', 'multimarkdown'.
 
 __Customer Emitters__
 
@@ -233,6 +231,7 @@ __Arguments__
 2. `options` (_Object_): An object of options to be applied when processing input.
   - `source` (_String_): source file required for resolving relative links and generating sourcemap.
   - `resolvers` (_Array[Function]_): An array of functions which are applied to resolve the URLs to content.
+  - `transclusionSyntax` (_String_): Choose transclusion link syntax. Supports 'hercule', 'aglio', 'marked', 'multimarkdown'.
 3. `callback(err, [output], [sourcemap])` (_Function_): A function that will be called after the input `str` has been processed.
   - `err` (_Error_): An error object.
   - `output` (_String_): A string containing processed input.
@@ -263,6 +262,7 @@ __Arguments__
 1. `source` (_String_): A path to a file to process.
 2. `options` (_Object_): An object of options to be applied when processing input.
   - `resolvers` (_Array[Function]_): An array of functions which are applied to resolve the URLs to content.
+  - `transclusionSyntax` (_String_): Choose transclusion link syntax. Supports 'hercule', 'aglio', 'marked', 'multimarkdown'.
 3. `callback(err, [output], [sourcemap])` (_Function_): A function that will be called after the `source` file has been processed.
   - `err` (_Error_): An error object.
   - `output` (_String_): A string containing processed input.
@@ -297,32 +297,40 @@ __Arguments__
 
 __Returns__
 
+- (_null_): Returns null if the url cannot be resolved.
 - (_Object_)
   - `content` (_Stream | String_): The content to be transcluded. Streams are processed for further transclusion links. Strings are assumed fully processed.
   - `url` (_String_): The absolute url of the input, allowing circular reference detection and nested transclusion.
 
-<a name="defaultResolvers" />
-
-#### defaultResolvers
-
-Ordered array of functions applied until a result is returned.
-
-TODO: write the rest of this
+__Examples__
 
-<a name="resolveHttpUrl" />
+```
+import { trancludeFile, resolveHttpUrl, resolveLocalUrl, resolveString } from 'hercule';
 
-#### resolveHttpUrl
+function myResolver(url, source) {
+  // Add your implementation here
+  // Return null to try next resolver
+  return null;
+}
 
-TODO: write this
+// Resolvers are tried in order
+const resolvers = [myResolver, resolveHttpUrl, resolveLocalUrl, resolveString];
 
-<a name="resolveLocalUrl" />
+trancludeFile('foo.md', { resolvers }, (err, output) => {
+  // Handle exceptions like dead links
+  if (err) console.log(err)
+  console.log(output);
+});
+```
 
-#### resolveLocalUrl
+---------------------------------------
 
-TODO: write this
+<a name="customSyntax" />
 
-<a name="resolveString" />
+### Custom Transclusion Syntax
 
-#### resolveString
+Hercule also has basic support for alternative transclusion link syntax, including:
 
-TODO: write this
+- [aglio](https://github.com/danielgtaylor/aglio/): `<!-- include(foo.md) -->`
+- [Marked](http://marked2app.com/help/Multi-File_Documents.html): `<<[sections/section1.md]`
+- [MultiMarkdown](http://fletcher.github.io/MultiMarkdown-5/transclusion.html): `{{bar.md}}`

package.json

@@ -1,6 +1,6 @@
 {
   "name": "hercule",
-  "version": "4.0.0",
+  "version": "4.0.0-beta.1",
   "description": "Markdown, API Blueprint and string transclusion",
   "main": "./lib/hercule",
   "scripts": {
@@ -17,7 +17,8 @@
     "check-commit": "./scripts/commit-msg",
     "release:major": "npm version major && git push --follow-tags",
     "release:minor": "npm version minor && git push --follow-tags",
-    "release:patch": "npm version patch && git push --follow-tags"
+    "release:patch": "npm version patch && git push --follow-tags",
+    "benchmark": "./test/benchmark.js"
   },
   "engines": {
     "node": ">=4.0.0"
@@ -66,12 +67,13 @@
   "dependencies": {
     "async": "^2.1.4",
     "clone-regexp": "^1.0.0",
-    "dashdash": "^1.10.1",
     "duplexer3": "^0.1.4",
+    "get-stream": "^3.0.0",
     "got": "^6.5.0",
     "isstream": "^0.1.2",
     "left-split": "^1.0.0",
     "lodash": "^4.0.0",
+    "meow": "^3.7.0",
     "source-map": "^0.5.3",
     "through2": "^2.0.0",
     "through2-get": "^0.0.2"
@@ -82,6 +84,7 @@
     "babel-core": "^6.4.0",
     "babel-preset-es2015": "^6.3.13",
     "babel-register": "^6.9.0",
+    "benchmark": "^2.1.3",
     "codecov.io": "^0.1.6",
     "eslint": "^3.12.0",
     "eslint-config-airbnb-base": "^11.0.0",

src/hercule.js

@@ -2,39 +2,31 @@ import fs from 'fs';
 import _ from 'lodash';
 import duplexer from 'duplexer3';
 import get from 'through2-get';
+import getStream from 'get-stream';
 
 import Transclude from './transclude';
 import Indent from './indent';
 import Trim from './trim';
 import Sourcemap from './sourcemap';
 
+export { resolveHttpUrl, resolveLocalUrl, resolveString } from './resolver';
+
 export function TranscludeStream(source = 'input', options) {
   const outputFile = _.get(options, 'outputFile');
   let sourceMap;
 
-  const transclude = new Transclude(source);
+  const transclude = new Transclude(source, options);
   const indenter = new Indent();
   const trim = new Trim();
   const sourcemap = new Sourcemap(outputFile);
   const stringify = get('content');
 
-  transclude
-    .pipe(trim)
-    .pipe(indenter)
-    .pipe(sourcemap)
-    .pipe(stringify);
-
-  const transcluder = duplexer(transclude, stringify);
+  transclude.on('error', () => transclude.end());
+  sourcemap.on('sourcemap', generatedSourceMap => (sourceMap = generatedSourceMap));
 
-  transclude.on('error', (err) => {
-    transcluder.emit('error', err);
-    transclude.end();
-  });
-
-  sourcemap.on('sourcemap', (generatedSourceMap) => {
-    sourceMap = generatedSourceMap;
-  });
+  transclude.pipe(trim).pipe(indenter).pipe(sourcemap).pipe(stringify);
 
+  const transcluder = duplexer({ bubbleErrors: false }, transclude, stringify);
   transcluder.on('end', () => transcluder.emit('sourcemap', sourceMap));
 
   return transcluder;
@@ -47,25 +39,15 @@ export function transcludeString(...args) {
   const source = _.get(options, 'source') || 'string';
 
   const transclude = new TranscludeStream(source, options);
-  let outputString = '';
   let sourceMap;
-  let cbErr = null;
-
-  transclude
-    .on('readable', function read() {
-      let content = null;
-      while ((content = this.read()) !== null) {
-        outputString += content.toString('utf8');
-      }
-    })
-    .on('error', (err) => {
-      if (!cbErr) cbErr = err;
-    })
-    .on('sourcemap', srcmap => (sourceMap = srcmap))
-    .on('end', () => cb(cbErr, outputString, sourceMap));
 
+  transclude.on('sourcemap', srcmap => (sourceMap = srcmap));
   transclude.write(input, 'utf8');
   transclude.end();
+
+  getStream(transclude)
+    .then(output => cb(null, output, sourceMap))
+    .catch(err => cb(err, err.bufferedData, sourceMap));
 }
 
 
@@ -76,24 +58,13 @@ export function transcludeFile(...args) {
 
   const transclude = new TranscludeStream(input, options);
   const inputStream = fs.createReadStream(input, { encoding: 'utf8' });
-  let outputString = '';
   let sourceMap;
-  let cbErr = null;
 
+  transclude.on('sourcemap', srcmap => (sourceMap = srcmap));
   inputStream.on('error', err => cb(err));
-
-  transclude
-    .on('readable', function read() {
-      let content = null;
-      while ((content = this.read()) !== null) {
-        outputString += content;
-      }
-    })
-    .on('error', (err) => {
-      if (!cbErr) cbErr = err;
-    })
-    .on('sourcemap', srcmap => (sourceMap = srcmap))
-    .on('end', () => cb(cbErr, outputString, sourceMap));
-
   inputStream.pipe(transclude);
+
+  getStream(transclude)
+    .then(output => cb(null, output, sourceMap))
+    .catch(err => cb(err, err.bufferedData, sourceMap));
 }

src/indent.js

@@ -1,9 +1,5 @@
 import through2 from 'through2';
 
-/**
-* Indents each line of a chunk by the provided indent amount
-*/
-
 export default function Indent() {
   const NEWLINE = '\n';
   const inputBuffer = [];
@@ -19,11 +15,9 @@ export default function Indent() {
       const beginsNewLine = inputBuffer[1].content.slice(0, 1) === NEWLINE;
 
       if (indent) {
-        if (preceededNewLine && !beginsNewLine) {
-          content = indent + content;
-        }
-        content = content.replace(/\n(?!\s|$)/g, `\n${indent}`);
+        if (preceededNewLine && !beginsNewLine) content = indent + content;
 
+        content = content.replace(/\n(?!\s|$)/g, `\n${indent}`);
         inputBuffer[1].content = content;
       }