Merge pull request #145 from apiaryio/kubula/source_files_at_results

Load blueprint-file from provided URL from http/https

Author: Adam Kliment

README.md

@@ -141,7 +141,7 @@ If `beforeAll` and `afterAll` are called multiple times, the callbacks are execu
 
     $ dredd --help
     Usage:
-      dredd <path to blueprint> <api_endpoint> [OPTIONS]
+      dredd <path or URL to blueprint> <api_endpoint> [OPTIONS]
 
     Example:
       dredd ./apiary.md http://localhost:3000 --dry-run
@@ -193,8 +193,8 @@ If `beforeAll` and `afterAll` are called multiple times, the callbacks are execu
                                                                     [default: false]
       --silent, -q         Silences commandline output.
                                                                     [default: false]
-      --path, -p           Additional blueprint paths. Can be used multiple times
-                           with glob pattern.                          [default: []]
+      --path, -p           Additional blueprint paths or URLs. Can be used multiple
+                           times with glob pattern for paths.          [default: []]
       --help               Show usage information.
 
       --version            Show version number.

bin/dredd

@@ -7,7 +7,7 @@ var Dredd = require('../lib/dredd');
 var version = parsePackageJson(path.join(__dirname, '../package.json'));
 
 var argv = optimist
-    .usage("Usage: \n  dredd <path to blueprint> <api_endpoint> [OPTIONS]" +
+    .usage("Usage: \n  dredd <path or URL to blueprint> <api_endpoint> [OPTIONS]" +
     "\n\nExample: \n  " + "dredd ./apiary.md http://localhost:3000 --dry-run")
     .options(Dredd.options)
     .wrap(80)

package.json

@@ -29,6 +29,7 @@
     "optimist": "~0.6.1",
     "protagonist": "~0.17.1",
     "proxyquire": "^1.3.1",
+    "request": "^2.53.0",
     "setimmediate": "^1.0.2",
     "uri-template": "~1.0.0",
     "winston": "^0.9.0"

src/dredd.coffee

@@ -6,6 +6,8 @@ fs = require 'fs'
 clone = require 'clone'
 protagonist = require 'protagonist'
 async = require 'async'
+request = require 'request'
+url = require 'url'
 
 logger = require './logger'
 options = require './options'
@@ -15,6 +17,8 @@ handleRuntimeProblems = require './handle-runtime-problems'
 blueprintAstToRuntime = require './blueprint-ast-to-runtime'
 configureReporters = require './configure-reporters'
 
+CONNECTION_ERRORS = ['ECONNRESET', 'ENOTFOUND', 'ESOCKETTIMEDOUT', 'ETIMEDOUT', 'ECONNREFUSED', 'EHOSTUNREACH', 'EPIPE']
+
 class Dredd
   constructor: (configOrigin) ->
     # do not touch passed configuration, rather work on a clone of it
@@ -65,6 +69,9 @@ class Dredd
     # expand all globs
     expandGlobs = (cb) ->
       async.each config.options.path, (globToExpand, globCallback) ->
+        if /^http(s)?:\/\//.test globToExpand
+          config.files = config.files.concat globToExpand
+          return globCallback()
         glob globToExpand, (err, match) ->
           globCallback err if err
           config.files = config.files.concat match
@@ -84,16 +91,43 @@ class Dredd
 
     # load all files
     loadFiles = (cb) ->
-      async.each config.files, (file, loadCallback) ->
-        fs.readFile file, 'utf8', (loadingError, data) ->
-          return loadCallback(loadingError) if loadingError
-          config.data[file] = {raw: data, filename: file}
-          loadCallback()
-
-      , (err) =>
+      # 6 parallel connections is a standard limit when connecting to one hostname,
+      # use the same limit of parallel connections for reading/downloading files
+      async.eachLimit config.files, 6, (fileUrlOrPath, loadCallback) ->
+        try
+          fileUrl = url.parse fileUrlOrPath
+        catch
+          fileUrl = null
+
+        if fileUrl and fileUrl.protocol in ['http:', 'https:'] and fileUrl.host
+          downloadFile fileUrlOrPath, loadCallback
+        else
+          readLocalFile fileUrlOrPath, loadCallback
+
+      , (err) ->
         return callback(err, stats) if err
         cb()
 
+    downloadFile = (fileUrl, downloadCallback) ->
+      request.get
+        url: fileUrl
+        timeout: 5000
+        json: false
+      , (downloadError, res, body) ->
+        if downloadError
+          downloadCallback {message: "Error when loading file from URL '#{fileUrl}'. Is the provided URL correct?"}
+        else if not body or res.statusCode < 200 or res.statusCode >= 300
+          downloadCallback {message: "Unable to load file from URL '#{fileUrl}'. Server did not send any blueprint back and responded with status code #{res.statusCode}."}
+        else
+          config.data[fileUrl] = {raw: body, filename: fileUrl}
+          downloadCallback()
+
+    readLocalFile = (filePath, readCallback) ->
+      fs.readFile filePath, 'utf8', (readingError, data) ->
+        return readCallback(readingError) if readingError
+        config.data[filePath] = {raw: data, filename: filePath}
+        readCallback()
+
     # parse all file blueprints
     parseBlueprints = (cb) ->
       async.each Object.keys(config.data), (file, parseCallback) ->

src/options.coffee

@@ -81,7 +81,7 @@ options =
 
   path:
     alias: "p"
-    description: "Additional blueprint paths. Can be used multiple times with glob pattern."
+    description: "Additional blueprint paths or URLs. Can be used multiple times with glob pattern for paths."
     default: []
 
   help:

test/integration/cli-test.coffee

@@ -5,7 +5,7 @@ clone = require 'clone'
 bodyParser = require 'body-parser'
 fs = require 'fs'
 
-PORT = '3333'
+PORT = 3333
 CMD_PREFIX = ''
 
 stderr = ''
@@ -38,7 +38,7 @@ execCommand = (cmd, options = {}, callback) ->
 
 describe "Command line interface", () ->
 
-  describe "When blueprint file not found", (done) ->
+  describe "When blueprint file not found", ->
     before (done) ->
       cmd = "./bin/dredd ./test/fixtures/nonexistent_path.md http://localhost:#{PORT}"
 
@@ -50,6 +50,114 @@ describe "Command line interface", () ->
     it 'should print error message to stderr', () ->
       assert.include stderr, 'not found'
 
+
+  describe "When blueprint file should be loaded from 'http(s)://...' url", ->
+    server = null
+    loadedFromServer = null
+    connectedToServer = null
+    notFound = null
+    fileFound = null
+
+    errorCmd = "./bin/dredd http://localhost:#{PORT+1}/connection-error.apib http://localhost:#{PORT+1}"
+    wrongCmd = "./bin/dredd http://localhost:#{PORT}/not-found.apib http://localhost:#{PORT}"
+    goodCmd = "./bin/dredd http://localhost:#{PORT}/file.apib http://localhost:#{PORT}"
+
+    afterEach ->
+      connectedToServer = null
+
+    before (done) ->
+      app = express()
+
+      app.use (req, res, next) ->
+        connectedToServer = true
+        next()
+
+      app.get '/', (req, res) ->
+        res.sendStatus 404
+
+      app.get '/file.apib', (req, res) ->
+        fileFound = true
+        res.type('text')
+        stream = fs.createReadStream './test/fixtures/single-get.apib'
+        stream.pipe res
+
+      app.get '/machines', (req, res) ->
+        res.setHeader 'Content-Type', 'application/json'
+        machine =
+          type: 'bulldozer'
+          name: 'willy'
+        response = [machine]
+        res.status(200).send response
+
+      app.get '/not-found.apib', (req, res) ->
+        notFound = true
+        res.status(404).end()
+
+      server = app.listen PORT, ->
+        done()
+
+    after (done) ->
+      server.close ->
+        app = null
+        server = null
+        done()
+
+    describe 'and I try to load a file from bad hostname at all', ->
+      before (done) ->
+        execCommand errorCmd, ->
+          done()
+
+      after ->
+        connectedToServer = null
+
+      it 'should not send a GET to the server', ->
+        assert.isNull connectedToServer
+
+      it 'should exit with status 1', ->
+        assert.equal exitStatus, 1
+
+      it 'should print error message to stderr', ->
+        assert.include stderr, 'Error when loading file from URL'
+        assert.include stderr, 'Is the provided URL correct?'
+        assert.include stderr, 'connection-error.apib'
+
+    describe 'and I try to load a file that does not exist from an existing server', ->
+      before (done) ->
+        execCommand wrongCmd, ->
+          done()
+
+      after ->
+        connectedToServer = null
+
+      it 'should connect to the right server', ->
+        assert.isTrue connectedToServer
+
+      it 'should send a GET to server at wrong URL', ->
+        assert.isTrue notFound
+
+      it 'should exit with status 1', ->
+        assert.equal exitStatus, 1
+
+      it 'should print error message to stderr', ->
+        assert.include stderr, 'Unable to load file from URL'
+        assert.include stderr, 'responded with status code 404'
+        assert.include stderr, 'not-found.apib'
+
+    describe 'and I try to load a file that actually is there', ->
+      before (done) ->
+        execCommand goodCmd, ->
+          done()
+
+      it 'should send a GET to the right server', ->
+        assert.isTrue connectedToServer
+
+      it 'should send a GET to server at good URL', ->
+        assert.isTrue fileFound
+
+      it 'should exit with status 0', ->
+        assert.equal exitStatus, 0
+
+
   describe "Arguments with existing blueprint and responding server", () ->
     describe "when executing the command and the server is responding as specified in the blueprint", () ->