add network fees API endpoint

Author: shekenahglory

README.md

@@ -52,6 +52,7 @@ General Methods:
 * [Get XRP Distribution - `GET /v2/network/xrp_distribution`](#get-xrp-distribution)
 * [Get Top Currencies - `GET /v2/network/top_currencies`](#get-top-currencies)
 * [Get Top Markets - `GET /v2/network/top_markets`](#get-top-markets)
+* [Get Network Fees - `GET /v2/network/fees`](#get-network-fees)
 * [Get Topology - `GET /v2/network/topology`](#get-topology)
 * [Get Topology Nodes - `GET /v2/network/topology/nodes`](#get-topology-nodes)
 * [Get Topology Node - `GET /v2/network/topology/nodes/{:pubkey}`](#get-topology-nodes)
@@ -2209,6 +2210,91 @@ Response:
 
 
 
+## Get Network Fees ##
+[[Source]<br>](https://github.com/ripple/rippled-historical-database/blob/develop/api/routesV2/network/getFees.js "Source")
+
+Returns network fee stats per ledger, hour, or day.  The data shows the average, minimum, maximum, and total fees incurred for the given interval/ledger. _(New in [v2.2.0][])_
+
+#### Request Format ####
+
+<!--<div class='multicode'>-->
+
+```
+GET /v2/network/fees
+```
+
+<!--</div>-->
+
+Optionally, you can include the following query parameters:
+
+| Field  | Value   | Description |
+|--------|---------|-------------|
+| start      | String - [Timestamp][]  | Start time of query range. Defaults to the start of the most recent interval. |
+| end        | String - [Timestamp][]  | End time of query range. Defaults to the end of the most recent interval. |
+| interval | String  | Aggregation interval - valid intervals are `ledger`, `hour`, or `day`. Defaults to `ledger`. |
+| descending | Boolean | Reverse chronological order |
+| limit    | Integer | Maximum results per page. Defaults to 200. Cannot be more than 1000. |
+| marker   | String  | [Pagination](#pagination) key from previously returned response |
+| format     | String  | Format of returned results: `csv` or `json`. Defaults to `json`. |
+
+[Try it! >](https://ripple.com/build/data-api-tool/#get-network-fees)
+
+
+#### Response Format ####
+
+A successful response uses the HTTP code **200 OK** and has a JSON body with the following:
+
+| Field  | Value | Description |
+|--------|-------|-------------|
+| result | String | The value `success` indicates that this is a successful response. |
+| marker | String | (May be omitted) [Pagination](#pagination) marker. |
+| count  | Integer | Number of results in the `markets` field. |
+| rows | Array of Fee Summary Objects | Network fee statistics for each specific interval. |
+
+Each Fee Summary object has the following fields:
+
+| Field  | Value | Description |
+|--------|-------|-------------|
+| avg | Number | Average network fee |
+| min | Number | Minimum network fee |
+| max | Number | Maximum network fee |
+| total | Number | Total XRP consumed as network fees |
+| tx_count | Number | Number of transactions in this interval |
+| date | String - [Timestamp][] | Interval start time or ledger close time |
+| ledger_index | Integer | Ledger index (present in `ledger` interval only) |
+
+#### Example ####
+
+Request:
+
+```
+GET /v2/network/fees?interval=hour
+```
+
+Response:
+
+```
+{
+  result: "success",
+  marker: "hour|20130124080000",
+  count: 200,
+  rows: [
+    {
+      avg: 0.00001,
+      max: 0.00001,
+      min: 0.00001,
+      total: 0.00001,
+      tx_count: 1,
+      date: "2013-01-02T06:00:00Z"
+    },
+    ...
+  ]
+}
+```
+
+
+
+
 ## Get Topology ##
 [[Source]<br>](https://github.com/ripple/rippled-historical-database/blob/develop/api/routesV2/network/getTopology.js "Source")
 

api/routesV2/accountExchanges.js

@@ -17,7 +17,7 @@ AccountExchanges = function (req, res, next) {
 
   } else if (!options.end) {
     errorResponse({
-      error: 'invalid start date format',
+      error: 'invalid end date format',
       code: 400
     });
     return;

api/routesV2/network/getFees.js

@@ -0,0 +1,113 @@
+'use strict';
+
+var Logger = require('../../../lib/logger');
+var log = new Logger({scope: 'network fees'});
+var smoment = require('../../../lib/smoment');
+var utils = require('../../../lib/utils');
+
+var hbase;
+var intervals = [
+  'ledger',
+  'hour',
+  'day'
+];
+
+var getFees = function(req, res) {
+  var options = {
+    interval: req.query.interval || 'ledger',
+    start: smoment(req.query.start || '2013-01-01'),
+    end: smoment(req.query.end),
+    limit: req.query.limit,
+    marker: req.query.marker,
+    descending: (/true/i).test(req.query.descending) ? true : false,
+    format: (req.query.format || 'json').toLowerCase()
+  };
+
+  if (!options.start) {
+    errorResponse({
+      error: 'invalid start date format',
+      code: 400
+    });
+    return;
+
+  } else if (!options.end) {
+    errorResponse({
+      error: 'invalid end date format',
+      code: 400
+    });
+    return;
+
+  } else if (intervals.indexOf(options.interval) === -1) {
+    errorResponse({
+      error: 'invalid interval',
+      code: 400
+    });
+    return;
+  }
+
+  if (isNaN(options.limit)) {
+      options.limit = 200;
+  } else if (options.limit > 1000) {
+      options.limit = 1000;
+  }
+
+  log.info('interval:', options.interval);
+
+  hbase.getNetworkFees(options)
+  .then(successResponse)
+  .catch(errorResponse);
+
+  /**
+  * errorResponse
+  * return an error response
+  * @param {Object} err
+  */
+
+  function errorResponse(err) {
+    log.error(err.error || err);
+    if (err.code && err.code.toString()[0] === '4') {
+      res.status(err.code).json({
+        result: 'error',
+        message: err.error
+      });
+    } else {
+      res.status(500).json({
+        result: 'error',
+        message: 'unable to retrieve fee summary(s)'
+      });
+    }
+  }
+
+  /**
+  * successResponse
+  * return a successful response
+  * @param {Object} markets
+  * @param {Object} options
+  */
+
+  function successResponse(data) {
+    var filename;
+
+    if (data.marker) {
+      utils.addLinkHeader(req, res, data.marker);
+    }
+
+    if (options.format === 'csv') {
+      filename = 'network fees.csv';
+      res.csv(data.rows, filename);
+
+    } else {
+      res.json({
+        result: 'success',
+        marker: data.marker,
+        count: data.rows.length,
+        rows: data.rows
+      });
+    }
+  }
+};
+
+module.exports = function(db) {
+  hbase = db;
+  return getFees;
+};

api/routesV2/network/index.js

@@ -7,6 +7,7 @@ module.exports = function(db) {
     xrpDistribution: require('./xrpDistribution')(db),
     topMarkets: require('./topMarkets')(db),
     topCurrencies: require('./topCurrencies')(db),
+    getFees: require('./getFees')(db),
     getNodes: require('./getNodes')(db),
     getLinks: require('./getLinks')(db),
     getTopology: require('./getTopology')(db),

api/server.js

@@ -68,6 +68,7 @@ var Server = function (options) {
     app.get('/v2/network/xrp_distribution', routesV2.network.xrpDistribution);
     app.get('/v2/network/top_markets/:date?', routesV2.network.topMarkets);
     app.get('/v2/network/top_currencies/:date?', routesV2.network.topCurrencies);
+    app.get('/v2/network/fees', routesV2.network.getFees);
     app.get('/v2/network/topology', routesV2.network.getTopology);
     app.get('/v2/network/topology/nodes', routesV2.network.getNodes);
     app.get('/v2/network/topology/nodes/:pubkey', routesV2.network.getNodes);

lib/hbase/hbase-thrift/data.js

@@ -1860,6 +1860,55 @@ HbaseClient.getTransactions = function(options, callback) {
   }
 };
 
+/**
+ * getNetworkFees
+ */
+
+HbaseClient.getNetworkFees = function(options) {
+  var self = this;
+  var startRow = [
+    options.interval,
+    options.start.hbaseFormatStartRow()
+  ].join('|');
+
+  var stopRow = [
+    options.interval,
+    options.end.hbaseFormatStopRow()
+  ].join('|');
+
+  return new Promise(function(resolve, reject) {
+    self.getScanWithMarker(self, {
+      table: 'network_fees',
+      startRow: startRow,
+      stopRow: stopRow,
+      limit: options.limit,
+      marker: options.marker,
+      descending: options.descending
+    }, function(err, resp) {
+      if (err) {
+        reject(err);
+      } else {
+        resp.rows.forEach(function(r) {
+          if (r.ledger_index) {
+            r.ledger_index = Number(r.ledger_index);
+          }
+
+          r.avg = Number(r.avg);
+          r.max = Number(r.max);
+          r.min = Number(r.min);
+          r.total = Number(r.total);
+          r.tx_count = Number(r.tx_count);
+
+          delete r.interval;
+          delete r.rowkey;
+        });
+
+        resolve(resp);
+      }
+    });
+  });
+}
+
 /**
  * getAccounts
  */

test/setup.importLedgers.js

@@ -8,6 +8,7 @@ var moment = require('moment');
 var exAggregation = require('../lib/aggregation/exchanges');
 var statsAggregation = require('../lib/aggregation/stats');
 var paymentsAggregation = require('../lib/aggregation/accountPayments');
+var feesAggregation = require('../lib/aggregation/fees');
 
 var fs = require('fs');
 var path = __dirname + '/mock/ledgers/';
@@ -17,10 +18,12 @@ var statsConfig;
 var updates = [];
 var exchanges = [];
 var payments = [];
+var fees = [];
 var pairs = { };
 var hbase;
 var stats;
 var aggPayments;
+var aggFees;
 
 
 hbaseConfig.prefix = config.get('prefix');
@@ -29,6 +32,7 @@ hbaseConfig.max_sockets = 100;
 hbaseConfig.timeout = 60000;
 
 aggPayments = new paymentsAggregation(hbaseConfig);
+aggFees = new feesAggregation(hbaseConfig);
 stats = new statsAggregation(hbaseConfig);
 hbase = new HBase(hbaseConfig);
 
@@ -46,6 +50,9 @@ describe('import ledgers', function(done) {
         //save payments
         payments.push.apply(payments, parsed.payments);
 
+        //save fees
+        fees.push(parsed.feeSummary);
+
         //save stats
         addStats(parsed);
         updates.push({
@@ -76,6 +83,19 @@ describe('import ledgers', function(done) {
     });
   });
 
+  it('should aggregate network fees', function(done) {
+    this.timeout(7000);
+    Promise.map(fees, function(feeSummary) {
+      return aggFees.handleFeeSummary(feeSummary);
+    })
+    .then(function() {
+      done();
+    })
+    .catch(function(e) {
+      assert.ifError(e);
+    });
+  });
+
   it('should save exchanges into hbase', function(done) {
     this.timeout(15000);
     exchanges.forEach(function(ex, i) {