Merge pull request #4 from caiges/master

Use official fastly api client for fetching realtime metrics.

Author: caiges

.env.example

@@ -0,0 +1,4 @@
+ACCOUNT_ID=yourNewRelicAccountId
+FASTLY_KEY=yourFastlyKey
+INSERT_KEY=yourNewRelicInsertKey
+SERVICES=space separated list of services

.gitignore

@@ -1 +1,2 @@
-node_modules/
+.env
+node_modules/

Dockerfile

@@ -2,7 +2,7 @@
 
 FROM alpine
 
-RUN apk add --update nodejs nodejs-npm
+RUN apk add --update nodejs npm
 
 WORKDIR /usr/src/app
 
@@ -12,4 +12,4 @@ RUN npm install
 
 COPY . .
 
-CMD ["node", "app.js"]
+CMD ["node", "index.js"]

Makefile

@@ -0,0 +1,7 @@
+image_name=newrelic/fastly-to-insights
+
+build:
+	docker build -t $(image_name) .
+
+run: build
+	docker run --rm --env-file .env --name fastly-to-insights $(image_name)

README.md

@@ -4,6 +4,12 @@ This is the New Relic blessed way to get your Fastly metrics into Insights, pack
 
 In order to use the Fastly to Insights Docker image, you will need an active New Relic account with Insights, an active Fastly account with Read access, a New Relic Insights Insert key and a Fastly API Key. The Docker image can be found [here](https://cloud.docker.com/swarm/newrelic/repository/docker/newrelic/fastly-to-insights/general).
 
+Once reporting starts, a `FastlySample` event will become available to query:
+
+```nrql
+FROM FastlySample select *
+```
+
 ## How to use this image
 
 Before you get started, make sure that you have a [Fastly API Key](https://docs.fastly.com/guides/account-management-and-security/using-api-tokens) and a [New Relic Insert Key](https://docs.newrelic.com/docs/insights/insights-data-sources/custom-data/insert-custom-events-insights-api#register).
@@ -17,6 +23,8 @@ The Fastly to Insights image is configured by environment variables. These are m
 
 `SERVICES` needs to be a string with the ids of the Fastly services you want to see data for in Insights, separated by a space. I know that's not ideal. A limitation of Fastly is that you have to query one service at a time, so I chose to create an array of service ids and loop through them to query Fastly. A limitation of Docker is that you can't pass an array via the command line, so I chose to split a string on "` `". If you have a better idea, I would love to hear it - please contribute!
 
+Additionally, the polling interval (how often realtime metrics are requested from Fastly) can be configured by setting the `POLL_INTERVAL` which, by default, is 3 minutes.
+
 ### Example
 
 ```shell
@@ -32,6 +40,12 @@ $ docker run \
 
 You are welcome to send pull requests to us - however, by doing so you agree that you are granting New Relic a non-exclusive, non-revokable, no-cost license to use the code, algorithms, patents, and ideas in that code in our products if we so choose. You also agree the code is provided as-is and you provide no warranties as to its fitness or correctness for any purpose.
 
+## Developing
+
+### Building and running the image locally
+
+A Makefile is provided which you can use to build and run the fastly to insights integration locally -- `make run` will build and then run the image. You can provide configuration via a `.env` file (`.env.example` provided as an example).
+
 ## More Information
 
 For more information on the Fastly Real-Time Analytics API, look [here](https://docs.fastly.com/api/analytics).

app.js

@@ -0,0 +1,85 @@
+// Copyright 2018 New Relic, Inc.  Licensed under the Apache License, version 2.0 (the "License"); you may not use this file except in compliance with the License.  You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, either express or implied. 
+//Copyright 2018 New Relic Inc. All rights reserved.
+
+import fetch from 'node-fetch';
+import { ApiClient, RealtimeApi, ServiceApi } from "fastly";
+
+const ACCOUNT_ID = process.env.ACCOUNT_ID
+const INSERT_KEY = process.env.INSERT_KEY
+const FASTLY_KEY = process.env.FASTLY_KEY
+const INSIGHTS_HOST = process.env.INSIGHTS_HOST || "insights-collector.newrelic.com";
+const EVENT_TYPE = process.env.EVENT_TYPE || "FastlySample";
+const POLL_INTERVAL = process.env.POLL_INTERVAL || 180000 // 3 minutes.
+
+// A limitation of Fastly is that you have to query one service at a time. I chose to create an array of my service ids, and loop through them.
+const LIST_OF_SERVICES = process.env.SERVICES.split(" ");
+
+let timestamp = 0;
+
+const realtimeApiClient = new ApiClient();
+realtimeApiClient.basePath = "https://rt.fastly.com";
+realtimeApiClient.authenticate(FASTLY_KEY);
+const realtimeApi = new RealtimeApi(realtimeApiClient);
+const apiClient = new ApiClient();
+apiClient.authenticate(FASTLY_KEY);
+const serviceApi = new ServiceApi(apiClient);
+
+// Perform an initial poll of the services and then set an interval to poll every 3 minutes.
+pollServices();
+setInterval(pollServices, POLL_INTERVAL) // 3 minutes
+
+function pollServices() {
+  LIST_OF_SERVICES.map(async (service) => {
+    pollFromFastly(service);
+  });
+}
+
+async function pollFromFastly(service) {
+  console.log(`Polling ${service}`);
+
+  try {
+    const serviceDetail = await serviceApi.getService({ service_id: service });
+    const valuableData = await realtimeApi.getStatsLastSecond({
+      service_id: service,
+      timestamp_in_seconds: timestamp
+    });
+
+    if (valuableData.Error) {
+      console.log('error in stats', valuableData.Error)
+    } else {
+      timestamp = valuableData.Timestamp
+      valuableData.Data.map((data) => {
+        batchAndSend(data.aggregated, { id: service, name: serviceDetail.name });
+      })
+    }
+
+  } catch (error) {
+    console.log('error fetching stats', error)
+  }
+}
+
+async function batchAndSend(aggregate, service) {
+  let message = {
+    "eventType": EVENT_TYPE,
+    "service_id": service.id,
+    "service_name": service.name,
+    ...aggregate,
+  }
+  await sendToInsights(message)
+}
+
+async function sendToInsights(logMessages) {
+  let insights_url = `https://${INSIGHTS_HOST}/v1/accounts/${ACCOUNT_ID}/events`;
+  try {
+    fetch(insights_url, {
+      method: "POST",
+      headers: {
+        'Content-Type': 'application/json',
+        "X-Insert-Key": INSERT_KEY
+      },
+      body: JSON.stringify(logMessages)
+    });
+  } catch (error) {
+    console.log('error posting to insights', error);
+  }
+}