Added filter for deployment (tiles)

.circleci/ci_nozzle_manifest.yml

@@ -0,0 +1,36 @@
+---
+applications:
+  - name: splunk-firehose-nozzle
+    memory: 512M
+    instances: 2
+    buildpack: go_buildpack
+    cmd: splunk-firehose-nozzle
+    env:
+      GOPACKAGENAME: main
+      API_ENDPOINT:
+      API_USER:
+      API_PASSWORD:
+      SPLUNK_HOST:
+      SPLUNK_TOKEN:
+      SPLUNK_INDEX:
+      SKIP_SSL_VALIDATION_CF: true
+      SKIP_SSL_VALIDATION_SPLUNK: true
+      JOB_NAME: splunk-nozzle
+      JOB_INDEX: -1
+      JOB_HOST: localhost
+      ADD_APP_INFO: true
+      IGNORE_MISSING_APP: true
+      MISSING_APP_CACHE_INVALIDATE_TTL: 3600s
+      APP_CACHE_INVALIDATE_TTL: 86440s
+      APP_LIMITS: 1000
+      BOLTDB_PATH: cache.db
+      EVENTS: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
+      EXTRA_FIELDS: name:update-ci-test
+      FIREHOSE_SUBSCRIPTION_ID: splunk-ci
+      FLUSH_INTERVAL: 5s
+      CONSUMER_QUEUE_SIZE: 10000
+      HEC_BATCH_SIZE: 1000
+      HEC_RETRIES: 5
+      HEC_WORKERS: 8
+      DEBUG: false
+      ENABLE_EVENT_TRACING: true

.circleci/config.yml

@@ -0,0 +1,92 @@
+version: 2 # use CircleCI 2.0
+jobs:
+  build:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps: # steps that comprise the `build` job
+      - checkout # check out source code to working directory
+      - run:
+          name: Install Dependencies
+          command: |
+            curl https://glide.sh/get | sh
+            go get -t ./...
+      - run:
+          name: Builder
+          command: make build
+      - run:
+          name: Run tests
+          command: |
+            make testall
+            cp splunk-firehose-nozzle /tmp
+      - persist_to_workspace:
+          root: /tmp
+          paths:
+            - splunk-firehose-nozzle
+
+  deploy-nozzle:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps: # steps that comprise the `deploy` job
+      - attach_workspace:
+          at: /tmp
+      - checkout # check out source code to working directory
+      - run:
+          name: Install dependencies
+          command: |
+            curl https://glide.sh/get | sh
+            go get -t ./...
+            cp -R /tmp/splunk-firehose-nozzle .
+      - run:
+          name: Deploy nozzle
+          command: |
+            .circleci/update_manifest.sh
+            .circleci/pre-req.sh
+            cf push -f .circleci/ci_nozzle_manifest.yml -u process --random-route
+      - run:
+          name: Teardown deployment env
+          command: |
+            sleep 10
+            echo "Teardown deployment env"
+            cf delete splunk-firehose-nozzle -f
+            cf delete-org splunk-ci -f
+
+  tile-builder:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - checkout
+      - run:
+          name: Tile builder
+          command: |
+            cp -R /tmp/splunk-firehose-nozzle .
+            .circleci/tile-builder.sh
+      - run:
+          name: Push tile to internal s3
+          command: |
+            cp -R /tmp/splunk-firehose-nozzle .
+            .circleci/push_tile.sh
+
+workflows:
+  version: 2
+  build-and-deploy-nozzle:
+    jobs:
+      - build
+      - deploy-nozzle:
+          requires:
+            - build
+          filters:
+            branches:
+              only:
+                - develop
+                - master
+      - tile-builder:
+          requires:
+            - deploy-nozzle
+          filters:
+            branches:
+              only: master

.circleci/pre-req.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+#Install CF CLI
+wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
+echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
+#Add support for https apt sources
+sudo apt-get install apt-transport-https ca-certificates
+sudo apt-get update
+sudo apt-get install cf-cli
+#CF Login
+cf login --skip-ssl-validation -a $API_ENDPOINT -u $API_USER -p $API_PASSWORD -o system -s system
+#Create splunk-ci org and space
+if [  "`cf o | grep "splunk-ci"`" == "splunk-ci" ]; then
+   echo "Its here"
+   cf target -o "splunk-ci" -s "splunk-ci"
+else
+   cf create-org splunk-ci
+   cf target -o splunk-ci
+   cf create-space splunk-ci
+   cf target -o "splunk-ci" -s "splunk-ci"
+fi

.circleci/push_tile.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -e
+sudo apt-get install -y python-pip libpython-dev > /dev/null 2>&1
+echo "Installing aws cli..."
+sudo pip install awscli > /dev/null 2>&1
+echo "Push Splunk tile to s3..."
+aws s3 cp tile/product/splunk-nozzle-*.pivotal s3://pcf-ci-artifacts/

.circleci/tile-builder.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+wget https://github.com/cf-platform-eng/tile-generator/releases/download/v13.0.2/pcf_linux-64bit > /dev/null 2>&1
+chmod +x pcf_linux-64bit
+sudo mv pcf_linux-64bit /usr/local/bin/tile
+sudo apt install python-pip > /dev/null 2>&1
+echo "installing vurtualenv"
+sudo /usr/bin/easy_install virtualenv > /dev/null 2>&1
+virtualenv -p python tile-generator-env
+source tile-generator-env/bin/activate
+echo "Installing tile-generator..."
+pip install tile-generator
+cd tile
+echo "Installing bosh..."
+wget https://github.com/cloudfoundry/bosh-cli/releases/download/v5.5.0/bosh-cli-5.5.0-linux-amd64 > /dev/null 2>&1
+mv bosh-cli-5.5.0-linux-amd64 bosh
+chmod +x ./bosh
+sudo mv ./bosh /usr/local/bin/bosh
+echo "Building PCF Tile for Splunk-firehose-nozzle"
+tile build

.circleci/update_manifest.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+#Set below params in CircleCI env variable settings
+# API_ENDPOINT, API_USER, API_PASSWORD, SPLUNK_TOKEN, SPLUNK_HOST, SPLUNK_INDEX
+#Update manifest for deployment
+sed -i 's@API_ENDPOINT:.*@'"API_ENDPOINT: $API_ENDPOINT"'@' .circleci/ci_nozzle_manifest.yml
+sed -i 's@API_USER:.*@'"API_USER: $API_USER"'@' .circleci/ci_nozzle_manifest.yml
+sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD"'@' .circleci/ci_nozzle_manifest.yml
+sed -i 's@SPLUNK_HOST:.*@'"SPLUNK_HOST: $SPLUNK_HOST"'@' .circleci/ci_nozzle_manifest.yml
+sed -i 's@SPLUNK_TOKEN:.*@'"SPLUNK_TOKEN: $SPLUNK_TOKEN"'@' .circleci/ci_nozzle_manifest.yml
+sed -i 's@SPLUNK_INDEX:.*@'"SPLUNK_INDEX: $SPLUNK_INDEX"'@' .circleci/ci_nozzle_manifest.yml
+#copy nozzle binary from shared workspace
+cp /tmp/splunk-firehose-nozzle .

.gitignore

@@ -4,6 +4,7 @@ tools/dump_app_info/dump_app_info
 tools/data_gen/data_gen
 
 
+
 cache.db
 *.out
 

README.md

@@ -1,3 +1,4 @@
+[![CircleCI](https://circleci.com/gh/git-lfs/git-lfs.svg?style=shield&circle-token=856152c2b02bfd236f54d21e1f581f3e4ebf47ad)](https://circleci.com/gh/cloudfoundry-community/splunk-firehose-nozzle)
 ## Splunk Nozzle
 
 Cloud Foundry Firehose-to-Splunk Nozzle
@@ -141,6 +142,24 @@ $ ./dump_app_info --skip-ssl-validation --api-endpoint=https://<your api endpoin
 After populating the application info cache file, user can copy to different Splunk nozzle deployments and start Splunk nozzle to pick up this cache file by
 specifying correct "--boltdb-path" flag or "BOLTDB_PATH" environment variable.
 
+###Per application index routing (deprecates instructions below)
+in your app manifest provide an env var called SPLUNK_INDEX and assign it the index you would like to send the data to
+
+```
+applications:
+- name: console
+  memory: 256M
+  disk_quota: 256M
+  host: console
+  timeout: 180
+  buildpack: https://github.com/SUSE/stratos-buildpack
+  health-check-type: port
+  services:
+  - splunk-index
+  env:
+    SPLUNK_INDEX: testing_index
+```
+
 #### Index routing
 Index routing is a feature that can be used to send different Cloud Foundry logs to different indexes for better ACL and data retention control in Splunk.
 Logs can be routed using fields such as app ID/name, space ID/name or org ID/name.