Merge 2.0.0 changes to master - V2 upgrade (#258)

* RLP gateway logging, error channel, configurable retry. change es channel size to 10000 (#239)
* update go-loggregator lib to include rlp client update for perf improvement (#246)
* added retry counter (i) in the error message (#223)
* Added additional test cases with running binary nuzzle (#226)
* Update test_nozzle_configurations.py (#234)
* Added performance tests
* Data flow monitor (#242)

Co-authored-by: Shubham Jain <[email protected]>

* Add telemetry data (appname and appversion) to HEC requests (#250)
* Metadata config filter (#251)
* updated tile to add fields for metadata filter, rlp-retries, queue monitoring, updated unit-tests
* Improve efficiency in event source monitoring and AddAppInfo param clean up (#254)
* clean event source and restructure AddAppInfo param
* upate app-info config param in the tile (#255)
* update config per new params and versions (#256)

Co-authored-by: Rock Baek <[email protected]>

Author: luckyj5

.circleci/ci_nozzle_manifest.yml

@@ -7,18 +7,18 @@ applications:
     cmd: splunk-firehose-nozzle
     env:
       GOPACKAGENAME: main
-      API_ENDPOINT:
-      CLIENT_ID:
-      CLIENT_SECRET:
-      SPLUNK_HOST:
-      SPLUNK_TOKEN:
-      SPLUNK_INDEX:
+      API_ENDPOINT: 
+      CLIENT_ID: 
+      CLIENT_SECRET: 
+      SPLUNK_HOST: 
+      SPLUNK_TOKEN: 
+      SPLUNK_INDEX: main
       SKIP_SSL_VALIDATION_CF: true
       SKIP_SSL_VALIDATION_SPLUNK: true
       JOB_NAME: splunk-nozzle
       JOB_INDEX: -1
       JOB_HOST: localhost
-      ADD_APP_INFO: true
+      ADD_APP_INFO: AppName,OrgName,OrgGuid,SpaceName,SpaceGuid
       IGNORE_MISSING_APP: true
       MISSING_APP_CACHE_INVALIDATE_TTL: 3600s
       APP_CACHE_INVALIDATE_TTL: 86440s
@@ -34,3 +34,5 @@ applications:
       HEC_WORKERS: 8
       DEBUG: false
       ENABLE_EVENT_TRACING: true
+      RLP_GATEWAY_RETRIES: 5
+      STATUS_MONITOR_INTERVAL: 0s

.circleci/config.yml

@@ -102,6 +102,37 @@ jobs:
           when:
             always
 
+  execute_perf_tests:
+    docker:
+      - image: circleci/golang:1.12
+    working_directory: /go/src/github.com/cloudfoundry-community/splunk-firehose-nozzle
+    steps:
+      - attach_workspace:
+          at: /tmp
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            curl https://glide.sh/get | sh
+            go get -t ./...
+            cp -R /tmp/splunk-firehose-nozzle .
+      - run:
+          name: Prepare test environment
+          command: |
+            .circleci/pre-req.sh
+            .circleci/pre-functional-test.sh
+      - run:
+          name: Executing perf tests
+          command: |
+            .circleci/performance-test.sh
+      - run:
+          name: Teardown
+          command: |
+            echo "Teardown deployment env"
+            cf delete-org splunk-ci-org -f
+          when:
+            always
+
 workflows:
   version: 2
   build-and-deploy-nozzle:
@@ -110,11 +141,6 @@ workflows:
       - deploy-nozzle:
           requires:
             - build
-          filters:
-            branches:
-              only:
-                - develop
-                - master
 #      - tile-builder:
 #          requires:
 #            - deploy-nozzle
@@ -123,10 +149,26 @@ workflows:
 #              only: master
       - execute_tests:
           requires:
-            - build
             - deploy-nozzle
-          filters:
-            branches:
-              only:
-                - develop
-                - master
+
+
+#      - execute_perf_tests:
+#          requires:
+#            - build
+#          filters:
+#            branches:
+#              only:
+#                - dev/pcf-performance-testing
+#  nightly:
+#    triggers:
+#      - schedule:
+#          cron: "0 7 * * *"
+#          filters:
+#            branches:
+#              only:
+#                - dev/pcf-performance-testing
+#    jobs:
+#      - build
+#      - execute_perf_tests:
+#          requires:
+#            - build

.circleci/functional-test.sh

@@ -2,4 +2,4 @@
 
 cd testing/integration
 . venv/bin/activate
-pytest
+pytest -v -m Critical

.circleci/performance-test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+cd testing/integration
+. venv/bin/activate
+pytest -v -m Perf_Binary
+pytest -v -m Perf_Romote

.gitignore

@@ -17,3 +17,6 @@ vendor/github.com/cloudfoundry/sonde-go/definitions/
 testing/integration/venv/
 testing/integration/config/local.ini
 *.pyc
+testing/integration/config/env.json
+testing/manual_perf/data_gen_manifest.yml
+

Makefile

@@ -65,6 +65,8 @@ testall: test vet race cov
 test:
 	@go test ${PKGS}
 
+testv:
+	@go test -v ${PKGS}
 # Run "short" unit tests
 test-short:
 	@go test -short ${PKGS}

README.md

@@ -79,7 +79,7 @@ This is recommended for dev environments only.
 This is recommended for dev environments only.
 * `FIREHOSE_SUBSCRIPTION_ID`: Tags nozzle events with a Firehose subscription id. See https://docs.pivotal.io/pivotalcf/1-11/loggregator/log-ops-guide.html.
 * `FIREHOSE_KEEP_ALIVE`: Keep alive duration for the Firehose consumer.
-* `ADD_APP_INFO`: Enriches raw data with app details.
+* `ADD_APP_INFO`: Enrich raw data with app info. A comma separated list of app metadata (AppName,OrgName,OrgGuid,SpaceName,SpaceGuid).
 * `IGNORE_MISSING_APP`: If the application is missing, then stop repeatedly querying application info from Cloud Foundry.
 * `MISSING_APP_CACHE_INVALIDATE_TTL`:  How frequently the missing app info cache invalidates.
 * `APP_CACHE_INVALIDATE_TTL`: How frequently the app info local cache invalidates.
@@ -95,6 +95,9 @@ This is recommended for dev environments only.
 * `HEC_WORKERS`: Set the amount of Splunk HEC workers to increase concurrency while ingesting in Splunk.
 * `ENABLE_EVENT_TRACING`: Enables event trace logging. Splunk events will now contain a UUID, Splunk Nozzle Event Counts, and a Subscription-ID for Splunk correlation searches.
 * `SPLUNK_VERSION`: The Splunk version that determines how HEC ingests metadata fields. Only required for Splunk version 6.3 or below.
+* `RLP_GATEWAY_RETRIES`: Number of retries to connect to RLP gateway.
+* `STATUS_MONITOR_INTERVAL`: Time interval for monitoring memory queue pressure to help with back-pressure insights.
+
     ###  Please note 
     > SPLUNK_VERSION configuration parameter is only required for Splunk version 6.3 and below. 
     For Splunk version 6.3 or below, please deploy nozzle via CLI. Update nozzle_manifest.yml with splunk_version (eg:- SPLUNK_VERSION: 6.3) as an env variable and [deploy nozzle as an app via CLI](#push-as-an-app-to-cloud-foundry).
@@ -134,11 +137,11 @@ on user authentication.
     ```
 
 #### Dump application info to boltdb ####
-If in production there are lots of PCF applications(say tens of thousands) and if the user would like to enrich
-application logs by including application meta data,querying all application metadata information from PCF may take some time.
+If in production there are lots of Cloud Foundry applications(say tens of thousands) and if the user would like to enrich
+application logs by including application meta data,querying all application metadata information from Cloud Foundry may take some time.
 For example if we include, add app name, space ID, space name, org ID and org name to the events.
 If there are multiple instances of Spunk nozzle deployed the situation will be even worse, since each of the Splunk nozzle(s) will query all applications meta data and
-cache the meta data information to the local boltdb file. These queries will introduce load to the PCF system and could potentially take a long time to finish.
+cache the meta data information to the local boltdb file. These queries will introduce load to the Cloud Foundry system and could potentially take a long time to finish.
 Users can run this tool to generate a copy of all application meta data and copy this to each Splunk nozzle deployment. Each Splunk nozzle can pick up the cache copy and update the cache file incrementally afterwards.
 
 Example of how to run the dump application info tool:
@@ -167,8 +170,6 @@ applications:
   timeout: 180
   buildpack: https://github.com/SUSE/stratos-buildpack
   health-check-type: port
-  services:
-  - splunk-index
   env:
     SPLUNK_INDEX: testing_index
 ```
@@ -238,7 +239,7 @@ This topic describes how to troubleshoot Splunk Firehose Nozzle for Cloud Foundr
 
   Are you searching for events and not finding them or looking at a dashboard and seeing "No result found"? Check Splunk Nozzle app logs.
 
-  To view the nozzle's logs running on PCF do the following:
+  To view the nozzle's logs running on Cloud Foundry do the following:
 
 <ol>
   <li>Log in as an admin via the CLI.</li>
@@ -310,7 +311,7 @@ A correct setup logs a start message with configuration parameters of the Nozzle
 
 <pre class="terminal">
   data:	{
-     add-app-info: true
+     add-app-info: AppName,OrgName,OrgGuid,SpaceName,SpaceGuid
      api-endpoint: https://api.endpoint.com
      app-cache-ttl: 0
      app-limits: 0
@@ -337,7 +338,8 @@ A correct setup logs a start message with configuration parameters of the Nozzle
      splunk-version: 6.6
      subscription-id: splunk-firehose
      trace-logging: true
-     version:
+     rlp-gateway-retries: 5
+     status-monitor-interval: 0s
      wanted-events: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
   }
   ip: 10.0.0.0
@@ -394,7 +396,7 @@ Make sure you have the following installed on your workstation:
 
 | Software | Version
 | --- | --- |
-| go | go1.8.x
+| go | go1.12.x
 | glide | 0.12.x
 
 Then install all dependent packages via [Glide](https://glide.sh/):
@@ -417,7 +419,7 @@ $ chmod +x tools/nozzle.sh
 Build project:
 
 ```
-$ make VERSION=1.1
+$ make VERSION=2.0.0
 ```
 
 Run tests with [Ginkgo](http://onsi.github.io/ginkgo/)

cache/boltdb.go

@@ -78,13 +78,13 @@ func (c *Boltdb) Open() error {
 	// Open bolt db
 	db, err := bolt.Open(c.config.Path, 0600, &bolt.Options{Timeout: 5 * time.Second})
 	if err != nil {
-		c.config.Logger.Error("Fail to open boltdb: ", err)
+		c.config.Logger.Error("Failed to open boltdb: ", err)
 		return err
 	}
 	c.appdb = db
 
 	if err := c.createBucket(); err != nil {
-		c.config.Logger.Error("Fail to create bucket: ", err)
+		c.config.Logger.Error("Failed to create bucket: ", err)
 		return err
 	}
 

eventrouter/default.go

@@ -9,14 +9,13 @@ import (
 	"github.com/cloudfoundry/sonde-go/events"
 )
 
-type Config struct {
-	SelectedEvents string
-}
+type Config = fevents.Config
 
 type router struct {
 	appCache       cache.Cache
 	sink           eventsink.Sink
 	selectedEvents map[string]bool
+	config         *Config
 }
 
 func New(appCache cache.Cache, sink eventsink.Sink, config *Config) (Router, error) {
@@ -30,6 +29,7 @@ func New(appCache cache.Cache, sink eventsink.Sink, config *Config) (Router, err
 		appCache:       appCache,
 		sink:           sink,
 		selectedEvents: selectedEvents,
+		config:         config,
 	}, nil
 }
 
@@ -64,7 +64,7 @@ func (r *router) Route(msg *events.Envelope) error {
 	event.AnnotateWithCFMetaData()
 
 	if _, hasAppId := event.Fields["cf_app_id"]; hasAppId {
-		event.AnnotateWithAppData(r.appCache)
+		event.AnnotateWithAppData(r.appCache, r.config)
 	}
 
 	if ignored, ok := event.Fields["cf_ignored_app"]; ok {

eventrouter/eventrouter_test.go

@@ -137,15 +137,6 @@ var _ = Describe("eventrouter", func() {
 		Expect(len(memSink.Messages)).To(Equal(0))
 	})
 
-	It("Route ignore app", func() {
-		noCache.SetIgnoreApp(true)
-		eventType = events.Envelope_LogMessage
-		err := r.Route(msg)
-		Ω(err).ShouldNot(HaveOccurred())
-		Expect(len(memSink.Events)).To(Equal(0))
-		Expect(len(memSink.Messages)).To(Equal(0))
-	})
-
 	It("Route sink error", func() {
 		memSink.ReturnErr = true
 		eventType = events.Envelope_LogMessage

events/events.go

@@ -18,6 +18,23 @@ type Event struct {
 	Type   string
 }
 
+type Config struct {
+	SelectedEvents string
+	AddAppName     bool
+	AddOrgName     bool
+	AddOrgGuid     bool
+	AddSpaceName   bool
+	AddSpaceGuid   bool
+}
+
+var AppMetadata = []string{
+	"AppName",
+	"OrgName",
+	"OrgGuid",
+	"SpaceName",
+	"SpaceGuid",
+}
+
 func HttpStartStop(msg *events.Envelope) *Event {
 	httpStartStop := msg.GetHttpStartStop()
 
@@ -125,7 +142,7 @@ func ContainerMetric(msg *events.Envelope) *Event {
 	}
 }
 
-func (e *Event) AnnotateWithAppData(appCache cache.Cache) {
+func (e *Event) AnnotateWithAppData(appCache cache.Cache, config *Config) {
 	cf_app_id := e.Fields["cf_app_id"]
 	appGuid := fmt.Sprintf("%s", cf_app_id)
 
@@ -142,39 +159,40 @@ func (e *Event) AnnotateWithAppData(appCache cache.Cache) {
 		cf_space_name := appInfo.SpaceName
 		cf_org_id := appInfo.OrgGuid
 		cf_org_name := appInfo.OrgName
-		cf_ignored_app := appInfo.IgnoredApp
+		//cf_ignored_app := appInfo.IgnoredApp
 		app_env := appInfo.CfAppEnv
 
-		if cf_app_name != "" {
+		if cf_app_name != "" && config.AddAppName {
 			e.Fields["cf_app_name"] = cf_app_name
 		}
 
-		if cf_space_id != "" {
+		if cf_space_id != "" && config.AddSpaceGuid {
 			e.Fields["cf_space_id"] = cf_space_id
 		}
 
-		if cf_space_name != "" {
+		if cf_space_name != "" && config.AddSpaceName {
 			e.Fields["cf_space_name"] = cf_space_name
 		}
 
-		if cf_org_id != "" {
+		if cf_org_id != "" && config.AddOrgGuid {
 			e.Fields["cf_org_id"] = cf_org_id
 		}
 
-		if cf_org_name != "" {
+		if cf_org_name != "" && config.AddOrgName {
 			e.Fields["cf_org_name"] = cf_org_name
 		}
 
 		if app_env["SPLUNK_INDEX"] != nil {
 			e.Fields["info_splunk_index"] = app_env["SPLUNK_INDEX"]
 		}
-
-		e.Fields["cf_ignored_app"] = cf_ignored_app
+		//removing cf_ignored_app as per INGEST-17639
+		//e.Fields["cf_ignored_app"] = cf_ignored_app
 	}
 }
 
 func (e *Event) AnnotateWithCFMetaData() {
-	e.Fields["cf_origin"] = "firehose"
+	//removing cf_origin as per INGEST-17639
+	//e.Fields["cf_origin"] = "firehose"
 	e.Fields["event_type"] = e.Type
 }
 
@@ -248,3 +266,7 @@ func ParseExtraFields(extraEventsString string) (map[string]string, error) {
 	}
 	return extraEvents, nil
 }
+
+func AuthorizedMetadata() string {
+	return strings.Join(AppMetadata, ", ")
+}