diff --git a/.fossa.yml b/.fossa.yml index 135ddc23..b36b7b03 100644 --- a/.fossa.yml +++ b/.fossa.yml @@ -2,4 +2,8 @@ version: 3 server: https://app.fossa.com project: id: "splunk-firehose-nozzle" - team: "TA-Automation" \ No newline at end of file + team: "TA-Automation" + +paths: + exclude: + - testing \ No newline at end of file diff --git a/.github/pre-functional-test.sh b/.github/pre-functional-test.sh index 099f54ed..c4a9b01d 100755 --- a/.github/pre-functional-test.sh +++ b/.github/pre-functional-test.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -sudo apt-get install python3.7 -sudo apt-get install python3-pip cd testing/integration pip3 install virtualenv virtualenv venv diff --git a/.github/pre-req.sh b/.github/pre-req.sh index bec30214..4692815e 100755 --- a/.github/pre-req.sh +++ b/.github/pre-req.sh @@ -10,7 +10,9 @@ sudo apt-get update sudo apt-get install apt-transport-https ca-certificates sudo apt-get install cf-cli #CF Login -cf login --skip-ssl-validation -a $API_ENDPOINT -u $API_USER -p $API_PASSWORD -o system -s system +API_PASSWORD_DECRYPTED=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DECRYPTED" + #Create splunk-ci org and space if [ "`cf o | grep "splunk-ci-org"`" == "splunk-ci-org" ]; then echo "splunk-ci-org org already exists" @@ -21,4 +23,13 @@ else cf target -o splunk-ci-org cf create-space splunk-ci-space cf target -o "splunk-ci-org" -s "splunk-ci-space" +fi + +gem install cf-uaac +uaac target "$API_UAA_ENDPOINT" --skip-ssl-validation +API_CLIENT_PASSWORD_DECRYPTED=$(echo "$API_CLIENT_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +uaac token client get "$API_USER" -s "$API_CLIENT_PASSWORD_DECRYPTED" + +if [ $(uaac client get "$CLIENT_ID" | grep -woc "$CLIENT_ID") -eq 0 ]; then + uaac client add "$CLIENT_ID" --name splunk-firehose --secret "$CLIENT_SECRET" --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only fi \ No newline at end of file diff --git a/.github/tile-builder.sh b/.github/tile-builder.sh index f71c2e90..507c9cec 100755 --- a/.github/tile-builder.sh +++ b/.github/tile-builder.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash echo "start" set -e -wget https://github.com/cf-platform-eng/tile-generator/releases/download/v13.0.2/pcf_linux-64bit > /dev/null 2>&1 +wget https://github.com/cf-platform-eng/tile-generator/releases/download/v15.0.2/pcf_linux-64bit > /dev/null 2>&1 chmod +x pcf_linux-64bit sudo mv pcf_linux-64bit /usr/local/bin/tile python3 -m venv tile-generator-env > /dev/null 2>&1 @@ -9,6 +9,7 @@ source tile-generator-env/bin/activate echo "Installing tile-generator..." pip install wheel pip install jinja2==3.0.3 +pip install setuptools pip install tile-generator cd tile echo "Installing bosh..." diff --git a/.github/update_manifest.sh b/.github/update_manifest.sh index 7d3baaa0..f144062f 100755 --- a/.github/update_manifest.sh +++ b/.github/update_manifest.sh @@ -2,10 +2,12 @@ set -e #Set below params in github env variable settings # API_ENDPOINT, API_USER, API_PASSWORD, SPLUNK_TOKEN, SPLUNK_HOST, SPLUNK_INDEX, SPLUNK_METRIC_INDEX +API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") +echo "$API_PASSWORD_DEC" #Update manifest for deployment sed -i 's@API_ENDPOINT:.*@'"API_ENDPOINT: $API_ENDPOINT"'@' scripts/ci_nozzle_manifest.yml sed -i 's@API_USER:.*@'"API_USER: $API_USER"'@' scripts/ci_nozzle_manifest.yml -sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD"'@' scripts/ci_nozzle_manifest.yml +sed -i 's@API_PASSWORD:.*@'"API_PASSWORD: $API_PASSWORD_DEC"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_ID:.*@'"CLIENT_ID: $CLIENT_ID"'@' scripts/ci_nozzle_manifest.yml sed -i 's@CLIENT_SECRET:.*@'"CLIENT_SECRET: $CLIENT_SECRET"'@' scripts/ci_nozzle_manifest.yml sed -i 's@SPLUNK_HOST:.*@'"SPLUNK_HOST: $SPLUNK_HOST"'@' scripts/ci_nozzle_manifest.yml diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 00000000..499231ab --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,33 @@ +name: ci +on: + push: + branches: + - main + - develop + tags: + - "v*" + +permissions: + contents: write + +jobs: + publish-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Install dependencies + run: | + pip install mkdocs-material + pip install mike + - name: Setup git user + run: | + git config --global user.name ${{github.actor}} + git config --global user.email ${{ github.actor }}@users.noreply.github.com + - name: Build docs website + run: mike deploy --push ${GITHUB_REF##*/} \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51f1a84e..d838dee7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,11 +2,15 @@ name: PCF on: push: branches: [ "develop" ] - pull_request_target: + pull_request: branches: [ "develop" ] - + workflow_dispatch: +env: + GO_VERSION: '1.18.1' + RUBY_VERSION: '3.3' + jobs: workflow_approval: name: Approve workflow @@ -35,7 +39,7 @@ jobs: env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - name: upload THIRDPARTY file - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: THIRDPARTY path: /tmp/THIRDPARTY @@ -44,7 +48,7 @@ jobs: fossa test --debug env: FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }} - + semgrep: runs-on: ubuntu-latest needs: @@ -59,10 +63,74 @@ jobs: with: publishToken: ${{ secrets.SEMGREP_PUBLISH_TOKEN }} + create-env: + needs: + - workflow_approval + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + API_ENDPOINT: ${{ steps.get-credentials.outputs.API_ENDPOINT }} + API_UAA_ENDPOINT: ${{ steps.get-credentials.outputs.API_UAA_ENDPOINT }} + API_PASSWORD: ${{ steps.get-credentials.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ steps.get-credentials.outputs.API_CLIENT_PASSWORD }} + ENV_ID: ${{ steps.get-credentials.outputs.ENV_ID }} + steps: + - name: Install Selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: Get authorization token + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + echo "::add-mask::$AUTH_TOKEN" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: Claim environment and wait + run: | + echo ENV_ID="$(./selfservice claimAndWait isv_ci_tas_lite_4_0 | jq -r '.id')" >> "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Set up Brew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install hammer and cf + run: | + brew tap pivotal/hammer https://github.com/pivotal/hammer + brew install hammer + + brew tap pivotal-cf/om https://github.com/pivotal-cf/om + brew install om + brew install cloudfoundry/tap/cf-cli + + - name: Credentials + id: get-credentials + run: | + ./selfservice get $ENV_ID | jq -r '.credentials' > env.json + echo API_ENDPOINT="https://api.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + echo API_UAA_ENDPOINT="https://uaa.$(jq -r '.sys_domain' ./env.json)" >> "$GITHUB_OUTPUT" + API_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_credentials | jq -r '.password')" + API_PASSWORD_ENCRYPTED="$(echo $API_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_PASSWORD=$API_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" + API_CLIENT_PASSWORD="$(hammer -t ./env.json om credentials -- -p cf -t json -c .uaa.admin_client_credentials | jq -r '.password')" + API_CLIENT_PASSWORD_ENCRYPTED="$(echo $API_CLIENT_PASSWORD | openssl aes-256-cbc -a -pbkdf2 -salt -pass pass:$ENCRYPT_KEY)" + echo API_CLIENT_PASSWORD=$API_CLIENT_PASSWORD_ENCRYPTED >> "$GITHUB_OUTPUT" + echo ENV_ID=$ENV_ID >> "$GITHUB_OUTPUT" + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + + build: runs-on: ubuntu-latest needs: - - workflow_approval + - create-env + steps: - name: Checkout uses: actions/checkout@v3 @@ -71,7 +139,7 @@ jobs: repository: ${{github.event.pull_request.head.repo.full_name}} - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version # Install Dependencies @@ -79,7 +147,7 @@ jobs: run: ls -la - name: Install Dependencies run: | - go mod vendor + go mod vendor # Builder - name: Builder @@ -90,16 +158,17 @@ jobs: run: | make testall - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: splunk-firehose-nozzle path: splunk-firehose-nozzle deploy_nozzle: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -108,43 +177,57 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - needs: build + needs: + - build + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_VERSION }} + - run: ruby -v - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: splunk-firehose-nozzle # Install dependencies - name: Install dependencies run: | - go mod vendor + go mod vendor # Deploy nozzle - name: Deploy nozzle run: | - .github/update_manifest.sh - .github/pre-req.sh - cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + .github/update_manifest.sh + .github/pre-req.sh + cf push -f scripts/ci_nozzle_manifest.yml -u process --random-route + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + # Nozzle Log + - name: Nozzle Log + run: | + cf logs splunk-firehose-nozzle & + tile-builder: - - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -153,45 +236,54 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} SPLUNK_METRIC_INDEX: ${{ secrets.SPLUNK_METRIC_INDEX }} - - needs: deploy_nozzle + needs: + - deploy_nozzle + - create-env runs-on: ubuntu-latest steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version - - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: splunk-firehose-nozzle - + # Tile builder - name: Tile builder run: | - .github/tile-builder.sh - - # Nozzle Log - - name: Nozzle Log + .github/tile-builder.sh + + - name: Get tile name run: | - cf logs splunk-firehose-nozzle & + echo "tile_name=$(ls tile/product | grep ".pivotal")" >> "$GITHUB_ENV" + + - name: Upload tile + uses: actions/upload-artifact@v4 + with: + name: ${{ env.tile_name }} + path: tile/product/${{ env.tile_name }} + # Skip test for now! execute_tests: - - needs: tile-builder + needs: + - tile-builder + - create-env runs-on: ubuntu-latest - env: - API_ENDPOINT: ${{ secrets.API_ENDPOINT }} - API_PASSWORD: ${{ secrets.API_PASSWORD }} + env: + API_ENDPOINT: ${{ needs.create-env.outputs.API_ENDPOINT }} + API_PASSWORD: ${{ needs.create-env.outputs.API_PASSWORD }} + API_CLIENT_PASSWORD: ${{ needs.create-env.outputs.API_CLIENT_PASSWORD }} + API_UAA_ENDPOINT: ${{ needs.create-env.outputs.API_UAA_ENDPOINT }} API_USER: ${{ secrets.API_USER }} CLIENT_ID: ${{ secrets.CLIENT_ID }} CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} @@ -204,58 +296,98 @@ jobs: SPLUNK_HOST: ${{ secrets.SPLUNK_HOST }} steps: - - name: Checkout uses: actions/checkout@v3 with: ref: ${{github.event.pull_request.head.sha}} repository: ${{github.event.pull_request.head.repo.full_name}} + + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - uses: actions/setup-go@v3 with: - go-version: '1.17.0' + go-version: ${{ env.GO_VERSION }} - run: go version + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_VERSION }} + - run: ruby -v - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: splunk-firehose-nozzle - # Install dependencies - name: Install dependencies run: | go mod vendor chmod +x splunk-firehose-nozzle - + # Deploy data-gen - name: Deploy data-gen run: | .github/pre-req.sh cf push -f scripts/data_gen_manifest.yml -u process -p tools/data_gen --random-route sleep 10 - + cf apps + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} # Nozzle Log - name: Nozzle Log run: | - cf logs splunk-firehose-nozzle & + cf logs splunk-firehose-nozzle & # Prepare test environment - name: Prepare test environment run: | .github/pre-functional-test.sh - # Executing tests - name: Executing tests - run: | - .github/functional-test.sh - + run: | + .github/functional-test.sh + # Teardown - name: Teardown if: always() run: | - echo "Teardown deployment env" - cf delete splunk-firehose-nozzle -f - cf delete data_gen -f - cf delete-org splunk-ci-org -f \ No newline at end of file + API_PASSWORD_DEC=$(echo "$API_PASSWORD" | openssl aes-256-cbc -d -pbkdf2 -a -pass pass:"$ENCRYPT_KEY") + cf login --skip-ssl-validation -a "$API_ENDPOINT" -u "$API_USER" -p "$API_PASSWORD_DEC" + echo "Teardown deployment env" + cf target -o "splunk-ci-org" -s "splunk-ci-space" + cf delete splunk-firehose-nozzle -f + cf delete data_gen -f + cf delete-org splunk-ci-org -f + env: + ENCRYPT_KEY: ${{ secrets.ENCRYPT_KEY }} + + + teardown-env: + runs-on: ubuntu-latest + needs: + - create-env + - execute_tests + if: always() + env: + ENV_ID: ${{ needs.create-env.outputs.ENV_ID }} + steps: + - name: install-selfservice + run: | + wget https://github.com/cf-platform-eng/selfservice/releases/download/0.2.9/selfservice + chmod +x selfservice + ls -latr + + - name: auth + run: | + echo "$(./selfservice auth $API_TOKEN | cut -c 8-)" >> "$GITHUB_ENV" + env: + API_TOKEN: ${{ secrets.API_TOKEN }} + + - name: release environment + run: | + ./selfservice release $ENV_ID \ No newline at end of file diff --git a/.gitignore b/.gitignore index a1accbbe..607d724e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ testing/manual_perf/data_gen_manifest.yml testing/integration/config/env.json testing/integration/config/local.ini +site \ No newline at end of file diff --git a/README.md b/README.md index f4cd892a..a8344681 100644 --- a/README.md +++ b/README.md @@ -1,544 +1,21 @@ -## Splunk Nozzle +# Documentation -Cloud Foundry Firehose-to-Splunk Nozzle +For deployment and user documentation [see](https://cloudfoundry-community.github.io/splunk-firehose-nozzle/develop/). -- - - - +There are plenty of versions you can browse: +1. Branch `develop` refers to the latest version +2. Tagged versions (ex. `1.3.1`) refers to the specific release -### VMware Tanzu Application Service version -Splunk Firehose Nozzle has been tested on v3.0.0 and v4.0.0 of Tanzu Application Service +# Compatibility -- - - - +## VMware Tanzu Application Service versions -### VMware Ops Manager version -Splunk Firehose Nozzle has been tested on v3.0.9 LTS of VMware Ops Manager +Splunk Firehose Nozzle has been tested on `v3.0`, `v4.0`, `v5.0` and `v6.0` of Tanzu Application Service. -- - - - +## VMware Ops Manager version -### Usage -Splunk nozzle is used to stream Cloud Foundry Firehose events to Splunk HTTP Event Collector. Using pre-defined Splunk sourcetypes, the nozzle automatically parses the events and enriches them with additional metadata before forwarding to Splunk. For detailed descriptions of each Firehose event type and their fields, refer to underlying [dropsonde protocol](https://github.com/cloudfoundry/dropsonde-protocol). Below is a mapping of each Firehose event type to its corresponding Splunk sourcetype. Refer to [Searching Events](#searching-events) for example Splunk searches. - -| Firehose event type | Splunk sourcetype | Description -|---|---|--- -| Error | `cf:error` | An Error event represents an error in the originating process -| HttpStartStop | `cf:httpstartstop` | An HttpStartStop event represents the whole lifecycle of an HTTP request -| LogMessage | `cf:logmessage` | A LogMessage contains a "log line" and associated metadata -| ContainerMetric | `cf:containermetric` | A ContainerMetric records resource usage of an app in a container -| CounterEvent | `cf:counterevent` | A CounterEvent represents the increment of a counter -| ValueMetric | `cf:valuemetric` | A ValueMetric indicates the value of a metric at an instant in time - -In addition, logs from the nozzle itself are of sourcetype `cf:splunknozzle`. - -### Setup - -The Nozzle requires a client with the authorities `doppler.firehose` and `cloud_controller.admin_read_only` (the latter is only required if `ADD_APP_INFO` is enabled) and grant-types `client_credentials` and `refresh_token`. If `cloud_controller.admin_read_only` is not -available in the system, switch to use `cloud_controller.admin`. - -You can either -* Add the client manually using [uaac](https://github.com/cloudfoundry/cf-uaac) -* Add the client to the deployment manifest; see [uaa.scim.users](https://github.com/cloudfoundry/uaa-release/blob/master/jobs/uaa/spec) - -Manifest example: - -```yaml - -# Clients -uaa.clients: - splunk-firehose: - id: splunk-firehose - override: true - secret: splunk-firehose-secret - authorized-grant-types: client_credentials,refresh_token - authorities: doppler.firehose,cloud_controller.admin_read_only -``` - -`uaac` example: -```shell -uaac target https://uaa.[system domain url] -uaac token client get admin -s [admin client credentials secret] -uaac client add splunk-firehose --name splunk-firehose -uaac client add splunk-firehose --secret [your_client_secret] -uaac client add splunk-firehose --authorized_grant_types client_credentials,refresh_token -uaac client add splunk-firehose --authorities doppler.firehose,cloud_controller.admin_read_only - -``` - -`cloud_controller.admin_read_only` will work for cf v241 -or later. Earlier versions should use `cloud_controller.admin` instead. - -- - - - -#### Environment Parameters -You can declare parameters by making a copy of the scripts/nozzle.sh.template. -* `DEBUG`: Enable debug mode (forward to standard out instead of Splunk). (Default: false). - -__Cloud Foundry configuration parameters:__ -* `API_ENDPOINT`: Cloud Foundry API endpoint address. It is required parameter. -* `CLIENT_ID`: UAA Client ID (Must have authorities and grant_types described above). It is required parameter. -* `CLIENT_SECRET`: Secret for Client ID. It is required parameter. - -__Splunk configuration parameters:__ -* `SPLUNK_TOKEN`: [Splunk HTTP event collector token](http://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector/). It is required parameter. -* `SPLUNK_HOST`: Splunk HTTP event collector host. example: https://example.cloud.splunk.com:8088. It is required parameter. -* `SPLUNK_INDEX`: The Splunk index events will be sent to. Warning: Setting an invalid index will cause events to be lost. This index must match one of the selected indexes for the Splunk HTTP event collector token used for the SPLUNK_TOKEN parameter. It is required parameter. - -__Advanced Configuration Features:__ -* `JOB_NAME`: Tags nozzle log events with job name. It is optional. (Default: 'splunk-nozzle') -* `JOB_INDEX`: Tags nozzle log events with job index. (Default: -1) -* `JOB_HOST`: Tags nozzle log events with job host. (Default: "") -* `SKIP_SSL_VALIDATION_CF`: Skips SSL certificate validation for connection to Cloud Foundry. Secure communications will not check SSL certificates against a trusted certificate authority. -This is recommended for dev environments only. (Default: false) -* `SKIP_SSL_VALIDATION_SPLUNK`: Skips SSL certificate validation for connection to Splunk. Secure communications will not check SSL certificates against a trusted certificate authority. (Default: false) -This is recommended for dev environments only. -* `FIREHOSE_SUBSCRIPTION_ID`: Tags nozzle events with a Firehose subscription id. See https://docs.pivotal.io/pivotalcf/1-11/loggregator/log-ops-guide.html. (Default: splunk-firehose) -* `FIREHOSE_KEEP_ALIVE`: Keep alive duration for the Firehose consumer. (Default: 25s) -* `ADD_APP_INFO`: Enrich raw data with app info. A comma separated list of app metadata (AppName,OrgName,OrgGuid,SpaceName,SpaceGuid). (Default: "") -* `ADD_TAGS`: Add additional tags from envelope to splunk event. (Default: false) - (Please note: Adding tags / Enabling this feature may slightly impact the performance due to the increased event size) -* `IGNORE_MISSING_APP`: If the application is missing, then stop repeatedly querying application info from Cloud Foundry. (Default: true) -* `MISSING_APP_CACHE_INVALIDATE_TTL`: How frequently the missing app info cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). (Default: 0s) (see below for more details) -* `APP_CACHE_INVALIDATE_TTL`: How frequently the app info local cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). (Default: 0s) (see below for more details) -* `ORG_SPACE_CACHE_INVALIDATE_TTL`: How frequently the org and space cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). (Default: 72h) -* `APP_LIMITS`: Restrict to APP_LIMITS the most updated apps per request when populating the app metadata cache. keep it 0 to update all the apps. (Default: 0) -* `BOLTDB_PATH`: Bolt database path. (Default: cache.db) -* `EVENTS`: A comma separated list of events to include. It is a required field. Possible values: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric. If no eventtype is selected, nozzle will automatically select LogMessage to keep the nozzle running. (Default: "ValueMetric,CounterEvent,ContainerMetric") -* `EXTRA_FIELDS`: Extra fields to annotate your events with (format is key:value,key:value). (Default: "") -* `FLUSH_INTERVAL`: Time interval (in s/m/h. For example, 3600s or 60m or 1h) for flushing queue to Splunk regardless of CONSUMER_QUEUE_SIZE. Protects against stale events in low throughput systems. (Default: 5s) -* `CONSUMER_QUEUE_SIZE`: Sets the internal consumer queue buffer size. Events will be pushed to Splunk after queue is full. (Default: 10000) -* `HEC_BATCH_SIZE`: Set the batch size for the events to push to HEC (Splunk HTTP Event Collector). (Default: 100) -* `HEC_RETRIES`: Retry count for sending events to Splunk. After expiring, events will begin dropping causing data loss. (Default: 5) -* `HEC_WORKERS`: Set the amount of Splunk HEC workers to increase concurrency while ingesting in Splunk. (Default: 8) -* `ENABLE_EVENT_TRACING`: Enables event trace logging. Splunk events will now contain a UUID, Splunk Nozzle Event Counts, and a Subscription-ID for Splunk correlation searches. (Default: false) -* `SPLUNK_LOGGING_INDEX`: The Splunk index where logs from the nozzle of the sourcetype `cf:splunknozzle` will be sent to. Warning: Setting an invalid index will cause events to be lost. This index must match one of the selected indexes for the Splunk HTTP event collector token used for the SPLUNK_TOKEN parameter. When not provided, all logging events will be forwarded to the default SPLUNK_INDEX. The default value is `""` -* `STATUS_MONITOR_INTERVAL`: Time interval (in s/m/h. For example, 3600s or 60m or 1h) for Enabling Monitoring (Metric data of insights with in the connectors). Default is 0s (Disabled). -* `SPLUNK_METRIC_INDEX`: Index in which metric data will be ingested when monitoring module is enabled -* `SELECTED_MONITORING_METRICS`: Name of the metrics that you want to monitor and add using comma seprated values. List of the metrics that are supported in the metrics modules are given below - -__About app cache params:__ - -When ADD_APP_INFO config is enabled, the nozzle will enrich the event with app metadata. For this, the nozzle maintains a cache of all the apps locally so that it doesn’t need to query from remote every time. - -Now, when there is a change in this app data in remote, the nozzle has to update this local cache. For this, the config has APP_CACHE_INVALIDATE_TTL parameter. At every APP_CACHE_INVALIDATE_TTL interval, the nozzle will update the local cache by querying the remote (CF APIs). - -If APP_CACHE_INVALIDATE_TTL is set to 10s, the nozzle will refresh the local cache at every 10s. So, AppCacheTTL should be set based on how frequently the app data is expected to change. - -When the nozzle receives events from the doppler, it will check the local cache for the given app-id. But on cache-miss, it will query remote for that specific app. If it doesn’t find the app data from remote too, then the nozzle will add that app to MissingAppCache (if IGNORE_MISSING_APP config is **enabled**. so that the nozzle does not waste time in querying the remote for an app which is likely not to be found). So, from the next time onwards, the nozzle will first check in the MissingAppCache, if found then it will ignore the app and move on to the next event with a warning. - -MISSING_APP_CACHE_INVALIDATE_TTL is used to clear the MissingAppCache so nozzle can retry querying from remote. - -For example, given MISSING_APP_CACHE_INVALIDATE_TTL is set to 60s, when nozzle receives event from app that is not available in local cache and remote, it’ll add it to MissingAppCache. Until next MISSING_APP_CACHE_INVALIDATE_TTL, nozzle will not query from remote for the missing app. - -- - - - - -### Push as an App to Cloud Foundry - -Push Splunk Firehose Nozzle as an application to Cloud Foundry. Please refer to **Setup** section for details -on user authentication. - -1. Download the latest release - - ```shell - git clone https://github.com/cloudfoundry-community/splunk-firehose-nozzle.git - cd splunk-firehose-nozzle - ``` - -2. Authenticate to Cloud Foundry - - ```shell - cf login -a https://api.[your cf system domain] -u [your id] - ``` - -3. Copy the manifest template and fill in needed values (using the credentials created during setup) - - ```shell - vim scripts/ci_nozzle_manifest.yml - ``` - -4. Push the nozzle - - ```shell - make deploy-nozzle - ``` - -#### Dump application info to boltdb #### -If in production where there are lots of CF applications (say tens of thousands) and if the user would like to enrich -application logs by including application metadata, querying all application metadata information from CF may take some time - -for example if we include: add app name, space ID, space name, org ID and org name to the events. -If there are multiple instances of Spunk nozzle deployed the situation will be even worse, since each of the Splunk nozzle(s) will query all applications meta data and -cache the metadata information to the local boltdb file. These queries will introduce load to the CF system and could potentially take a long time to finish. -Users can run this tool to generate a copy of all application metadata and copy this to each Splunk nozzle deployment. Each Splunk nozzle can pick up the cache copy and update the cache file incrementally afterwards. - -Example of how to run the dump application info tool: - -``` -$ cd tools/dump_app_info -$ go build dump_app_info.go -$ ./dump_app_info --skip-ssl-validation --api-endpoint=https:// --user= --password= -``` - -After populating the application info cache file, user can copy to different Splunk nozzle deployments and start Splunk nozzle to pick up this cache file by -specifying correct "--boltdb-path" flag or "BOLTDB_PATH" environment variable. - -### Disable logging for noisy applications -Set F2S_DISABLE_LOGGING = true as a environment variable in applications's manifest to disable logging. - - -## Index routing -Index routing is a feature that can be used to send different Cloud Foundry logs to different indexes for better ACL and data retention control in Splunk. - -### Per application index routing via application manifest -To enable per app index routing, -* Please set environment variable `SPLUNK_INDEX` in your application's manifest ([example below](#example-manifest-file)) -* Make sure Splunk nozzle is configured with `ADD_APP_INFO` (Select at least one of AppName,OrgName,OrgGuid,SpaceName,SpaceGuid) to enable app info caching -* Make sure `SPLUNK_INDEX` specified in app's manifest exist in Splunk and can receive data for the configured Splunk HEC token. - -> **WARNING**: If `SPLUNK_INDEX` is invalid, events from other apps may also get lost as splunk will drop entire event batch if any of the event from batch is invalid (i.e. invalid index) - -There are two ways to set the variable: - -In your app manifest provide an environment variable called `SPLUNK_INDEX` and assign it the index you would like to send the app data to. - -#### Example Manifest file -``` -applications: -- name: - memory: 256M - disk_quota: 256M - ... - env: - SPLUNK_INDEX: - ... -``` - -You can also update the env on the fly using cf-cli command: -``` -cf set-env SPLUNK_INDEX -``` -#### Please note -> If you are updating env on the fly, make sure that `APP_CACHE_INVALIDATE_TTL` is greater tha 0s. Otherwise cached app-info will not be updated and events will not be sent to required index. - - -### Index routing via Splunk configuration -Logs can be routed using fields such as app ID/name, space ID/name or org ID/name. -Users can configure the Splunk configuration files props.conf and transforms.conf on Splunk indexers or Splunk Heavy Forwarders if deployed. - -Below are few sample configuration: - -1. Route data from application ID `95930b4e-c16c-478e-8ded-5c6e9c5981f8` to a Splunk `prod` index: - -*$SPLUNK_HOME/etc/system/local/props.conf* -``` -[cf:logmessage] -TRANSFORMS-index_routing = route_data_to_index_by_field_cf_app_id -``` - - -*$SPLUNK_HOME/etc/system/local/transforms.conf* -``` -[route_data_to_index_by_field_cf_app_id] -REGEX = "(\w+)":"95930b4e-c16c-478e-8ded-5c6e9c5981f8" -DEST_KEY = _MetaData:Index -FORMAT = prod -``` - - -2. Routing application logs from any Cloud Foundry orgs whose names are prefixed with `sales` to a Splunk `sales` index. - -*$SPLUNK_HOME/etc/system/local/props.conf* -``` -[cf:logmessage] -TRANSFORMS-index_routing = route_data_to_index_by_field_cf_org_name - -``` - -*$SPLUNK_HOME/etc/system/local/transforms.conf* -``` -[route_data_to_index_by_field_cf_org_name] -REGEX = "cf_org_name":"(sales.*)" -DEST_KEY = _MetaData:Index -FORMAT = sales -``` - -3. Routing data from sourcetype `cf:splunknozzle` to index `new_index`: - -*$SPLUNK_HOME/etc/system/local/props.conf* -``` -[cf:splunknozzle] -TRANSFORMS-route_to_new_index = route_to_new_index -``` - -*$SPLUNK_HOME/etc/system/local/transforms.conf* -``` -[route_to_new_index] -SOURCE_KEY = MetaData:Sourcetype -DEST_KEY =_MetaData:Index -REGEX = (sourcetype::cf:splunknozzle) -FORMAT = new_index -``` -

Note:Moving from version 1.2.4 to 1.2.5, timestamp will use nanosecond precision instead of milliseconds.

- - -__Monitoring(Metric data Ingestion):__ - -| Metric Name | Description -|---|--- -| `nozzle.queue.percentage` | Shows how much internal queue is filled -| `splunk.events.dropped.count` | Number of events dropped from splunk HEC -| `splunk.events.sent.count` | Number of events sent to splunk -| `firehose.events.dropped.count` | Number of events dropped from nozzle -| `firehose.events.received.count` | Number of events received from firehose(websocket) -| `splunk.events.throughput` | Average Payload size -| `nozzle.usage.ram` | RAM Usage -| `nozzle.usage.cpu` | CPU Usage -| `nozzle.cache.memory.hit` | How many times it has successfully retrieved the data from memory -| `nozzle.cache.memory.miss` | How many times it has unsuccessfully tried to retreive the data from memory -| `nozzle.cache.remote.hit` | How many times it has successfully retrieved the data from remote -| `nozzle.cache.remote.miss` | How many times it has unsuccessfully tried to retrieve the data from remote -| `nozzle.cache.boltdb.hit` | How many times it has successfully retrieved the data from BoltDB -| `nozzle.cache.boltdb.miss` | How many times it has unsuccessfully tried to retrieve the data from BoltDB - -![event_count](https://user-images.githubusercontent.com/89519924/200804220-1adff84c-e6f1-4438-8d30-6e2cce4984f5.png) - -![nozzle_logs](https://user-images.githubusercontent.com/89519924/200804285-22ad7863-1db3-493a-8196-cc589837db76.png) - -

Note:Select value Rate(Avg) for Aggregation from Analysis tab on the top right.

- -## Troubleshooting -This topic describes how to troubleshoot Splunk Firehose Nozzle for Cloud Foundry. - -#### 1. I can't find my data! - - Are you searching for events and not finding them or looking at a dashboard and seeing "No result found"? Check Splunk Nozzle app logs. - - To view the nozzle's logs running on CF do the following: - -
    -
  1. Log in as an admin via the CLI.
    2. -
  2. Target the org created by the tile.
    - 
    cf target -o SPLUNK-NOZZLE-ORG
    -
    3. -
  3. View the recent app Splunk Nozzle logs (the version number installed by the tile will vary).
    - 
    cf logs --recent splunk-firehoze-nozzle
    -
    4. -
  4. Alternatively, you can stream the app logs as they're emitted.
    - 
    cf logs splunk-firehose-nozzle
    -
    5. -
- - -### Here are a few common errors and possible resolutions: - -#### Splunk configuration related errors: - -
{"timestamp":"
- -This error usually occurs when SSL is enabled on the Splunk HEC endpoint. Confirm that you're using https' in the Splunk HEC URL. - - -
{"timestamp":"
- -This usually means the index value specified in the configuration doesn't exist on Splunk Host. Confirm that you're using the correct Splunk index value. - - -
{"timestamp":"
- -This can occur when the Splunk HEC Token value is invalid. Confirm that you're using a valid token. - -
{"timestamp":"
- -This usually means that there was no valid SSL certificate found. Confirm that you're using a valid SSL certificate for the Splunk server, or set 'Skip SSL Validation' to `true` under Splunk settings. - -

Note:Disabling SSL validation is not recommended for production environments.

- - -
{"timestamp":"
- -This error can occur when the Splunk server is offline or when the Splunk HEC URL is not valid. Confirm that both the Splunk server is running and that you're using a valid URL. - -#### Cloud Foundry configuration related errors: - -
{"timestamp":"
- -This error can occur when the credentials provided for CF environment are invalid. Confirm that the API User and API Password each have access to the CF environment. - - -
{"timestamp":"
- -This means that no valid SSL certificate was found. To remediate this error, provide a valid SSL certificate for Cloud Foundry or set 'Skip SSL Validation' to true under Cloud Foundry Settings. - -

Note:Disabling SSL validation is not recommended for production environments.

- -The following troubleshooting tips assume you have access to Splunk to run basic searches against index `_internal` and the user-specified index for Firehose events. - -### 2. Ensure Splunk Nozzle is forwarding events from the Firehose: - -Search app logs of the Nozzle to confirm correct behavior: - -
-sourcetype="cf:splunknozzle"
-
- -A correct setup logs a start message with configuration parameters of the Nozzle logged as a JSON object, for example: - -
-  data:	{
-     add-app-info: AppName,OrgName,OrgGuid,SpaceName,SpaceGuid
-     api-endpoint: https://api.endpoint.com
-     app-cache-ttl: 0
-     app-limits: 0
-     batch-size: 1000
-     boltdb-path: cache.db
-     branch: null
-     buildos: null
-     commit: null
-     debug:	 false
-     extra-fields:
-     flush-interval: 5000000000
-     hec-workers: 8
-     ignore-missing-apps: true
-     job-host:
-     job-index: -1
-     job-name: splunk-nozzle
-     keep-alive: 25000000000
-     missing-app-cache-ttl:	 0
-     queue-size: 10000
-     retries: 2
-     skip-ssl: true
-     splunk-host: http://localhost:8088
-     splunk-index: atomic
-     subscription-id: splunk-firehose
-     trace-logging: true
-     status-monitor-interval: 0s
-     version:
-     wanted-events: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric
-  }
-  ip: 10.0.0.0
-  log_level: 1
-  logger_source: splunk-nozzle-logger
-  message: splunk-nozzle-logger.Running splunk-firehose-nozzle with following configuration variables
-  origin: splunk_nozzle
-
- - Search app logs of the Nozzle for any errors: -
-sourcetype="cf:splunknozzle" data.error=*
-
- -Errors are logged with corresponding message and stacktrace. - -### 3. Check for dropped events due to HTTP Event Collector availability: - -As the Splunk Firehose Nozzle sends data to Splunk via HTTPS using the HTTP Event Collector, it is also susceptible to any network issues across the network path from point to point. Run the following search to determine if Splunk has indexed any events indicating issues with the HEC Endpoint. - -
-  sourcetype="cf:splunknozzle" "dropping events"
-
- -### 4. Check for dropped events due to slow downstream(Network/Splunk): - -If the nozzle emits the ‘dropped events’ warning saying that downstream is slow, then the network or Splunk environment might needs to be scaled. (eg. Splunk HEC receiver node, Splunk Indexer, LB etc) - -Run the following search to determine if Splunk has indexed any events indicating such issues. - -
-  sourcetype="cf:splunknozzle" "dropped Total of"
-
- -### 5. Check for data loss inside the Splunk Firehose Nozzle: - -If "Event Tracing" is enabled, extra metadata will be attached to events. This allows searches to calculate the percentage of data loss inside the Splunk Firehose Nozzle, if applicable. - -Each instance of the Splunk Firehose Nozzle will run with a randomly generated UUID. The query below will display the message success rate for each UUID (Please update the index value based on your nozzle configuration). - -
-index=main | stats count as total_events, min(nozzle-event-counter) as min_number, max(nozzle-event-counter) as max_number by uuid | eval event_number =  max_number - min_number | eval success_percentage = total_events/event_number*100 | stats max(success_percentage) by uuid
-
- -### 6. Authentication is not working even if correct CF Client ID/secret is configured: (applicable in v1.2.3) - -Due to a known issue in an indirect dependency (an OAuth library), if the client secret has any special characters (eg. *!#$&@^) then it will not work. For now, user has to configure a client secret without any of this characters. Once the library in question is updated in the next release it will work even with the special characters. - -#### Searching Events - -Here are two short Splunk queries to start exploring some of the Cloud Foundry events in Splunk. - -``` -sourcetype="cf:valuemetric" - | stats avg(value) by job_instance, name -``` - -``` -sourcetype="cf:counterevent" - | eval job_and_name=source+"-"+name - | stats values(job_and_name) -``` - -### 7. Nozzle is not collecting any data with 'websocket' (bad handshake) error - -If the nozzle reports below error, then check if the configured "subscription-id" has '#' as a prefix. Please remove the prefix or prepend any other character than '#' to fix this issue. -``` -Error dialing trafficcontroller server: websocket: bad handshake.\nPlease ask your Cloud Foundry Operator to check the platform configuration (trafficcontroller is wss://****:443). -``` - -### Development - -#### Software Requirements - -Make sure you have the following installed on your workstation: - -| Software | Version -| --- | --- | -| go | go1.17.x - -Then make sure that all dependent packages are there: - -``` -$ cd -$ make installdeps -``` - -#### Environment - -For development against [bosh-lite](https://github.com/cloudfoundry/bosh-lite), -copy `tools/nozzle.sh.template` to `tools/nozzle.sh` and supply missing values: - -``` -$ cp tools/nozzle.sh.template tools/nozzle.sh -$ chmod +x tools/nozzle.sh -``` - -Build project: - -``` -$ make VERSION=1.3.0 -``` - -Run tests with [Ginkgo](http://onsi.github.io/ginkgo/) - -``` -$ ginkgo -r -``` - -Run all kinds of testing - -``` -$ make test # run all unittest -$ make race # test if there is race condition in the code -$ make vet # examine GoLang code -$ make cov # code coverage test and code coverage html report -``` - -Or run all testings: unit test, race condition test, code coverage etc -``` -$ make testall -``` - -Run app - -``` -# this will run: go run main.go -$ ./tools/nozzle.sh -``` +Splunk Firehose Nozzle has been tested on `v3.0.9 LTS` of VMware Ops Manager # Maintenance And Support Splunk Firehose Nozzle project is supported through Splunk Support assuming the customer has a current Splunk support entitlement. For customers that do not have a current Splunk support entitlement, please file an issue at [create a new issue](https://github.com/cloudfoundry-community/splunk-firehose-nozzle/issues/new/choose) - diff --git a/cache/cache_easyjson.go b/cache/cache_easyjson.go index 5fa2fa54..31811f1e 100644 --- a/cache/cache_easyjson.go +++ b/cache/cache_easyjson.go @@ -4,6 +4,7 @@ package cache import ( json "encoding/json" + jlexer "github.com/mailru/easyjson/jlexer" jwriter "github.com/mailru/easyjson/jwriter" ) @@ -43,25 +44,7 @@ func easyjsonA591d1bcDecodeGithubComCloudfoundryCommunitySplunkFirehoseNozzleCac case "OrgGuid": out.OrgGuid = string(in.String()) case "CfAppEnv": - if in.IsNull() { - in.Skip() - } else { - in.Delim('{') - if !in.IsDelim('}') { - out.CfAppEnv = make(map[string]interface{}) - } else { - out.CfAppEnv = nil - } - for !in.IsDelim('}') { - key := string(in.String()) - in.WantColon() - var v1 interface{} - v1 = in.Interface() - (out.CfAppEnv)[key] = v1 - in.WantComma() - } - in.Delim('}') - } + parseCfAppEnv(in, out) case "IgnoredApp": out.IgnoredApp = bool(in.Bool()) default: @@ -71,6 +54,28 @@ func easyjsonA591d1bcDecodeGithubComCloudfoundryCommunitySplunkFirehoseNozzleCac } in.Delim('}') } + +func parseCfAppEnv(in *jlexer.Lexer, out *App) { + if in.IsNull() { + in.Skip() + } else { + in.Delim('{') + if !in.IsDelim('}') { + out.CfAppEnv = make(map[string]interface{}) + } else { + out.CfAppEnv = nil + } + for !in.IsDelim('}') { + key := string(in.String()) + in.WantColon() + v1 := in.Interface() + (out.CfAppEnv)[key] = v1 + in.WantComma() + } + in.Delim('}') + } +} + func easyjsonA591d1bcEncodeGithubComCloudfoundryCommunitySplunkFirehoseNozzleCache(out *jwriter.Writer, in App) { out.RawByte('{') first := true diff --git a/dashboards/dashboard.xml b/dashboards/dashboard.xml new file mode 100644 index 00000000..af5318e7 --- /dev/null +++ b/dashboards/dashboard.xml @@ -0,0 +1,263 @@ +
+ +
+ + + + -24h@h + now + + + + + * + * + + + + * + * + +
+ + + Event count + + + index=$index_token$ sourcetype="cf:*" | timechart span=10m count + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + + + Distributions of events by index + + + index=$index_token$ sourcetype="cf:*" | stats count by index + $main_time_range.earliest$ + $main_time_range.latest$ + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Total number of PCF events by index + + + index=$index_token$ sourcetype="cf:*" | chart sparkline(count) AS "Indexes Trend" count AS Total BY index + $main_time_range.earliest$ + $main_time_range.latest$ + 1 + + + + + + + + + +
+ + + + + Events by sources + + Events by source + + index=$index_token$ sourcetype="cf:*" | top source + $main_time_range.earliest$ + $main_time_range.latest$ + + + + + + + + + + + Events by sourcetype + + Events by sourcetype + + index=$index_token$ sourcetype="cf:*" | top sourcetype + $main_time_range.earliest$ + $main_time_range.latest$ + + + + + + + + + + + + + Internal Splunk errors count + + + index=_internal component=HttpInputDataHandler (log_level="WARN" OR log_level="ERROR") | timechart span=10m count + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + Internal Splunk errors sample + + + index=_internal component=HttpInputDataHandler (log_level="WARN" OR log_level="ERROR") | table name,parsing_err | stats count by name,parsing_err + $main_time_range.earliest$ + $main_time_range.latest$ + 1 + + + + + +
+ + + + + Nozzle errors count + + + index=$index_token$ sourcetype="cf:splunknozzle" message="*error*" | timechart span=10m count + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + Nozzle errors sample + + + index=$index_token$ sourcetype="cf:splunknozzle" message="*error*" | table source,logger_source,message | stats count by source,logger_source,message + $main_time_range.earliest$ + $main_time_range.latest$ + 1 + + + + + +
+ + + + + Nozzle CPU usage + + + | mstats avg("nozzle.usage.cpu") prestats=true WHERE "index"=$metrics_index_token$ span=10s | timechart avg("nozzle.usage.cpu") AS Avg span=10s | fields - _span* + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + + + Nozzle RAM usage + + + | mstats avg("nozzle.usage.ram") prestats=true WHERE "index"=$metrics_index_token$ span=10s | timechart avg("nozzle.usage.ram") AS Avg span=10s | fields - _span* + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + + + Nozzle events sent + + + | mstats rate_avg("firehose.events.received.count") as "Rate (Avg) /s" chart=true WHERE "index"=$metrics_index_token$ span=30s | fields - _span* + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + + Nozzle events dropped + + + | mstats rate_avg("firehose.events.dropped.count") as "Rate (Avg) /s" chart=true WHERE "index"=$metrics_index_token$ span=30s | fields - _span* + $main_time_range.earliest$ + $main_time_range.latest$ + 5m + delay + + + + + + + +
\ No newline at end of file diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..57ae9479 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,59 @@ +# Development + +## Software Requirements + +Make sure you have the following installed on your workstation: + +| Software | Version | +|----------|----------| +| go | go1.17.x | + +Then make sure that all dependent packages are there: + +``` +$ cd +$ make installdeps +``` + +## Environment + +For development against [bosh-lite](https://github.com/cloudfoundry/bosh-lite), +copy `tools/nozzle.sh.template` to `tools/nozzle.sh` and supply missing values: + +``` +$ cp tools/nozzle.sh.template tools/nozzle.sh +$ chmod +x tools/nozzle.sh +``` + +Build project: + +``` +$ make VERSION=1.3.1 +``` + +Run tests with [Ginkgo](http://onsi.github.io/ginkgo/) + +``` +$ ginkgo -r +``` + +Run all kinds of testing + +``` +$ make test # run all unittest +$ make race # test if there is race condition in the code +$ make vet # examine GoLang code +$ make cov # code coverage test and code coverage html report +``` + +Or run all testings: unit test, race condition test, code coverage etc +``` +$ make testall +``` + +Run app + +``` +# this will run: go run main.go +$ ./tools/nozzle.sh +``` diff --git a/docs/environment-variables.md b/docs/environment-variables.md new file mode 100644 index 00000000..6cd9e550 --- /dev/null +++ b/docs/environment-variables.md @@ -0,0 +1,79 @@ +# Environment Parameters + +You can declare parameters by making a copy of the `scripts/ci_nozzle_manifest.sh`. + +| Variable name | Description | Default value | +|---------------|---------------------------------------------------------------|---------------| +| `DEBUG` | Enable debug mode (forward to standard out instead of Splunk) | false | + +## Cloud Foundry configuration parameters: + +| Variable name | Description | Default value | Mandatory parameter | +|-----------------|-----------------------------------------------------------------------|---------------|---------------------| +| `API_ENDPOINT` | Cloud Foundry API endpoint address | - | Yes | +| `CLIENT_ID` | UAA Client ID (Must have authorities and grant_types described above) | - | Yes | +| `CLIENT_SECRET` | Secret for Client ID. | - | Yes | + +## Splunk configuration parameters: +| Variable name | Description | Default value | Mandatory parameter | +|-----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------| +| `SPLUNK_TOKEN ` | [Splunk HTTP event collector token](http://docs.splunk.com/Documentation/Splunk/latest/Data/UsetheHTTPEventCollector/) | - | Yes | +| `SPLUNK_HOST` | Splunk HTTP event collector host, example: https://example.cloud.splunk.com:8088 | - | Yes | +| `SPLUNK_INDEX` | The Splunk index events will be sent to. Warning: Setting an invalid index will cause events to be lost. This index must match one of the selected indexes for the Splunk HTTP event collector token used for the `SPLUNK_TOKEN` parameter. | - | Yes | + +## Advanced Configuration Features: + +| Variable name | Description | Default value | Mandatory parameter | +|------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------|---------------------| +| `JOB_NAME` | Tags nozzle log events with job name. | 'splunk-nozzle' | No | +| `JOB_INDEX` | Tags nozzle log events with job index. | -1 | No | +| `JOB_HOST` | Tags nozzle log events with job host. | "" | No | +| `SKIP_SSL_VALIDATION_CF` | Skips SSL certificate validation for connection to Cloud Foundry. Secure communications will not check SSL certificates against a trusted certificate authority. This is recommended for dev environments only. | false | No | +| `SKIP_SSL_VALIDATION_SPLUNK` | Skips SSL certificate validation for connection to Splunk. Secure communications will not check SSL certificates against a trusted certificate authority. This is recommended for dev environments only. | false | No | +| `FIREHOSE_SUBSCRIPTION_ID` | Tags nozzle events with a Firehose subscription id. See [here](https://docs.vmware.com/en/VMware-Tanzu-Application-Service/6.0/tas-for-vms/log-ops-guide.html). | splunk-firehose | No | +| `FIREHOSE_KEEP_ALIVE` | Keep alive duration for the Firehose consumer. | 25s | No | +| `ADD_APP_INFO` | Enrich raw data with app info. A comma separated list of app metadata (`AppName,OrgName,OrgGuid,SpaceName,SpaceGuid`). | "" | No | +| `ADD_TAGS` | Add additional tags from envelope to splunk event. (Please note: Enabling this feature may slightly impact the performance due to the increased event size) | false | No | +| `IGNORE_MISSING_APP` | If the application is missing, then stop repeatedly querying application info from Cloud Foundry. | true | No | +| `MISSING_APP_CACHE_INVALIDATE_TTL` | How frequently the missing app info cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). See [about app cache params](#about-app-cache-params) | 0s | No | +| `APP_CACHE_INVALIDATE_TTL` | How frequently the app info local cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). See [about app cache params](#about-app-cache-params) | 0s | No | +| `ORG_SPACE_CACHE_INVALIDATE_TTL` | How frequently the org and space cache invalidates (in s/m/h. For example, 3600s or 60m or 1h). | 72h | No | +| `APP_LIMITS` | Restrict to `APP_LIMITS` the most updated apps per request when populating the app metadata cache. Keep it 0 to update all the apps. | 0 | No | +| `BOLTDB_PATH` | Bolt database path. | cache.db | No | +| `EVENTS` | A comma separated list of events to include. Possible values: ValueMetric,CounterEvent,Error,LogMessage,HttpStartStop,ContainerMetric. If no event type is selected, nozzle will automatically select LogMessage to keep the nozzle running. | "ValueMetric,CounterEvent,ContainerMetric" | Yes | +| `EXTRA_FIELDS` | Extra fields to annotate your events with (format is key:value,key:value). | "" | No | +| `FLUSH_INTERVAL` | Time interval (in s/m/h. For example, 3600s or 60m or 1h) for flushing queue to Splunk regardless of `CONSUMER_QUEUE_SIZE`. Protects against stale events in low throughput systems. | 5s | No | +| `CONSUMER_QUEUE_SIZE` | Sets the internal consumer queue buffer size. Events will be pushed to Splunk after queue is full. | 10000 | No | +| `HEC_BATCH_SIZE` | Set the batch size for the events to push to HEC (Splunk HTTP Event Collector). | 100 | No | +| `HEC_RETRIES` | Retry count for sending events to Splunk. After expiring, events will begin dropping causing data loss. | 5 | No | +| `HEC_WORKERS` | Set the amount of Splunk HEC workers to increase concurrency while ingesting in Splunk. | 8 | No | +| `ENABLE_EVENT_TRACING` | Enables event trace logging. Splunk events will now contain a UUID, Splunk Nozzle Event Counts, and a Subscription-ID for Splunk correlation searches. | false | No | +| `SPLUNK_LOGGING_INDEX` | The Splunk index where logs from the nozzle of the sourcetype `cf:splunknozzle` will be sent to. Warning: Setting an invalid index will cause events to be lost. This index must match one of the selected indexes for the Splunk HTTP event collector token used for the `SPLUNK_TOKEN` parameter. When not provided, all logging events will be forwarded to the default `SPLUNK_INDEX`. | "" | No | +| `STATUS_MONITOR_INTERVAL` | Time interval (in s/m/h. For example, 3600s or 60m or 1h) to enable monitoring of metric data within the connector. (This increases CPU load and should be used only for insights purposes). | 0s | No | +| `SPLUNK_METRIC_INDEX` | Index in which metric data will be ingested when monitoring module is enabled | - | No | +| `SELECTED_MONITORING_METRICS` | Name of the metrics that you want to monitor and add using comma seprated values. List of the metrics that are supported in the metrics modules are given below | - | No | +| `REFRESH_SPLUNK_CONNECTION` | If set to true, PCF will periodically refresh connection to Splunk (how often depends on `KEEP_ALIVE_TIMER` value). If set to false connection will be kept alive and reused. | false | No | +| `KEEP_ALIVE_TIMER` | Time after which connection to Splunk will be refreshed, if `REFRESH_SPLUNK_CONNECTION` is set to true (in s/m/h. For example, 3600s or 60m or 1h). | 30s | No | +| `MEMORY_BALLAST_SIZE` | Size of memory allocated to reduce GC cycles. Size should be less than the total memory. | 0 | No | + +### About app cache params: + +When `ADD_APP_INFO` config is enabled, the nozzle will enrich the event with app metadata. +For this, the nozzle maintains a cache of all the apps locally so that it does not need to query from remote every time. + +Now, when there is a change in this app data in remote, the nozzle has to update this local cache. +For this, the config has `APP_CACHE_INVALIDATE_TTL` parameter. At every `APP_CACHE_INVALIDATE_TTL` interval, the nozzle will update the local cache by querying the remote (CF APIs). + +If `APP_CACHE_INVALIDATE_TTL` is set to 10s, the nozzle will refresh the local cache at every 10s. +So, AppCacheTTL should be set based on how frequently the app data is expected to change. + +When the nozzle receives events from the doppler, it will check the local cache for the given app-id. +But on cache-miss, it will query remote for that specific app. +If it doesn’t find the app data from remote too, then the nozzle will add that app to MissingAppCache +(if `IGNORE_MISSING_APP` config is **enabled**. so that the nozzle does not waste time in querying the remote for an app which is likely not to be found). +So, from the next time onwards, the nozzle will first check in the MissingAppCache, if found then it will ignore the app and move on to the next event with a warning. + +`MISSING_APP_CACHE_INVALIDATE_TTL` is used to clear the MissingAppCache so nozzle can retry querying from remote. + +For example, given `MISSING_APP_CACHE_INVALIDATE_TTL` is set to 60s, when nozzle receives event from app that is not available in local cache and remote, +it’ll add it to MissingAppCache. Until next `MISSING_APP_CACHE_INVALIDATE_TTL`, nozzle will not query from remote for the missing app. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..5b3bb4a0 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,34 @@ +# General information + +Cloud Foundry Firehose-to-Splunk Nozzle + +## Compatibility + +### VMware Tanzu Application Service versions + +Splunk Firehose Nozzle has been tested on `v3.0`, `v4.0`, `v5.0` and `v6.0` of Tanzu Application Service. + +### VMware Ops Manager version + +Splunk Firehose Nozzle has been tested on `v3.0.9 LTS` of VMware Ops Manager + +## Usage + +Splunk nozzle is used to stream Cloud Foundry Firehose events to Splunk HTTP Event Collector. Using pre-defined Splunk sourcetypes, +the nozzle automatically parses the events and enriches them with additional metadata before forwarding to Splunk. +For detailed descriptions of each Firehose event type and their fields, refer to underlying [dropsonde protocol](https://github.com/cloudfoundry/dropsonde-protocol). +Below is a mapping of each Firehose event type to its corresponding Splunk sourcetype. +Refer to [Searching Events](./troubleshooting#searching-events) for example Splunk searches. + +| Firehose event type | Splunk sourcetype | Description | +|---------------------|----------------------|--------------------------------------------------------------------------| +| Error | `cf:error` | An Error event represents an error in the originating process | +| HttpStartStop | `cf:httpstartstop` | An HttpStartStop event represents the whole lifecycle of an HTTP request | +| LogMessage | `cf:logmessage` | A LogMessage contains a "log line" and associated metadata | +| ContainerMetric | `cf:containermetric` | A ContainerMetric records resource usage of an app in a container | +| CounterEvent | `cf:counterevent` | A CounterEvent represents the increment of a counter | +| ValueMetric | `cf:valuemetric` | A ValueMetric indicates the value of a metric at an instant in time | + +In addition, logs from the nozzle itself are of sourcetype `cf:splunknozzle`. + + diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 00000000..2be5f035 --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,223 @@ +# Initial setup + +The Nozzle requires a client with the authorities `doppler.firehose` and `cloud_controller.admin_read_only` (the latter is only required if `ADD_APP_INFO` is enabled) and grant-types `client_credentials` and `refresh_token`. If `cloud_controller.admin_read_only` is not +available in the system, switch to use `cloud_controller.admin`. + +You can either +* Add the client manually using [uaac](https://github.com/cloudfoundry/cf-uaac) +* Add the client to the deployment manifest; see [uaa.scim.users](https://github.com/cloudfoundry/uaa-release/blob/master/jobs/uaa/spec) + +Manifest example: + +```yaml + +# Clients +uaa.clients: + splunk-firehose: + id: splunk-firehose + override: true + secret: splunk-firehose-secret + authorized-grant-types: client_credentials,refresh_token + authorities: doppler.firehose,cloud_controller.admin_read_only +``` + +`uaac` example: +```shell +uaac target https://uaa.[system domain url] +uaac token client get admin -s [admin client credentials secret] +uaac client add splunk-firehose --name splunk-firehose +uaac client add splunk-firehose --secret [your_client_secret] +uaac client add splunk-firehose --authorized_grant_types client_credentials,refresh_token +uaac client add splunk-firehose --authorities doppler.firehose,cloud_controller.admin_read_only + +``` + +`cloud_controller.admin_read_only` will work for cf v241 +or later. Earlier versions should use `cloud_controller.admin` instead. + +### Push as an App to Cloud Foundry + +Push Splunk Firehose Nozzle as an application to Cloud Foundry. Please refer to **Setup** section for details +on user authentication. + +1. Download the latest release + + ```shell + git clone https://github.com/cloudfoundry-community/splunk-firehose-nozzle.git + cd splunk-firehose-nozzle + ``` + +2. Authenticate to Cloud Foundry + + ```shell + cf login -a https://api.[your cf system domain] -u [your id] + ``` + +3. Copy the manifest template and fill in needed values (using the credentials created during setup) + + ```shell + vim scripts/ci_nozzle_manifest.yml + ``` + +4. Push the nozzle + + ```shell + make deploy-nozzle + ``` + +#### Dump application info to boltdb +If in production where there are lots of CF applications (say tens of thousands) and if the user would like to enrich +application logs by including application metadata, querying all application metadata information from CF may take some time - +for example if we include: add app name, space ID, space name, org ID and org name to the events. +If there are multiple instances of Spunk nozzle deployed the situation will be even worse, since each of the Splunk nozzle(s) will query all applications meta data and +cache the metadata information to the local boltdb file. These queries will introduce load to the CF system and could potentially take a long time to finish. +Users can run this tool to generate a copy of all application metadata and copy this to each Splunk nozzle deployment. Each Splunk nozzle can pick up the cache copy and update the cache file incrementally afterwards. + +Example of how to run the dump application info tool: + +``` +$ cd tools/dump_app_info +$ go build dump_app_info.go +$ ./dump_app_info --skip-ssl-validation --api-endpoint=https:// --user= --password= +``` + +After populating the application info cache file, user can copy to different Splunk nozzle deployments and start Splunk nozzle to pick up this cache file by +specifying correct "--boltdb-path" flag or "BOLTDB_PATH" environment variable. + +### Disable logging for noisy applications +Set `F2S_DISABLE_LOGGING` = true as a environment variable in applications's manifest to disable logging. + + +## Index routing +Index routing is a feature that can be used to send different Cloud Foundry logs to different indexes for better ACL and data retention control in Splunk. + +### Per application index routing via application manifest +To enable per app index routing, +* Please set environment variable `SPLUNK_INDEX` in your application's manifest ([example below](#example-manifest-file)) +* Make sure Splunk nozzle is configured with `ADD_APP_INFO` (Select at least one of AppName,OrgName,OrgGuid,SpaceName,SpaceGuid) to enable app info caching +* Make sure `SPLUNK_INDEX` specified in app's manifest exist in Splunk and can receive data for the configured Splunk HEC token. + +> **WARNING**: If `SPLUNK_INDEX` is invalid, events from other apps may also get lost as splunk will drop entire event batch if any of the event from batch is invalid (i.e. invalid index) + +There are two ways to set the variable: + +In your app manifest provide an environment variable called `SPLUNK_INDEX` and assign it the index you would like to send the app data to. + +#### Example Manifest file +``` +applications: +- name: + memory: 256M + disk_quota: 256M + ... + env: + SPLUNK_INDEX: + ... +``` + +You can also update the env on the fly using cf-cli command: +``` +cf set-env SPLUNK_INDEX +``` +#### Please note +> If you are updating env on the fly, make sure that `APP_CACHE_INVALIDATE_TTL` is greater tha 0s. Otherwise cached app-info will not be updated and events will not be sent to required index. + + +### Index routing via Splunk configuration +Logs can be routed using fields such as app ID/name, space ID/name or org ID/name. +Users can configure the Splunk configuration files props.conf and transforms.conf on Splunk indexers or Splunk Heavy Forwarders if deployed. + +Below are few sample configuration: + +1. Route data from application ID `95930b4e-c16c-478e-8ded-5c6e9c5981f8` to a Splunk `prod` index: + +*$SPLUNK_HOME/etc/system/local/props.conf* +``` +[cf:logmessage] +TRANSFORMS-index_routing = route_data_to_index_by_field_cf_app_id +``` + + +*$SPLUNK_HOME/etc/system/local/transforms.conf* +``` +[route_data_to_index_by_field_cf_app_id] +REGEX = "(\w+)":"95930b4e-c16c-478e-8ded-5c6e9c5981f8" +DEST_KEY = _MetaData:Index +FORMAT = prod +``` + + +2. Routing application logs from any Cloud Foundry orgs whose names are prefixed with `sales` to a Splunk `sales` index. + +*$SPLUNK_HOME/etc/system/local/props.conf* +``` +[cf:logmessage] +TRANSFORMS-index_routing = route_data_to_index_by_field_cf_org_name + +``` + +*$SPLUNK_HOME/etc/system/local/transforms.conf* +``` +[route_data_to_index_by_field_cf_org_name] +REGEX = "cf_org_name":"(sales.*)" +DEST_KEY = _MetaData:Index +FORMAT = sales +``` + +3. Routing data from sourcetype `cf:splunknozzle` to index `new_index`: + +*$SPLUNK_HOME/etc/system/local/props.conf* +``` +[cf:splunknozzle] +TRANSFORMS-route_to_new_index = route_to_new_index +``` + +*$SPLUNK_HOME/etc/system/local/transforms.conf* +``` +[route_to_new_index] +SOURCE_KEY = MetaData:Sourcetype +DEST_KEY =_MetaData:Index +REGEX = (sourcetype::cf:splunknozzle) +FORMAT = new_index +``` +**Note:** Moving from version 1.2.4 to 1.2.5, timestamp will use nanosecond precision instead of milliseconds. + + +## Monitoring (Metric data Ingestion): + +| Metric Name | Description | +|----------------------------------|-----------------------------------------------------------------------------| +| `nozzle.queue.percentage` | Shows how much internal queue is filled | +| `splunk.events.dropped.count` | Number of events dropped from splunk HEC | +| `splunk.events.sent.count` | Number of events sent to splunk | +| `firehose.events.dropped.count` | Number of events dropped from nozzle | +| `firehose.events.received.count` | Number of events received from firehose(websocket) | +| `splunk.events.throughput` | Average Payload size | +| `nozzle.usage.ram` | RAM Usage | +| `nozzle.usage.cpu` | CPU Usage | +| `nozzle.cache.memory.hit` | How many times it has successfully retrieved the data from memory | +| `nozzle.cache.memory.miss` | How many times it has unsuccessfully tried to retreive the data from memory | +| `nozzle.cache.remote.hit` | How many times it has successfully retrieved the data from remote | +| `nozzle.cache.remote.miss` | How many times it has unsuccessfully tried to retrieve the data from remote | +| `nozzle.cache.boltdb.hit` | How many times it has successfully retrieved the data from BoltDB | +| `nozzle.cache.boltdb.miss` | How many times it has unsuccessfully tried to retrieve the data from BoltDB | + +![event_count](https://user-images.githubusercontent.com/89519924/200804220-1adff84c-e6f1-4438-8d30-6e2cce4984f5.png) + +![nozzle_logs](https://user-images.githubusercontent.com/89519924/200804285-22ad7863-1db3-493a-8196-cc589837db76.png) + +**Note:** Select value Rate(Avg) for Aggregation from Analysis tab on the top right. + +You can find a pre-made dashboard that can be used for monitoring in the `dashboards` directory. + +### Routing data through edge processor via HEC +Logs can be routed to Splunk via Edge Processor. Assuming that you have a working Edge Processor instance, you can use it with minimal +changes to nozzle configuration. + +Configuration fields that you should change are: +* `SPLUNK_HOST`: Use the host of your Edge Processor instance instead of Splunk. Example: https://x.x.x.x:8088. +* `SPLUNK_TOKEN`: It is a required parameter. A token used to authorize your request, can be found in Edge Processor settings. If your + EP token authentication is turned off, you can enter a placeholder values instead (e.x. "-"). + + + diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..12d8974f --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,195 @@ +# Troubleshooting + +This topic describes how to troubleshoot Splunk Firehose Nozzle for Cloud Foundry. + +## No data in Splunk + +Are you searching for events and not finding them or looking at a dashboard and seeing "No result found"? Check Splunk Nozzle app logs. + +To view the nozzle's logs running on CF do the following: + +1. Log in as an admin via the CLI. +2. Target the org created by the tile. + ``` + cf target -o SPLUNK-NOZZLE-ORG + ``` +3. View the recent app Splunk Nozzle logs (the version number installed by the tile will vary). + ``` + cf logs --recent splunk-firehoze-nozzle + ``` +4. Alternatively, you can stream the app logs as they are emitted. + ``` + cf logs splunk-firehose-nozzle + ``` + +### Here are a few common errors and possible resolutions: + +#### Splunk configuration related errors: + +``` +{"timestamp":"