CB-31071: prep for 3.7.0 release (#189)

dseidel-b9 · dseidel · jragozzinocb · web-flow · commit be08f7650063 · 2020-06-22T09:56:22.000-04:00
* CB-31071: begin prep for 3.7.0 release * CB-31071: updat README and CHANGELOG, remove stale TODO * CB-31071: update RPM spec * CB-31071: prepend make command with "RELEASE=1" to get RPM without timestamp * CB-31071: deprecate Upstart in favor of sysvinit (which we already support) * CB-31071: RPM fix attempt * CB-31071: slight makefile cleanup * CB-31071: missed a spot * CB-31071: remove accidental dependency * CB-31071: remove more makefile cruft * CB-31071: another rpm fix * CB-30664: Add AMSI events to Event Forwarder (#188) * CB-31071: add CHANGELOG note about AMSI support * CB-31071: fix typo in README Co-authored-by: dseidel <dseidel@carbonblack.com> Co-authored-by: jragozzinocb <50180271+jragozzinocb@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,21 @@
-# CB Event Forwarder Changelog
+# CB EDR Event Forwarder Changelog
 
-## v.3.7.0
- * Reverted use of confluentinc kafak client library to pure go sarama client
- * specify CA/Client cert/keys in PEM format
+## v3.7.0
 
-## v.3.6.3
+#### Features
+
+ * We now support Antimalware Scan Interface (AMSI) events. This event is called `ingress.event.filelessscriptload`. Please note that you will need EDR 7.2.0 in order to receive these events.
+ * New command-line option `-pid-file <pid_filename>` for better parity with other services, and to facilitate process monitoring.
+
+#### Bug Fixes / Changes
+
+ * Reverted use of Confluent Kafka client library to the pure Go Sarama client.
+ * Removed configuration settings `api_token`, `api_verify_ssl`, and `api_proxy_ssl`. Event Forwarder no longer needs to use the EDR API to perform event post-processing. EDR now has built-in capability for adding report titles to feed hit events.
+ * Changed some log messages in the protobuf processing code to debug level, to avoid filling log files with unneeded entries.
+ * Specify CA/Client cert/keys in PEM format.
+ * Deprecate Upstart in favor of sysvinit for service control on EL6 systems
+
+## v3.6.3
 
 #### Features
 
diff --git a/MANIFEST6 b/MANIFEST6
@@ -2,5 +2,4 @@
 /usr/share/cb/integrations/event-forwarder/kafka-util
 /etc/cb/integrations/event-forwarder/cb-event-forwarder.conf
 /etc/init.d/cb-event-forwarder
-/etc/init/cb-event-forwarder.conf
 /usr/share/cb/integrations/event-forwarder/content/*
diff --git a/MANIFEST7 b/MANIFEST7
@@ -2,5 +2,4 @@
 /usr/share/cb/integrations/event-forwarder/kafka-util
 /etc/cb/integrations/event-forwarder/cb-event-forwarder.conf
 /etc/systemd/system/cb-event-forwarder.service
-/etc/init/cb-event-forwarder.conf
 /usr/share/cb/integrations/event-forwarder/content/*
diff --git a/MANIFEST8 b/MANIFEST8
@@ -2,5 +2,4 @@
 /usr/share/cb/integrations/event-forwarder/kafka-util
 /etc/cb/integrations/event-forwarder/cb-event-forwarder.conf
 /etc/systemd/system/cb-event-forwarder.service
-/etc/init/cb-event-forwarder.conf
 /usr/share/cb/integrations/event-forwarder/content/*
diff --git a/Makefile b/Makefile
@@ -1,43 +1,42 @@
 #GIT_VERSION := $(shell git describe --tags)
 #VERSION := $(shell cat VERSION)
 
-GIT_VERSION := 3.6.3
-VERSION := 3.6.3
+GIT_VERSION := 3.7.0
+VERSION := 3.7.0
 GO_PREFIX := github.com/carbonblack/cb-event-forwarder
 EL_VERSION := $(shell rpm -E %{rhel})
 TARGET_OS=linux
-PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/lib/pkgconfig/:`find rdkafka.pc 2>/dev/null`
 export GO111MODULE=on
 
+# non-release builds include a timestamp in the RPM name
+# use "RELEASE=1 make rpm" for a release build, which will not use the timestamp
+# RELEASE has a default value of 0
+RELEASE ?= 0
+
 .PHONY: clean test rpmbuild rpminstall build rpm
 
 cb-event-forwarder: build
 
-librdkafka:
-ifeq ($TARGET_OS,"linux")
-	ldconfig -p | grep librdkafka
-endif
-
 compile-protobufs:
+	go get -u github.com/gogo/protobuf/protoc-gen-gogofast
+	go mod tidy
 	protoc --gogofast_out=.  ./cmd/cb-event-forwarder/sensor_events.proto
 	sed -i 's/package sensor_events/package main/g' ./cmd/cb-event-forwarder/sensor_events.pb.go
 
-build-no-static: compile-protobufs librdkafka
-	go get -u github.com/gogo/protobuf/protoc-gen-gogofast
-	go mod tidy
+build-no-static: compile-protobufs
 	go mod verify
 	go build ./cmd/cb-event-forwarder
 	go build ./cmd/kafka-util
 
-build: compile-protobufs librdkafka
-	go get -u github.com/gogo/protobuf/protoc-gen-gogofast
-	go mod tidy
+build: compile-protobufs
 	go mod verify
 	go build -tags static ./cmd/cb-event-forwarder
 	go build -tags static ./cmd/kafka-util
 
-rpmbuild: compile-protobufs librdkafka
+rpmbuild:
 	go get -u github.com/gogo/protobuf/protoc-gen-gogofast
+	protoc --gogofast_out=.  ./cmd/cb-event-forwarder/sensor_events.proto
+	sed -i 's/package sensor_events/package main/g' ./cmd/cb-event-forwarder/sensor_events.pb.go
 	go build -tags static -ldflags "-X main.version=${VERSION}" ./cmd/cb-event-forwarder
 	go build -tags static -ldflags "-X main.version=${VERSION}" ./cmd/kafka-util
 
@@ -48,7 +47,6 @@ rpminstall:
 	cp -p cb-edr-fix-permissions.sh ${RPM_BUILD_ROOT}/usr/share/cb/integrations/event-forwarder/cb-edr-fix-permissions.sh
 	mkdir -p ${RPM_BUILD_ROOT}/etc/cb/integrations/event-forwarder
 	cp -p conf/cb-event-forwarder.example.ini ${RPM_BUILD_ROOT}/etc/cb/integrations/event-forwarder/cb-event-forwarder.conf
-	mkdir -p ${RPM_BUILD_ROOT}/etc/init
 ifeq (${EL_VERSION},6)
 	mkdir -p ${RPM_BUILD_ROOT}/etc/init.d
 	cp -p init-scripts/cb-event-forwarder ${RPM_BUILD_ROOT}/etc/init.d/cb-event-forwarder
@@ -57,7 +55,6 @@ else
 	mkdir -p ${RPM_BUILD_ROOT}/etc/systemd/system
 	cp -p cb-event-forwarder.service ${RPM_BUILD_ROOT}/etc/systemd/system/cb-event-forwarder.service
 endif
-	cp -p init-scripts/cb-event-forwarder.conf ${RPM_BUILD_ROOT}/etc/init/cb-event-forwarder.conf
 	mkdir -p ${RPM_BUILD_ROOT}/usr/share/cb/integrations/event-forwarder/content
 	cp -rp static/* ${RPM_BUILD_ROOT}/usr/share/cb/integrations/event-forwarder/content
 
@@ -77,12 +74,12 @@ clean:
 	rm -rf dist
 	rm -rf build
 	rm -f VERSION
-	rm -rf librdkafka
 
 bench:
 	go test -bench=. ./cmd/cb-event-forwarder/
 
 sdist:
+	$(info RELEASE is ${RELEASE})
 	mkdir -p build/cb-event-forwarder-${GIT_VERSION}/src/${GO_PREFIX}
 	echo "${GIT_VERSION}" > build/cb-event-forwarder-${GIT_VERSION}/VERSION
 	cp -rp cb-event-forwarder cb-edr-fix-permissions.sh cb-event-forwarder.service Makefile go.mod cmd static conf init-scripts build/cb-event-forwarder-${GIT_VERSION}/src/${GO_PREFIX}
@@ -95,4 +92,4 @@ sdist:
 rpm: sdist
 	mkdir -p ${HOME}/rpmbuild/SOURCES
 	cp -p dist/cb-event-forwarder-${GIT_VERSION}.tar.gz ${HOME}/rpmbuild/SOURCES/
-	rpmbuild --define 'version ${GIT_VERSION}' --define 'release 0' -bb cb-event-forwarder.rpm.spec
+	rpmbuild -v --define 'release_pkg ${RELEASE}' --define 'version ${GIT_VERSION}' --define 'release 1' -bb cb-event-forwarder.rpm.spec
diff --git a/README.md b/README.md
@@ -1,37 +1,37 @@
-# CB Event Forwarder
+# CB EDR Event Forwarder
 
 ## Overview
 
-The CB EDR Event Forwarder is a standalone service that will listen on the Cb EDR enterprise bus and export
-events (both watchlist/feed hits as well as raw endpoint events, if configured) in a normalized JSON or LEEF format.
+The CB EDR Event Forwarder is a standalone service which listens on the CB EDR enterprise bus and exports
+events (watchlist/feed hits, as well as raw endpoint events, if configured) in a normalized JSON or LEEF format.
 The events can be saved to a file, delivered to a network service or archived automatically to an Amazon AWS S3 bucket.
 These events can be consumed by any external system that accepts JSON or LEEF, including Splunk and IBM QRadar.
 
-The list of events to collect is configurable. By default all feed and watchlist hits, alerts, binary notifications, and
-raw sensor events are exported into JSON. The configuration file for the connector is stored in
+The list of events to collect is configurable. By default, Event Forwarder exports all feed and watchlist hits, alerts,
+binary notifications, and raw sensor events as JSON. You can find the configuration file for the connector at
 `/etc/cb/integrations/event-forwarder/cb-event-forwarder.conf`.
 
 Starting with version 7.1.0 of Carbon Black EDR, you can use the EDR web console to configure and control Event Forwarder,
 as long as you follow the installation and configuration steps detailed below.
 
 ## Support
 
-The pre-built RPM is supported via our [User eXchange (Jive)](https://community.carbonblack.com/community/developer-relations) 
+We support pre-built RPM via our [User eXchange (Jive)](https://community.carbonblack.com/community/developer-relations) 
 and via email to dev-support@carbonblack.com. 
 
 ## Raw Sensor Events 
 
-We have seen a performance impact when exporting all raw sensor events onto the enterprise bus. We do not recommend
-exporting all the events. The performance impacts are seen when the events are broadcast on the bus, by enabling the
-"DatastoreBroadcastEventTypes". We recommend that at most, only process and netconn events be broadcast on the event
+We have seen a performance impact when exporting all raw sensor events onto the enterprise bus by setting
+"DatastoreBroadcastEventTypes=True" in the EDR configuration (more on this below). We do not recommend exporting all
+the events, and recommend that you configure -- at most -- only process and netconn events for broadcasting on the event
 bus. 
 
 ## Quickstart Guide
 
 The cb-event-forwarder can be installed on any 64-bit Linux machine running CentOS 6.x. 
 It can be installed on the same machine as the Cb EDR server, or another machine. 
 If you are forwarding a large volume of events to QRadar (for example, all file modifications and/or registry 
-modifications), or are forwarding events from a Cb EDR cluster, then installing it on a separate machine is recommended. 
+modifications), or are forwarding events from a Cb EDR cluster, we recommend installing it on a separate machine. 
 Otherwise, it is acceptable to install the cb-event-forwarder on the Cb EDR server itself.
 
 ### Installation
@@ -93,8 +93,8 @@ If you want to capture raw sensor events or the `binaryinfo.*` notifications, yo
 * If you are capturing binary observed events you also need to edit the `EnableSolrBinaryInfoNotifications` option in 
 `/etc/cb/cb.conf` and set it to `True`.
 
-Cb EDR needs to be restarted if any variables were changed in `/etc/cb/cb.conf` by executing
-`service cb-enterprise restart`. 
+Cb EDR needs to be restarted if any you change any variables in `/etc/cb/cb.conf` by executing
+`service cb-enterprise restart`.
 
 If you are configuring the cb-event-forwarder on a Cb EDR cluster, the `DatastoreBroadcastEventTypes` and/or
 `EnableSolrBinaryInfoNotifications` settings
@@ -104,16 +104,14 @@ the `/usr/share/cb/cbcluster stop && /usr/share/cb/cbcluster start` command.
 ### Starting and Stopping the Service
 
 #### CentOS 6.x
-Once the service is installed, it is managed by the Upstart init system in CentOS 6.x. You can control the service via the 
-initctl command.
-* To start the service, `initctl start cb-event-forwarder`
-* To stop the service, `initctl stop cb-event-forwarder`
+* To start the service: `service cb-event-forwarder start`
+* To stop the service: `service cb-event-forwarder stop`
 
 #### CentOS 7.x
-* To start the service, `systemctl start cb-event-forwarder`
-* To stop the service, `systemctl stop cb-event-forwarder`
+* To start the service: `systemctl start cb-event-forwarder`
+* To stop the service: `systemctl stop cb-event-forwarder`
 
-Once the service is installed, it is configured to start automatically on system boot.
+Once you install the service, it is configured to start automatically on system boot.
 
 ## Splunk
 
diff --git a/TODO.md b/TODO.md
diff --git a/cb-event-forwarder.docker.ini b/cb-event-forwarder.docker.ini
@@ -261,6 +261,7 @@ use_raw_sensor_exchange=true
 #   ingress.event.remotethread
 #   ingress.event.processblock
 #   ingress.event.emetmitigation
+#   ingress.event.filelessscriptload
 #   ALL for all of the above
 #   0 - to disable all raw sensor events.
 events_raw_sensor=ALL
diff --git a/cb-event-forwarder.dockerkafka.ini b/cb-event-forwarder.dockerkafka.ini
@@ -263,6 +263,7 @@ use_raw_sensor_exchange=true
 #   ingress.event.remotethread
 #   ingress.event.processblock
 #   ingress.event.emetmitigation
+#   ingress.event.filelessscriptload
 #   ALL for all of the above
 #   0 - to disable all raw sensor events.
 events_raw_sensor=ALL
diff --git a/cb-event-forwarder.dockerkafkassl.ini b/cb-event-forwarder.dockerkafkassl.ini
@@ -262,6 +262,7 @@ use_raw_sensor_exchange=true
 #   ingress.event.remotethread
 #   ingress.event.processblock
 #   ingress.event.emetmitigation
+#   ingress.event.filelessscriptload
 #   ALL for all of the above
 #   0 - to disable all raw sensor events.
 events_raw_sensor=ALL
diff --git a/cb-event-forwarder.rpm.spec b/cb-event-forwarder.rpm.spec
@@ -3,13 +3,29 @@
 %global debug_package %{nil}
 %global __os_install_post /usr/lib/rpm/brp-compress %{nil}
 
-%define bare_version 3.6.3
+%define bare_version 3.7.0
 %define build_timestamp %(date +%%y%%m%%d.%%H%%m%%S)
+
+# If release_pkg is defined and has the value of 1, use a plain version string;
+# otherwise, use the version string with a timestamp appended.
+#
+# if not otherwise defined (we do so on the rpmbuild command-line), release_pkg
+# defaults to 0.
+#
+# see https://backreference.org/2011/09/17/some-tips-on-rpm-conditional-macros/
+%if 0%{?release_pkg:1}
+%if "%{release_pkg}" == "1"
+%define decorated_version %{bare_version}
+%else
+%define decorated_version %{bare_version}.%{build_timestamp}
+%endif
+%endif
+
 %define release 1
 
-Summary: Carbon Black event forwarder
+Summary: Carbon Black Event Forwarder
 Name: %{name}
-Version: %{bare_version}.%{build_timestamp}
+Version: %{decorated_version}
 Release: %{release}%{?dist}
 Source0: %{name}-%{bare_version}.tar.gz
 License: MIT
@@ -48,6 +64,10 @@ rm -rf $RPM_BUILD_ROOT
 #!/bin/sh
 # since the "old" cb-event-forwarder controls itself through the file we're about to replace
 # we should stop it before we install anything on upgrade
+# but first we have to stop the service if already running under Upstart
+%if "%{dist}" == ".el6"
+initctl stop cb-event-forwarder &> /dev/null || :
+%endif
 
 if [ -x /etc/init.d/cb-event-forwarder ] || [ -e /etc/systemd/system/cb-event-forwarder.service ]; then
     service cb-event-forwarder stop &> /dev/null || :
diff --git a/cmd/cb-event-forwarder/config.go b/cmd/cb-event-forwarder/config.go
@@ -206,6 +206,7 @@ func (c *Configuration) parseEventTypes(input *ini.File) {
 			"ingress.event.remotethread",
 			"ingress.event.processblock",
 			"ingress.event.emetmitigation",
+			"ingress.event.filelessscriptload",
 		}},
 		{"events_binary_observed", []string{
 			"binaryinfo.#",
diff --git a/cmd/cb-event-forwarder/main.go b/cmd/cb-event-forwarder/main.go
@@ -34,7 +34,7 @@ var (
 	debug              = flag.Bool("debug", false, "Enable debugging mode")
 )
 
-var version = "NOT FOR RELEASE"
+var version = "3.7.0"
 
 var inputChannelSize = 100000
 var outputChannelSize = 1000000
diff --git a/cmd/cb-event-forwarder/pb_message_processor.go b/cmd/cb-event-forwarder/pb_message_processor.go
@@ -318,6 +318,12 @@ func ProcessProtobufMessage(routingKey string, body []byte, headers amqp.Table)
             } else {
                 return nil, nil
             }
+        case cbMessage.Scriptexe != nil:
+            if _, ok:= config.EventMap["ingress.event.filelessscriptload"]; ok {
+                writeScriptexeMsg(inmsg,outmsg)
+            } else {
+                return nil, nil
+            }
         case cbMessage.NetconnBlockedv2 != nil:
             // generated by sensor, but not used in CbR (CB-23994)
             log.Debugf("ignoring unhandled event netconnblockedv2")
@@ -466,6 +472,14 @@ func writeProcessMessage(message *convertedCbMessage, kv map[string]interface{})
     return nil
 }
 
+func writeScriptexeMsg(message *convertedCbMessage, kv map[string]interface{}) {
+    kv["event_type"] = "FilelessScriptload"
+    kv["type"] = "ingress.event.filelessscriptload"
+
+    kv["script"] = message.OriginalMessage.Scriptexe.GetScript()
+    kv["script_sha256"] = message.OriginalMessage.Scriptexe.GetSha256()
+}
+
 func writeModloadMessage(message *convertedCbMessage, kv map[string]interface{}) {
     kv["event_type"] = "modload"
     kv["type"] = "ingress.event.moduleload"
diff --git a/cmd/cb-event-forwarder/sensor_events.proto b/cmd/cb-event-forwarder/sensor_events.proto
diff --git a/conf/cb-event-forwarder.example.ini b/conf/cb-event-forwarder.example.ini
diff --git a/go.sum b/go.sum
diff --git a/init-scripts/cb-event-forwarder.conf b/init-scripts/cb-event-forwarder.conf
