feat: add NLB log support (#72)

* feat: add NLB log support

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* Apply suggestion from @Copilot

Co-authored-by: Copilot <[email protected]>

* Apply suggestion from @Copilot

Co-authored-by: Copilot <[email protected]>

* review: field name

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* chore: improve readme

Signed-off-by: Kavindu Dodanduwa <[email protected]>

* Update generators/nlb.go

Co-authored-by: Copilot <[email protected]>

---------

Signed-off-by: Kavindu Dodanduwa <[email protected]>
Co-authored-by: Copilot <[email protected]>

Author: Kavindu-Dodan

README.md

@@ -1,6 +1,6 @@
-# Data Generator
+# Synthetic Data Generator
 
-A simple data generator with export capability to various destinations
+A flexible synthetic data generator in Go, designed to produce realistic logs and metrics for testing and development.
 
 ## Quick Start
 
@@ -25,7 +25,7 @@ Given below are supported input types and their related environment variable ove
 
 | YAML Property     | Environment Variable        | Description                                                                                                      |
 |-------------------|-----------------------------|------------------------------------------------------------------------------------------------------------------|
-| `type`            | `ENV_INPUT_TYPE`            | Specifies the input data type (e.g., `LOGS`, `METRICS`, `ALB`, `VPC`, `CLOUDTRAIL`, `WAF`).                      |
+| `type`            | `ENV_INPUT_TYPE`            | Specifies the input data type (e.g., `LOGS`, `METRICS`, `ALB`, `NLB`, `VPC`, `CLOUDTRAIL`, `WAF`).               |
 | `delay`           | `ENV_INPUT_DELAY`           | Delay between a data point. Accepts value in format like `5s` (5 seconds), `10ms` (10 milliseconds).             |
 | `batching`        | `ENV_INPUT_BATCHING`        | Set time delay between data batches. Accepts a time value similar to delay. Default is set to `0` (no batching). |
 | `max_batch_size`  | `ENV_INPUT_MAX_BATCH_SIZE`  | Set maximum byte size of a batch. Default is to ignore (no max size).                                            |
@@ -36,7 +36,8 @@ Note about input `type`,
 
 - `LOGS` : ECS (Elastic Common Schema) formatted logs based on zap
 - `METRICS`: Generate metrics similar to a CloudWatch metrics entry
-- `ALB` : Generate AWS ALB formatted log with some random content
+- `ALB` : Generate AWS ALB formatted logs with some random content
+- `NLB` : Generate AWS NLB formatted logs with some random content
 - `VPC`: Generate AWS VPC formatted logs with randomized content
 - `CLOUDTRAIL`: Generate AWS CloudTrail formatted logs with randomized content. Data is generated for AWS S3 Data Event.
 - `WAF`: Generate AWS WAF formatted logs with randomized content

generators/elb.go generators/alb.gogenerators/elb.go renamed to generators/alb.go

@@ -13,6 +13,7 @@ const (
 	logs       = "LOGS"
 	metrics    = "METRICS"
 	alb        = "ALB"
+	nlb        = "NLB"
 	vpc        = "VPC"
 	waf        = "WAF"
 	CloudTrail = "CLOUDTRAIL"
@@ -33,6 +34,8 @@ func GeneratorFor(config *conf.Config) (*Generator, error) {
 		return newGenerator(config.Input, NewMetricGenerator()), nil
 	case alb:
 		return newGenerator(config.Input, NewALBGen()), nil
+	case nlb:
+		return newGenerator(config.Input, NewNLBGen()), nil
 	case vpc:
 		return newGenerator(config.Input, newVPCGen()), nil
 	case waf:

generators/elb_test.go generators/alb_test.gogenerators/elb_test.go renamed to generators/alb_test.go

@@ -0,0 +1,88 @@
+package generators
+
+import (
+	"fmt"
+	"math/rand"
+)
+
+type NLBgen struct {
+	buf trackedBuffer
+}
+
+func NewNLBGen() *NLBgen {
+	return &NLBgen{
+		buf: newTrackedBuffer(),
+	}
+}
+
+func (a *NLBgen) Generate() (int64, error) {
+	customizer := nlbCustomizer{
+		time:              iso8601Now(),
+		name:              fmt.Sprintf("net/my-nlb/%s", randomID()),
+		elbID:             randomID(),
+		clientIPPort:      fmt.Sprintf("%s:%d", randomIP(), randomPort()),
+		destinationIPPort: fmt.Sprintf("%s:%d", randomIP(), randomPort()),
+		conMs:             rand.Intn(1000),
+		tlsHSMs:           rand.Intn(1000),
+		receivedBytes:     randomBytesSize(),
+		sentBytes:         randomBytesSize(),
+		tlsAlert:          "-",
+		certARN:           randomCertArn(),
+		cipher:            randomSSLCipher(),
+		protocol:          sslProtocol(),
+		domain:            randomDomain(),
+		feProtocol:        "-",
+		beProtocol:        "-",
+		alpnList:          "-",
+		creationTime:      iso8601Now(),
+	}
+
+	err := a.buf.write([]byte(buildNLBLogLine(customizer)))
+	if err != nil {
+		return 0, err
+	}
+
+	return a.buf.size(), err
+}
+
+func (a *NLBgen) GetAndReset() []byte {
+	return a.buf.getAndReset()
+}
+
+// helpers
+
+type nlbCustomizer struct {
+	time              string
+	name              string
+	elbID             string
+	clientIPPort      string
+	destinationIPPort string
+	conMs             int
+	tlsHSMs           int
+	receivedBytes     int
+	sentBytes         int
+	tlsAlert          string
+	certARN           string
+	cipher            string
+	protocol          string
+	domain            string
+	feProtocol        string
+	beProtocol        string
+	alpnList          string
+	creationTime      string
+}
+
+func buildNLBLogLine(input nlbCustomizer) string {
+	// Construct the log line
+	log := fmt.Sprintf(
+		"%s %s %s %s %s %s %s %d %d %d %d %s %s %s %s %s %s %s %s %s %s %s\n",
+		"tls", "2.0", input.time, input.name, input.elbID, input.clientIPPort, input.destinationIPPort, input.conMs,
+		input.tlsHSMs, input.receivedBytes, input.sentBytes,
+		input.tlsAlert, input.certARN, "-", // Placeholder for future fields
+		input.cipher, input.protocol, "-", // Placeholder for future fields
+		input.domain,
+		input.feProtocol, input.beProtocol, input.alpnList, input.creationTime,
+	)
+
+	return log
+}

generators/generator.go

@@ -0,0 +1,37 @@
+package generators
+
+import (
+	"github.com/stretchr/testify/require"
+	"strings"
+	"testing"
+)
+
+// refer http example of https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html#access-log-entry-format
+const upstreamNLB = `tls 2.0 2020-04-01T08:51:42 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com h2 h2 "h2","http/1.1" 2020-04-01T08:51:20`
+
+func Test_buildNLB(t *testing.T) {
+	t.Run("Validate AWS documented HTTP NLB line", func(t *testing.T) {
+		line := buildNLBLogLine(nlbCustomizer{
+			time:              "2020-04-01T08:51:42",
+			name:              "net/my-network-loadbalancer/c6e77e28c25b2234",
+			elbID:             "g3d4b5e8bb8464cd",
+			clientIPPort:      "72.21.218.154:51341",
+			destinationIPPort: "172.100.100.185:443",
+			conMs:             5,
+			tlsHSMs:           2,
+			receivedBytes:     98,
+			sentBytes:         246,
+			tlsAlert:          "-",
+			certARN:           "arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99",
+			cipher:            "ECDHE-RSA-AES128-SHA",
+			protocol:          "tlsv12",
+			domain:            "my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com",
+			feProtocol:        "h2",
+			beProtocol:        "h2",
+			alpnList:          `"h2","http/1.1"`,
+			creationTime:      "2020-04-01T08:51:20",
+		})
+
+		require.Equal(t, upstreamNLB, strings.TrimSpace(line))
+	})
+}

generators/nlb.go

@@ -16,7 +16,7 @@ var tt = true
 
 var contentTypes = []string{"text/html", "application/json", "text/plain", "application/xml"}
 var countryCodes = []string{"US", "GB", "DE", "FR", "IN", "CN", "JP", "AU", "CA", "BR"}
-var sampleDomains = []string{"example.com", "test.com", "sample.org", "demo.net"}
+var randomIDs = []string{"id123", "id456", "id789", "id101", "id112", "id131"}
 var httpMethods = []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
 var httpSchema = []string{"http", "https"}
 var httpSourceIDs = []string{"E2A1BCD34FGH56", "E3B2CDE45GHI67", "E4C3DEF56HIJ78", "E5D4EFG67IJK89"}
@@ -26,6 +26,7 @@ var randomPhrases = []string{"some random phrase", "another random phrase", "yet
 var regions = []string{"us-east-1", "us-west-1", "us-west-2", "eu-west-1", "eu-central-1"}
 var s3EventNames = []string{"PutObject", "GetObject", "DeleteObject", "ListObjects"}
 var sampleAccountIDs = []string{"123456789012", "987654321098", "111122223333", "444455556666", "777788889999"}
+var sampleDomains = []string{"example.com", "test.com", "sample.org", "demo.net"}
 var samplePrincipalIDs = []string{"AID1234567890", "AID0987654321", "AID1111222233", "AID7777888899"}
 var sampleRuleIDs = []string{"rule-1", "rule-2", "rule-3", "rule-4", "rule-5"}
 var sslCiphers = []string{"ECDHE-RSA-AES128-GCM-SHA256", "ECDHE-RSA-AES256-GCM-SHA384", "AES128-GCM-SHA256"}
@@ -38,6 +39,7 @@ var vpcActions = []string{"ACCEPT", "REJECT"}
 var wafActions = []string{"ALLOW", "BLOCK", "COUNT"}
 var wafRuleTypes = []string{"REGULAR", "RATE_BASED", "GROUP"}
 var wafSampleHTTPSourceNames = []string{"ALB", "CloudFront", "API Gateway"}
+var certARN = []string{"arn:aws:acm:us-east-1:123456789012:certificate/1", "arn:aws:acm:us-east-1:123456789012:certificate/2", "arn:aws:acm:us-east-1:123456789012:certificate/3"}
 
 // ipPrefix contains example public IP address prefixes, representing geo-distributed or commonly used public IP ranges.
 var ipPrefix = []int{1, 8, 31, 41, 91, 123, 179, 201, 210, 250}
@@ -59,6 +61,14 @@ func randomProcessingTime() float32 {
 	return (0.5 + rand.Float32()*1000) / 1000
 }
 
+func randomID() string {
+	return randomIDs[rand.Intn(len(randomIDs))]
+}
+
+func randomCertArn() string {
+	return certARN[rand.Intn(len(certARN))]
+}
+
 func randomStatus() string {
 	return statuses[rand.Intn(len(statuses))]
 }