Merge pull request #11 from benzaita/lx-2265-add-colselection-to-object-groups

LX-2265 Added ColSelection parameter to Object Group resource

Author: kzonov

Makefile

@@ -3,7 +3,7 @@ HOSTNAME=registry.terraform.io
 NAMESPACE=benzaita
 NAME=chaossearch
 BINARY=terraform-provider-${NAME}
-VERSION=0.7.0
+VERSION=0.8.0
 OS_ARCH=$(shell go env GOOS)_$(shell go env GOARCH)
 
 default: install

chaossearch/client/configuration.go

@@ -6,7 +6,7 @@ type Configuration struct {
 	AccessKeyID     string
 	SecretAccessKey string
 	AWSServiceName  string
-	Region       string
+	Region          string
 }
 
 // NewConfiguration creates a default Configuration struct

chaossearch/client/model.go

@@ -18,30 +18,32 @@ type ReadObjectGroupRequest struct {
 }
 
 type ReadObjectGroupResponse struct {
-	Compression      string
-	FilterJSON       string
-	Format           string
-	Pattern          string
-	LiveEventsSqsArn string
-	PartitionBy      string
-	SourceBucket     string
-	IndexRetention   int
+	Compression       string
+	FilterJSON        string
+	Format            string
+	Pattern           string
+	LiveEventsSqsArn  string
+	PartitionBy       string
+	SourceBucket      string
+	IndexRetention    int
 	ArrayFlattenDepth *int
-	ColumnRenames    map[string]string
+	ColumnRenames     map[string]string
+	ColumnSelection   []map[string]interface{}
 }
 
 type CreateObjectGroupRequest struct {
-	Name             string
-	Compression      string
-	FilterJSON       string
-	Format           string
-	LiveEventsSqsArn string
-	PartitionBy      string
-	SourceBucket     string
-	Pattern          string
-	IndexRetention   int
+	Name              string
+	Compression       string
+	FilterJSON        string
+	Format            string
+	LiveEventsSqsArn  string
+	PartitionBy       string
+	SourceBucket      string
+	Pattern           string
+	IndexRetention    int
 	ArrayFlattenDepth *int
-	ColumnRenames    map[string]interface{}
+	ColumnRenames     map[string]interface{}
+	ColumnSelection   map[string]interface{}
 }
 
 type UpdateIndexingStateRequest struct {

chaossearch/client/operation_create_object_group.go

@@ -36,9 +36,9 @@ func marshalCreateObjectGroupRequest(req *CreateObjectGroupRequest) ([]byte, err
 		"bucket": req.Name,
 		"source": req.SourceBucket,
 		"format": map[string]interface{}{
-			"_type":       req.Format,
-			"horizontal":  true,
-			"stripPrefix": true,
+			"_type":             req.Format,
+			"horizontal":        true,
+			"stripPrefix":       true,
 			"arrayFlattenDepth": req.ArrayFlattenDepth,
 		},
 		"indexRetention": req.IndexRetention,
@@ -56,6 +56,24 @@ func marshalCreateObjectGroupRequest(req *CreateObjectGroupRequest) ([]byte, err
 		options["colRenames"] = req.ColumnRenames
 	}
 
+	if len(req.ColumnSelection) > 0 {
+		var options = body["options"].(map[string]interface{})
+		// @example
+		//"colSelection": [
+		//	{
+		//	"includes": [
+		//		"orig._originalSource",
+		//		"attrs.version",
+		//		"line.message",
+		//		"line.correlation_id",
+		//		"Timestamp"
+		//	],
+		//	"type": "whitelist"
+		//	}
+		//],
+		options["colSelection"] = []map[string]interface{}{req.ColumnSelection}
+	}
+
 	if req.Compression != "" {
 		var options = body["options"].(map[string]interface{})
 		options["compression"] = req.Compression

chaossearch/client/operation_read_indexing_state.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 )
 
-
 // There are other nested structures that we don't marshal here because we don't need to.
 //  Feel free to add them in the future when needed
 type readBucketMetadataResponse struct {
@@ -21,7 +20,7 @@ func (client *Client) ReadIndexingState(ctx context.Context, req *ReadIndexingSt
 	method := "POST"
 	body := &readBucketMetadataRequest{
 		BucketName: req.ObjectGroupName,
-		Stats: false,
+		Stats:      false,
 	}
 
 	response, err := makeGetBucketMetadataRequest(method, client, ctx, body)
@@ -31,7 +30,7 @@ func (client *Client) ReadIndexingState(ctx context.Context, req *ReadIndexingSt
 
 	bucketMetadata := &IndexingState{
 		ObjectGroupName: response.Bucket,
-		Active: false,
+		Active:          false,
 	}
 
 	if response.State == "Active" || response.State == "Idle" {
@@ -60,12 +59,11 @@ func makeGetBucketMetadataRequest(method string, client *Client, ctx context.Con
 	}
 	defer httpResp.Body.Close()
 
-
 	var response readBucketMetadataResponse
 	err = client.unmarshalJSONBody(httpResp.Body, &response)
 	if err != nil {
 		return nil, fmt.Errorf("failed to unmarshal response: %s", err)
 	}
 
 	return &response, nil
-}
+}

chaossearch/client/operation_read_object_group.go

@@ -53,7 +53,8 @@ func (client *Client) readAttributesFromDatasetEndpoint(ctx context.Context, req
 	var getDatasetResp struct {
 		PartitionBy string `json:"partitionBy"`
 		Options     struct {
-			ColumnRenames map[string]string `json:"colRenames"`
+			ColumnRenames   map[string]string        `json:"colRenames"`
+			ColumnSelection []map[string]interface{} `json:"colSelection"`
 		} `json:"options"`
 	}
 	if err := client.unmarshalJSONBody(httpResp.Body, &getDatasetResp); err != nil {
@@ -62,6 +63,7 @@ func (client *Client) readAttributesFromDatasetEndpoint(ctx context.Context, req
 
 	resp.PartitionBy = getDatasetResp.PartitionBy
 	resp.ColumnRenames = getDatasetResp.Options.ColumnRenames
+	resp.ColumnSelection = getDatasetResp.Options.ColumnSelection
 
 	return nil
 }
@@ -110,9 +112,9 @@ func mapBucketTaggingToResponse(tagging *s3.GetBucketTaggingOutput, v *ReadObjec
 	}
 
 	var filterObject struct {
-		Type    string `json:"_type"`
-		Pattern string `json:"pattern"`
-		ArrayFlattenDepth *int `json:"arrayFlattenDepth"`
+		Type              string `json:"_type"`
+		Pattern           string `json:"pattern"`
+		ArrayFlattenDepth *int   `json:"arrayFlattenDepth"`
 	}
 	if err := readJSONTagValue(tagging, "cs3.dataset-format", &filterObject); err != nil {
 		return err

chaossearch/provider.go

@@ -40,7 +40,7 @@ func Provider() *schema.Provider {
 			},
 		},
 		ResourcesMap: map[string]*schema.Resource{
-			"chaossearch_object_group": resourceObjectGroup(),
+			"chaossearch_object_group":   resourceObjectGroup(),
 			"chaossearch_indexing_state": resourceIndexingState(),
 		},
 		DataSourcesMap: map[string]*schema.Resource{

chaossearch/resource_object_group.go

@@ -78,11 +78,11 @@ func resourceObjectGroup() *schema.Resource {
 				ForceNew:    false,
 			},
 			"array_flatten_depth": {
-				Type:        schema.TypeInt,
-				Default: 	 0,
-				Description: "Array flattening level. 0 - disabled, -1 - unlimited, >1 - the respective flattening level",
-				Optional:    true,
-				ForceNew:    true,
+				Type:         schema.TypeInt,
+				Default:      0,
+				Description:  "Array flattening level. 0 - disabled, -1 - unlimited, >1 - the respective flattening level",
+				Optional:     true,
+				ForceNew:     true,
 				ValidateFunc: validation.IntAtLeast(-1),
 			},
 			"column_renames": {
@@ -95,6 +95,28 @@ func resourceObjectGroup() *schema.Resource {
 				ForceNew:    true,
 				Description: "A map specifying names of columns to rename (keys) and what to rename them to (values)",
 			},
+			"column_selection": {
+				Type: schema.TypeSet,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"type": {
+							Type:      schema.TypeString,
+							Required:  true,
+						},
+						"includes": {
+							Type:      schema.TypeList,
+							Elem: &schema.Schema{
+								Type: schema.TypeString,
+							},
+							Required:  true,
+						},
+					},
+
+				},
+				Optional:    true,
+				ForceNew:    true,
+				Description: "List of fields in logs to include or exclude from parsing. If nothing is specified, all fields will be parsed",
+			},
 
 			// Workaround. Otherwise Terraform fails with "All fields are ForceNew or Computed w/out Optional, Update is superfluous"
 			"description": {
@@ -106,36 +128,45 @@ func resourceObjectGroup() *schema.Resource {
 	}
 }
 
-
-
 func resourceObjectGroupCreate(ctx context.Context, data *schema.ResourceData, meta interface{}) diag.Diagnostics {
 	c := meta.(*ProviderMeta).Client
 
 	// "unlimited" flattening represented as "null" in the api, and as -1 in the terraform module
 	// because the terraform sdk doesn't support nil values in configs https://github.com/hashicorp/terraform-plugin-sdk/issues/261
 	// We represent "null" as an int pointer to nil in the code.
-	array_flatten_tf := data.Get("array_flatten_depth").(int)
-	var array_flatten_cs *int
-	if (array_flatten_tf == -1) {
+	arrayFlattenTF := data.Get("array_flatten_depth").(int)
+	var arrayFlattenCS *int
+	if arrayFlattenTF == -1 {
 		// -1 in terraform represents "null" in the ChaosSearch API call
-		array_flatten_cs = nil
+		arrayFlattenCS = nil
 	} else {
 		// any other value is passed as is
-		array_flatten_cs = &array_flatten_tf
+		arrayFlattenCS = &arrayFlattenTF
+	}
+	var columnSelection map[string]interface{}
+
+	if data.Get("column_selection").(*schema.Set).Len() > 0 {
+		columnSelectionInterfaces := data.Get("column_selection").(*schema.Set).List()[0]
+		columnSelectionInterface := columnSelectionInterfaces.(map[string]interface{})
+		columnSelection = map[string]interface{}{
+			"type": columnSelectionInterface["type"].(string),
+			"includes": columnSelectionInterface["includes"].([]interface{}),
+		}
 	}
 
 	createObjectGroupRequest := &client.CreateObjectGroupRequest{
-		Name:             data.Get("name").(string),
-		SourceBucket:     data.Get("source_bucket").(string),
-		FilterJSON:       data.Get("filter_json").(string),
-		Format:           data.Get("format").(string),
-		Compression:      data.Get("compression").(string),
-		LiveEventsSqsArn: data.Get("live_events_sqs_arn").(string),
-		PartitionBy:      data.Get("partition_by").(string),
-		Pattern:          data.Get("pattern").(string),
-		IndexRetention:   data.Get("index_retention").(int),
-		ArrayFlattenDepth: array_flatten_cs,
-		ColumnRenames:    data.Get("column_renames").(map[string]interface{}),
+		Name:              data.Get("name").(string),
+		SourceBucket:      data.Get("source_bucket").(string),
+		FilterJSON:        data.Get("filter_json").(string),
+		Format:            data.Get("format").(string),
+		Compression:       data.Get("compression").(string),
+		LiveEventsSqsArn:  data.Get("live_events_sqs_arn").(string),
+		PartitionBy:       data.Get("partition_by").(string),
+		Pattern:           data.Get("pattern").(string),
+		IndexRetention:    data.Get("index_retention").(int),
+		ArrayFlattenDepth: arrayFlattenCS,
+		ColumnRenames:     data.Get("column_renames").(map[string]interface{}),
+		ColumnSelection:   columnSelection,
 	}
 
 	if err := c.CreateObjectGroup(ctx, createObjectGroupRequest); err != nil {
@@ -180,10 +211,12 @@ func resourceObjectGroupRead(ctx context.Context, data *schema.ResourceData, met
 	data.Set("pattern", resp.Pattern)
 	data.Set("source_bucket", resp.SourceBucket)
 
+	data.Set("column_selection", resp.ColumnSelection)
+
 	// "unlimited" flattening represented as "null" in the api, and as -1 in the terraform module
 	// because the terraform sdk doesn't support nil values in configs https://github.com/hashicorp/terraform-plugin-sdk/issues/261
 	// We represent "null" as an int pointer to nil in the code.
-	if (resp.ArrayFlattenDepth == nil) {
+	if resp.ArrayFlattenDepth == nil {
 		data.Set("array_flatten_depth", -1)
 	} else {
 		data.Set("array_flatten_depth", resp.ArrayFlattenDepth)

examples/main.tf

@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     chaossearch = {
-      versions = ["0.6.2"]
+      version = "0.8.0"
       source = "benzaita/chaossearch"
     }
   }
@@ -33,6 +33,14 @@ resource "chaossearch_object_group" "my-object-group" {
 
   partition_by = "<regex>"
   array_flatten_depth = -1
+
+  column_selection {
+    type = "whitelist"
+    includes = [
+      "host",
+      "source",
+    ]
+  }
 }
 
 resource "chaossearch_indexing_state" "my-object-group" {