New Plugin: Aws Cloud Control API

.github/workflows/tests.yml

@@ -18,6 +18,10 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.18
+
+    # TODO add AWS credentials to generate 
+    # - name: Generate
+    #   run: go generate -v ./...
 
     - name: Build
       run: go build -v ./...

.gitignore

@@ -26,4 +26,5 @@ consul_input.yaml
 # Go workspace file
 go.work
 
-ignore
+ignore.DS_Store
+lib/.DS_Store

.goreleaser.yml

@@ -2,6 +2,10 @@
 builds: 
   -
     binary: surf
+    # TODO add AWS credentials before can generate
+    # hooks:
+    #   pre: go generate ./...
+
 brews:
   -
     name: surf

README.md

@@ -1,3 +1,4 @@
+
 # SURF
 
 Free Text Search across your infrastructure platforms via CLI.
@@ -12,6 +13,7 @@ S.U.R.F is an acronym for: `Search-Unified-Recursive-Fast`
 # Supported Platforms
 
 - [X] [AWS Route53 and Targets](https://github.com/Isan-Rivkin/route53-cli)
+- [X] [AWS CloudControl](https://aws.amazon.com/cloudcontrolapi/)
 - [X] [AWS ACM](https://aws.amazon.com/certificate-manager/)
 - [X] [AWS S3](https://aws.amazon.com/s3/)
 - [X] [AWS DynamoDB](https://aws.amazon.com/dynamodb/)
@@ -23,24 +25,29 @@ S.U.R.F is an acronym for: `Search-Unified-Recursive-Fast`
 
 # Table of Contents 
 
+- [SURF](#surf)
+- [Supported Platforms](#supported-platforms)
+- [Table of Contents](#table-of-contents)
 - [Overview](#overview)
-- [Examples](#usage-examples)
-  * [AWS Route53 Usage](#aws-route53-usage)
-  * [AWS ACM Usage](#aws-acm-usage)
-  * [AWS DynamoDB Usage](#aws-dynamodb-usage)
-  * [AWS S3 Usage](#aws-s3-usage)
-  * [Hashicorp Vault Usage](#hashicorp-vault-usage)
-  * [Hashicorp Consul Usage](#hashicorp-consul-usage)
-  * [ElasticSearch / OpenSearch Usage](#elasticsearch-and-opensearch-usage)
-  * [Logz.io Usage](#logzio-usage)
+- [Usage Examples](#usage-examples)
+  - [AWS Route53 Usage](#aws-route53-usage)
+  - [AWS Cloud Control Usage](#aws-cloud-control-usage)
+  - [AWS ACM Usage](#aws-acm-usage)
+  - [AWS DynamoDB Usage](#aws-dynamodb-usage)
+  - [AWS S3 Usage](#aws-s3-usage)
+  - [Hashicorp Vault Usage](#hashicorp-vault-usage)
+  - [Hashicorp Consul Usage](#hashicorp-consul-usage)
+  - [ElasticSearch and OpenSearch Usage](#elasticsearch-and-opensearch-usage)
+  - [Logz.io Usage](#logzio-usage)
 - [Install](#install)
-    + [Brew](#brew)
-    + [Download Binary](#download-binary)
-    + [Install from Source](#install-from-source)
+    - [Brew](#brew)
+    - [Download Binary](#download-binary)
+    - [Install from Source](#install-from-source)
 - [Authentication](#authentication)
-  * [Supported Authentication Methods](#supported-authentication-methods)
+  - [Supported Authentication Methods](#supported-authentication-methods)
 - [Version check](#version-check)
 - [How it Works](#how-it-works)
+- [Contributors](#contributors)
 
 # Overview
 
@@ -60,6 +67,48 @@ Based on [AWS Route53](https://github.com/Isan-Rivkin/route53-cli): Search what'
 surf r53 -q api.my-corp.com
 ```
 
+## AWS Cloud Control Usage 
+
+Cloud Control API allows searching across AWS resources in a resource agnostic manner. 
+All resource types has CRUD and based on REST so we can list, get, create, delete.
+
+The Supported resource types are auto generated into the code during every build see [cloudformation_resources.gen.go](./lib/awsu/cloudformationgenerated/cloudformation_resources.gen.go).
+
+**List Supported resource types**
+
+Note: certain resources require additional fields to be specified, such as `AWS::EKS::Addon`
+And list-types will show the required fields (e.g `ClusterName`).
+
+```bash
+surf aws list-types
+```
+
+**List Resources**
+
+Example: List all EKS clusters: 
+
+```bash
+surf aws list --type eks::cluster
+```
+
+**Search existing resources**
+
+```bash
+surf aws search  -q <query>  -t [<resource-type>] -a [<field-key>=<field-val>]
+```
+
+Example search: resources containing the `my-app` *anywhere* and are of types `eks` or `vpc`:
+
+```bash
+surf aws search -q my-app -t eks -t vpc -a `ClusterName=MyClusterName`
+```
+
+Example Describe EKS cluster `my-cluster`:
+
+```bash
+surf aws get --type eks::cluster --id my-cluster
+```
+
 ## AWS ACM Usage 
 
 Search inside ACM Certificates in AWS.

cmd/aws-cloudcontrol-generator/main.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"go/format"
+	"io/ioutil"
+	"text/template"
+	"time"
+
+	"github.com/isan-rivkin/surf/lib/awsu"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	templateData = `package cloudformationgenerated
+
+/*
+THIS FILE WAS AUTO GENERATED AT {{ .now }}
+
+Do not edit this file, run "go generate" to re-generate this file with an
+updated resources of CloudFormation list.
+*/
+
+// resource types https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html
+type generatedCCResourceProperty struct {
+	//service-provider::service-name::data-type-name
+	ServiceProvider          string
+	ServiceName              string
+	DataTypeName             string
+	ResourceProvisioningType string
+}
+// Define the main struct
+type resourceSchema struct {
+	TypeIdentifier           string
+	RawSchemaJson            string
+	AdditionalRequiredFields []string
+}
+
+var GenCloudformationProperties = []generatedCCResourceProperty{ 
+	{{ range $i, $r := .resources -}}
+	{
+		ServiceProvider: "{{ $r.ServiceProvider }}",
+		ServiceName: "{{ $r.ServiceName }}",
+		DataTypeName: "{{ $r.DataTypeName}}",
+		ResourceProvisioningType: "{{ $r.ResourceProvisioningType }}",
+	},
+	{{ end -}}
+}
+
+var GenCloudFormationResourceSchemas = map[string]resourceSchema{
+	{{- range $i, $r := .descriptions }}
+	"{{ $r.TypeIdentifier }}": {
+		AdditionalRequiredFields: []string{
+			{{- range $field := $r.AdditionalRequiredFields }}
+			"{{ $field }}",
+			{{- end }}
+		},
+	},
+	{{- end }}
+}
+`
+)
+
+func createFile(args map[string]interface{}) {
+	tmpl := template.Must(template.New("").Parse(templateData))
+	buf := bytes.Buffer{}
+
+	if err := tmpl.Execute(&buf, args); err != nil {
+		panic(err)
+	}
+
+	fileBytes, err := format.Source(buf.Bytes())
+	if err != nil {
+		panic(fmt.Errorf("formatting go source: %w", err))
+	}
+
+	newFile := "./lib/awsu/cloudformationgenerated/cloudformation_resources.gen.go"
+
+	_ = ioutil.WriteFile(newFile, fileBytes, 0644)
+	fmt.Printf("Generated file: %s\n", newFile)
+}
+
+func main() {
+	log.Info("running code generation for cloudformation resources")
+	auth, err := awsu.NewSessionInput("", "")
+	if err != nil {
+		panic(fmt.Errorf("creating session in AWS: %w", err))
+	}
+	cfClient, err := awsu.NewCloudFormation(auth)
+	if err != nil {
+		panic(fmt.Errorf("creating cloudformation client: %w", err))
+	}
+	cfApi := awsu.NewCloudFormationAPI(cfClient)
+	resp, err := cfApi.GetAllSupportedCloudControlAPIResources()
+	if err != nil {
+		panic(fmt.Errorf("getting all supported cloud control api resources: %w", err))
+	}
+	resources, err := resp.GetResources()
+	if err != nil {
+		panic(fmt.Errorf("parsing resources: %w", err))
+	}
+	// create channel in the size of resources
+	descriptions := make([]*awsu.ResourceSchema, 0, len(resources))
+	for _, r := range resources {
+		retryAttempts := 5
+		for i := 0; i < retryAttempts; i++ {
+			desc, err := cfApi.DescribeResourceType(r)
+			if err != nil {
+				if errors.Is(err, awsu.ErrCloudFormationRateLimit) {
+					// retry
+					time.Sleep(1 * time.Second)
+					fmt.Printf("retrying rate limit %d/%d '%s'\n", i, retryAttempts, r.String())
+					continue
+				} else {
+					panic(fmt.Errorf("describing resource type: %w", err))
+				}
+			}
+			resourceSchema, err := awsu.NewResourceSchemaFromDescribe(r, desc)
+			if err != nil {
+				panic(fmt.Errorf("creating resource schema from describe: %w", err))
+			}
+			fmt.Printf("Described Resource: %s Props %s\n", resourceSchema.AdditionalRequiredFields, r.String())
+			descriptions = append(descriptions, resourceSchema)
+			break
+		}
+	}
+
+	fmt.Println("done descriptions: ", len(descriptions))
+	args := map[string]interface{}{
+		"resources":    resources,
+		"descriptions": descriptions,
+		"now":          time.Now().Format("2006-01-02 15:04:05"),
+	}
+
+	createFile(args)
+}