Merge pull request #24 from yannh/fs-cache

Cache schemas downloaded over HTTP

Author: yannh

Readme.md

@@ -13,7 +13,7 @@ It is inspired by, contains code from and is designed to stay close to
  * **high performance**: will validate & download manifests over multiple routines, caching
    downloaded files in memory
  * configurable list of **remote, or local schemas locations**, enabling validating Kubernetes
-   custom resources (CRDs)
+   custom resources (CRDs) and offline validation capabilities.
 
 ### A small overview of Kubernetes manifest validation
 
@@ -49,6 +49,10 @@ configuration errors.
 ```
 $ ./bin/kubeconform -h
 Usage: ./bin/kubeconform [OPTION]... [FILE OR FOLDER]...
+  -cache string
+        cache schemas downloaded via HTTP to this folder
+  -cpu-prof string
+        debug - log CPU profiling to file
   -exit-on-error
         immediately stop execution when the first error is encountered
   -h    show help information

acceptance.bats

@@ -202,3 +202,17 @@
   [ "$status" -eq 0 ]
   [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
 }
+
+@test "Pass when parsing a valid Kubernetes config YAML file and store cache" {
+  run mkdir cache
+  run bin/kubeconform -cache cache -summary fixtures/valid.yaml
+  [ "$status" -eq 0 ]
+  [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
+  [ "`ls cache/ | wc -l`" -eq 1 ]
+}
+
+@test "Fail when cache folder does not exist" {
+  run bin/kubeconform -cache cache_does_not_exist -summary fixtures/valid.yaml
+  [ "$status" -eq 1 ]
+  [ "$output" = "failed opening cache folder cache_does_not_exist: stat cache_does_not_exist: no such file or directory" ]
+}

cmd/kubeconform/main.go

@@ -82,6 +82,7 @@ func realMain() int {
 	}
 
 	v, err := validator.New(cfg.SchemaLocations, validator.Opts{
+		Cache:                cfg.Cache,
 		SkipTLS:              cfg.SkipTLS,
 		SkipKinds:            cfg.SkipKinds,
 		RejectKinds:          cfg.RejectKinds,

pkg/cache/cache.go

@@ -0,0 +1,6 @@
+package cache
+
+type Cache interface {
+	Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error)
+	Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error
+}

pkg/cache/inmemory.go

@@ -0,0 +1,49 @@
+package cache
+
+import (
+	"fmt"
+	"sync"
+)
+
+// SchemaCache is a cache for downloaded schemas, so each file is only retrieved once
+// It is different from pkg/registry/http_cache.go in that:
+//   - This cache caches the parsed Schemas
+type inMemory struct {
+	sync.RWMutex
+	schemas map[string]interface{}
+}
+
+// New creates a new cache for downloaded schemas
+func NewInMemoryCache() Cache {
+	return &inMemory{
+		schemas: map[string]interface{}{},
+	}
+}
+
+func key(resourceKind, resourceAPIVersion, k8sVersion string) string {
+	return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion)
+}
+
+// Get retrieves the JSON schema given a resource signature
+func (c *inMemory) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) {
+	k := key(resourceKind, resourceAPIVersion, k8sVersion)
+	c.RLock()
+	defer c.RUnlock()
+	schema, ok := c.schemas[k]
+
+	if ok == false {
+		return nil, fmt.Errorf("schema not found in in-memory cache")
+	}
+
+	return schema, nil
+}
+
+// Set adds a JSON schema to the schema cache
+func (c *inMemory) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error {
+	k := key(resourceKind, resourceAPIVersion, k8sVersion)
+	c.Lock()
+	defer c.Unlock()
+	c.schemas[k] = schema
+
+	return nil
+}

pkg/cache/ondisk.go

@@ -0,0 +1,48 @@
+package cache
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sync"
+)
+
+type onDisk struct {
+	sync.RWMutex
+	folder string
+}
+
+// New creates a new cache for downloaded schemas
+func NewOnDiskCache(cache string) Cache {
+	return &onDisk{
+		folder: cache,
+	}
+}
+
+func cachePath(folder, resourceKind, resourceAPIVersion, k8sVersion string) string {
+	hash := md5.Sum([]byte(fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion)))
+	return path.Join(folder, hex.EncodeToString(hash[:]))
+}
+
+// Get retrieves the JSON schema given a resource signature
+func (c *onDisk) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) {
+	c.RLock()
+	defer c.RUnlock()
+
+	f, err := os.Open(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion))
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutil.ReadAll(f)
+}
+
+// Set adds a JSON schema to the schema cache
+func (c *onDisk) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error {
+	c.Lock()
+	defer c.Unlock()
+	return ioutil.WriteFile(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion), schema.([]byte), 0644)
+}

pkg/cache/schemacache.go

@@ -9,6 +9,7 @@ import (
 )
 
 type Config struct {
+	Cache                  string
 	CPUProfileFile         string
 	ExitOnError            bool
 	Files                  []string
@@ -75,6 +76,7 @@ func FromFlags(progName string, args []string) (Config, string, error) {
 	flags.StringVar(&c.OutputFormat, "output", "text", "output format - json, tap, text")
 	flags.BoolVar(&c.Verbose, "verbose", false, "print results for all resources (ignored for tap output)")
 	flags.BoolVar(&c.SkipTLS, "insecure-skip-tls-verify", false, "disable verification of the server's SSL certificate. This will make your HTTPS connections insecure")
+	flags.StringVar(&c.Cache, "cache", "", "cache schemas downloaded via HTTP to this folder")
 	flags.StringVar(&c.CPUProfileFile, "cpu-prof", "", "debug - log CPU profiling to file")
 	flags.BoolVar(&c.Help, "h", false, "show help information")
 	flags.Usage = func() {

pkg/config/config.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"time"
+
+	"github.com/yannh/kubeconform/pkg/cache"
 )
 
 type httpGetter interface {
@@ -16,10 +19,11 @@ type httpGetter interface {
 type SchemaRegistry struct {
 	c                  httpGetter
 	schemaPathTemplate string
+	cache              cache.Cache
 	strict             bool
 }
 
-func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *SchemaRegistry {
+func newHTTPRegistry(schemaPathTemplate string, cacheFolder string, strict bool, skipTLS bool) (*SchemaRegistry, error) {
 	reghttp := &http.Transport{
 		MaxIdleConns:       100,
 		IdleConnTimeout:    3 * time.Second,
@@ -30,11 +34,25 @@ func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *Sche
 		reghttp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
 
+	var filecache cache.Cache = nil
+	if cacheFolder != "" {
+		fi, err := os.Stat(cacheFolder)
+		if err != nil {
+			return nil, fmt.Errorf("failed opening cache folder %s: %s", cacheFolder, err)
+		}
+		if !fi.IsDir() {
+			return nil, fmt.Errorf("cache folder %s is not a directory", err)
+		}
+
+		filecache = cache.NewOnDiskCache(cacheFolder)
+	}
+
 	return &SchemaRegistry{
 		c:                  &http.Client{Transport: reghttp},
 		schemaPathTemplate: schemaPathTemplate,
+		cache:              filecache,
 		strict:             strict,
-	}
+	}, nil
 }
 
 // DownloadSchema downloads the schema for a particular resource from an HTTP server
@@ -44,6 +62,12 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers
 		return nil, err
 	}
 
+	if r.cache != nil {
+		if b, err := r.cache.Get(resourceKind, resourceAPIVersion, k8sVersion); err == nil {
+			return b.([]byte), nil
+		}
+	}
+
 	resp, err := r.c.Get(url)
 	if err != nil {
 		return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err)
@@ -63,5 +87,11 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers
 		return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err)
 	}
 
+	if r.cache != nil {
+		if err := r.cache.Set(resourceKind, resourceAPIVersion, k8sVersion, body); err != nil {
+			return nil, fmt.Errorf("failed writing schema to cache: %s", err)
+		}
+	}
+
 	return body, nil
 }

pkg/registry/http.go

@@ -12,11 +12,11 @@ type LocalRegistry struct {
 }
 
 // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames
-func newLocalRegistry(pathTemplate string, strict bool) *LocalRegistry {
+func newLocalRegistry(pathTemplate string, strict bool) (*LocalRegistry, error) {
 	return &LocalRegistry{
 		pathTemplate,
 		strict,
-	}
+	}, nil
 }
 
 // DownloadSchema retrieves the schema from a file for the resource