Merge pull request #21 from Zondax/ezequiel/local_data

Add local backend for data store

Author: Ezequiel Raynaudo

.circleci/config.yml

@@ -0,0 +1,40 @@
+name: Build jobs
+
+on:
+  push:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+      - name: Build
+        run: |
+          make build
+
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16'
+      - run: make build
+      - name: ModTidy check
+        run: make check-modtidy
+      - name: Lint check
+        run: |
+          make install_lint
+          make lint

.github/workflows/main.yml

@@ -1,4 +1,4 @@
 # zindexer - Common Components
 
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
-[![CircleCI](https://circleci.com/gh/Zondax/zindexer/tree/main.svg?style=shield&circle-token=82a6e7f3ee0cfe2c1438e1f3dc25d337951e9402)](https://circleci.com/gh/Zondax/zindexer/tree/main)
+![example workflow](https://github.com/Zondax/zindexer/actions/workflows/main.yml/badge.svg?branch=main)

README.md

@@ -2,13 +2,23 @@ package data_store
 
 import (
 	"fmt"
+	"go.uber.org/zap"
+)
+
+const (
+	MinIOStorage = "minio"
+	LocalStorage = "local"
 )
 
 func NewDataStoreClient(config DataStoreConfig) (DataStoreClient, error) {
+	zap.S().Infof("[DataStore] - Creating client for service '%s'", config.Service)
 	switch config.Service {
 	case MinIOStorage:
 		client, err := newMinioClient(config)
 		return DataStoreClient{client}, err
+	case LocalStorage:
+		client, err := newLocalClient(config)
+		return DataStoreClient{client}, err
 	default:
 		return DataStoreClient{}, fmt.Errorf("unrecognized data store service '%s'", config.Service)
 	}

connections/data_store/builder.go

@@ -4,6 +4,7 @@ type IDataStoreClient interface {
 	GetFile(name string, location string) (*[]byte, error)
 	UploadFile(name string, dest string) error
 	List(dir string, prefix string) *[]string
+	StorageType() string
 }
 
 type DataStoreClient struct {
@@ -15,4 +16,5 @@ type DataStoreConfig struct {
 	User     string
 	Password string
 	Service  string
+	DataPath string
 }

connections/data_store/interface.go

@@ -0,0 +1,77 @@
+package data_store
+
+import (
+	"fmt"
+	"go.uber.org/zap"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+)
+
+type LocalClient struct {
+	DataPath string
+}
+
+func newLocalClient(config DataStoreConfig) (LocalClient, error) {
+	return LocalClient{DataPath: config.DataPath}, nil
+}
+
+func (c LocalClient) GetFile(object string, bucket string) (*[]byte, error) {
+	targetObject := fmt.Sprintf("%s/%s/%s", c.DataPath, bucket, object)
+	data, err := ioutil.ReadFile(targetObject)
+	if err != nil {
+		zap.S().Errorf("err when getting object from store: %v", err.Error())
+		return nil, err
+	}
+
+	return &data, nil
+}
+
+func (c LocalClient) List(bucket string, prefix string) *[]string {
+	var list []string
+	files, err := os.ReadDir(fmt.Sprintf("%s/%s", c.DataPath, bucket))
+	if err != nil {
+		zap.S().Errorf("could not read directory '%s': %v", bucket, err)
+		return &list
+	}
+
+	for _, file := range files {
+		fileName := file.Name()
+		if strings.Contains(fileName, prefix) {
+			list = append(list, fileName)
+		}
+	}
+
+	return &list
+}
+
+func (c LocalClient) UploadFile(name string, dest string) error {
+	file, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	fileStat, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	if !fileStat.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", name)
+	}
+
+	out, err := os.Create(fmt.Sprintf("%s/%s/%s", c.DataPath, dest, name))
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, file)
+	return err
+}
+
+func (c LocalClient) StorageType() string {
+	return LocalStorage
+}

connections/data_store/local.go

@@ -9,8 +9,6 @@ import (
 	"os"
 )
 
-const MinIOStorage = "minio"
-
 type MinioClient struct {
 	client *minio.Client
 }
@@ -76,3 +74,7 @@ func (c MinioClient) UploadFile(name string, dest string) error {
 	}
 	return nil
 }
+
+func (c MinioClient) StorageType() string {
+	return MinIOStorage
+}

connections/data_store/minio.go

@@ -8,12 +8,12 @@ require (
 	github.com/eapache/queue v1.1.0
 	github.com/gorilla/mux v1.8.0
 	github.com/hasura/go-graphql-client v0.2.0
-	github.com/minio/minio-go/v7 v7.0.14
+	github.com/minio/minio-go/v7 v7.0.15
 	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.1
 	go.mongodb.org/mongo-driver v1.7.2
 	go.uber.org/zap v1.19.1
-	gorm.io/driver/postgres v1.0.6
-	gorm.io/gorm v1.21.15
+	gorm.io/driver/postgres v1.1.2
+	gorm.io/gorm v1.21.16
 )