initial import

Signed-off-by: Vasiliy Tolstov <v.tolstov@selfip.ru>

Author: vtolstov

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016-2018 sdstack and other authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+storage
+=======
+

api/api.go

@@ -0,0 +1,58 @@
+package api
+
+/*
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sdstack/storage/kv"
+)
+
+var apiTypes map[string]Api
+
+func init() {
+	apiTypes = make(map[string]Api)
+}
+
+func RegisterApi(engine string, api Api) {
+	apiTypes[engine] = proxy
+}
+
+// Proxy represents proxy interface
+type Proxy interface {
+	Start() error
+	Stop() error
+	Configure(*kv.KV, interface{}) error
+	//	Format() error
+	//	Check() error
+	//	Recover() error
+	//	Info() (*Info, error)
+	//	Snapshot() error
+	//	Reweight() error
+	//	Members() []Member
+}
+
+func New(ptype string, cfg interface{}, engine *kv.KV) (Proxy, error) {
+	var err error
+
+	proxy, ok := proxyTypes[ptype]
+	if !ok {
+		return nil, fmt.Errorf("unknown proxy type %s, only %s supported", ptype, strings.Join(ProxyTypes(), ","))
+	}
+
+	err = proxy.Configure(engine, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return proxy, nil
+}
+
+func ProxyTypes() []string {
+	var ptypes []string
+	for ptype, _ := range proxyTypes {
+		ptypes = append(ptypes, ptype)
+	}
+	return ptypes
+}
+*/

backend/backend.go

@@ -0,0 +1,87 @@
+package backend
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+/*
+You could do "files" as variable length blocks, where you have a list of blocks needed to reconstruct. To insert data you create a new block, keep the original, but specify that you only need parts of the data.
+A file is a list of blocks. When you insert new data, you create a new block, and adjust start/end of affected blocks.
+This allows you to insert data at the same speed as end-of-file writes. You can add some lazy cleanup, that removes unused parts of blocks and recompresses them, but the important part is that you do not need to do that when the file is updated.
+There will only be a read overhead when "start" of a block is >0. That will be the same for using "ReadAt". You skip in your blocks until you get to the position you want and start decoding from there.
+If blocks are stored based on their uncompressed hash, you furthermore get a coarse high-level deduplication, though that of course makes it more difficult to figure out when a block is no longer used by any file.
+With that, you should get pretty good performance, even with block sizes up to 16-64MB.
+For each block, I would probably go with variably sized de-duplication, with a 4K average, and deflate the result @level 1. That should give you ~50-60MB/core throughput. If you need higher write throughput, you can always set deflate to level 0 (store), and leave it up to a lazy task to compress the data. That should give you about 150MB/core. Obviously you can de-duplicate/compress several blocks in parallel.
+*/
+
+/*
+type File struct {
+	Name   string
+	Size   uint64
+	Blocks []Block
+	Ring   *config.Ring
+}
+
+type Block struct {
+	Start uint64
+	Stop  uint64
+	Size  uint64
+}
+*/
+
+var (
+	ErrIO = errors.New("IO error")
+)
+
+var backendTypes map[string]Backend
+
+func init() {
+	backendTypes = make(map[string]Backend)
+}
+
+func RegisterBackend(engine string, backend Backend) {
+	backendTypes[engine] = backend
+}
+
+func New(btype string, cfg interface{}) (Backend, error) {
+	var err error
+
+	backend, ok := backendTypes[btype]
+	if !ok {
+		return nil, fmt.Errorf("unknown backend type %s. only %s supported", btype, strings.Join(BackendTypes(), ","))
+	}
+
+	if cfg == nil {
+		return backend, nil
+	}
+
+	err = backend.Init(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return backend, nil
+}
+
+func BackendTypes() []string {
+	var btypes []string
+	for btype, _ := range backendTypes {
+		btypes = append(btypes, btype)
+	}
+	return btypes
+}
+
+type Backend interface {
+	Configure(interface{}) error
+	Init(interface{}) error
+	ReaderFrom(string, io.Reader, int64, int64, int, int) (int64, error)
+	WriterTo(string, io.Writer, int64, int64, int, int) (int64, error)
+	WriteAt(string, []byte, int64, int, int) (int, error)
+	ReadAt(string, []byte, int64, int, int) (int, error)
+	Allocate(string, int64, int, int) error
+	Remove(string, int, int) error
+	Exists(string, int, int) (bool, error)
+}

backend/block/block.go

@@ -0,0 +1,100 @@
+package block
+
+/*
+import (
+	"fmt"
+	"os"
+
+	"github.com/mitchellh/mapstructure"
+	"golang.org/x/sys/unix"
+	"github.com/sdstack/storage/backend"
+)
+
+type StoreBlock struct {
+	store []string
+}
+
+type ObjectBlock struct {
+	fs *StoreBlock
+	fp *os.File
+}
+
+func init() {
+	backend.RegisterBackend("block", &StoreBlock{})
+}
+func (s *StoreBlock) Init(data interface{}) error {
+	var err error
+
+	err = mapstructure.Decode(data, &s.store)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%#+v\n", s)
+	return nil
+}
+
+func (s *StoreBlock) Configure(data interface{}) error {
+	var err error
+
+	err = mapstructure.Decode(data, &s.store)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%#+v\n", s)
+	return nil
+}
+
+func (s *StoreBlock) Open() error {
+	return nil
+}
+
+func (s *StoreBlock) Close() error {
+	return nil
+}
+
+func (o *ObjectBlock) Allocate(mode uint64, offset uint64, length uint64) error {
+	return unix.Fallocate(int(o.fp.Fd()), uint32(mode), int64(offset), int64(length))
+}
+
+func (o *ObjectBlock) Delete(flags uint64) error {
+	return nil
+	//	return os.Remove(filepath.Join())
+}
+
+func (o *ObjectBlock) ReadAt(data []byte, offset uint64, flags uint64) (uint64, error) {
+	return 0, nil
+}
+
+func (o *ObjectBlock) WriteAt(data []byte, offset uint64, flags uint64) (uint64, error) {
+	return 0, nil
+}
+
+func (o *ObjectBlock) Read(data []byte) (int, error) {
+	return o.fp.Read(data)
+}
+
+func (o *ObjectBlock) Write(data []byte) (int, error) {
+	return o.fp.Write(data)
+}
+
+func (o *ObjectBlock) Seek(offset int64, whence int) (int64, error) {
+	return o.fp.Seek(offset, whence)
+}
+
+func (o *StoreBlock) ObjectExists(name string) (bool, error) {
+	_, err := os.Stat(name)
+	return os.IsNotExist(err), nil
+}
+
+func (o *ObjectBlock) Sync() error {
+	return o.fp.Sync()
+}
+
+func (o *ObjectBlock) Close() error {
+	return o.fp.Close()
+}
+
+func (o *ObjectBlock) Remove() error {
+	return nil
+}
+*/