Use storage.Inuse in compaction and minor fixes in README.md

Author: buraksezer

README.md

@@ -5,6 +5,8 @@
 Distributed, eventually consistent and in-memory key/value data store and cache. It can be used both as an embedded Go 
 library and as a language-independent service.
 
+With Olric, you can instantly create a fast, scalable, shared pool of RAM across a cluster of computers.
+
 ## At a glance
 
 * Designed to share some transient, approximate, fast-changing data between servers,
@@ -88,6 +90,7 @@ Olric is in early stages of development. The package API and client protocol may
 * Implements a fast and simple binary protocol,
 * Embeddable but can be used as a language-independent service with olricd,
 * GC-friendly storage engine,
+* O(1) running time for lookups,
 * Supports atomic operations,
 * Provides a lock implementation which can be used for non-critical purposes,
 * Different eviction policies: LRU, MaxIdleDuration and Time-To-Live(TTL),
@@ -96,20 +99,22 @@ Olric is in early stages of development. The package API and client protocol may
 * Provides best-effort consistency guarantees without being a complete CP solution,
 * Distributes load fairly among cluster members with a [consistent hash function](https://github.com/buraksezer/consistent),
 * Supports replication by default(with sync and async options),
+* Quorum-based voting for replica control,
+* Repairs itself after a network partitioning,
 * Thread-safe by default,
 * Provides a command-line-interface to access the cluster directly from the terminal,
-* Very simple package API,
-* Offers a built-in Go client,
-* Gob, JSON and MessagePack are supportted by default as serialization format,
-* Simplicity as a key concept with a small set of features.
+* Supports different serialization formats. Gob, JSON and MessagePack are supported out of the box,
+* Provides a locking primitive which inspired by [SETNX of Redis](https://redis.io/commands/setnx#design-pattern-locking-with-codesetnxcode).
+
+See [Architecture](#architecture) section to see details.
 
 ## Planned Features
 
 * On-disk persistence mode to work beyond RAM,
 * Anti-entropy system to repair inconsistencies in DMaps,
 * Publish/Subscribe for messaging,
 * Eviction listeners by using Publish/Subscribe,
-* Memcache interface,
+* Memcached interface,
 * Client implementations for different languages: Java, Python and JavaScript,
 * REST API.
 

dmap_eviction.go

@@ -19,7 +19,6 @@ import (
 	"math/rand"
 	"runtime"
 	"sort"
-	"sync"
 	"time"
 
 	"github.com/buraksezer/olric/internal/storage"
@@ -60,32 +59,22 @@ func (db *Olric) evictKeysAtBackground() {
 }
 
 func (db *Olric) evictKeys() {
-	dmCount := 0
 	partID := uint64(rand.Intn(int(db.config.PartitionCount)))
 	part := db.partitions[partID]
-	var wg sync.WaitGroup
 	part.m.Range(func(name, tmp interface{}) bool {
 		dm := tmp.(*dmap)
-		// Picks 20 dmap instances randomly to check out expired keys. Then waits until all the goroutines done.
-		dmCount++
-		if dmCount >= 20 {
-			return false
-		}
-		wg.Add(1)
-		go db.scanDMapForEviction(partID, name.(string), dm, &wg)
+		db.scanDMapForEviction(partID, name.(string), dm)
 		return true
 	})
-	wg.Wait()
 }
 
-func (db *Olric) scanDMapForEviction(partID uint64, name string, dm *dmap, wg *sync.WaitGroup) {
+func (db *Olric) scanDMapForEviction(partID uint64, name string, dm *dmap) {
 	/*
 		From Redis Docs:
 			1- Test 20 random keys from the set of keys with an associated expire.
 			2- Delete all the keys found expired.
 			3- If more than 25% of keys were expired, start again from step 1.
 	*/
-	defer wg.Done()
 
 	// We need limits to prevent CPU starvation. delKeyVal does some network operation
 	// to delete keys from the backup nodes and the previous owners.

internal/storage/compaction.go

@@ -32,7 +32,7 @@ func (s *Storage) CompactTables() bool {
 			err := fresh.putRaw(hkey, vdata)
 			if err == errNotEnoughSpace {
 				// Create a new table and put the new k/v pair in it.
-				nt := newTable(fresh.allocated * 2)
+				nt := newTable(s.Inuse() * 2)
 				s.tables = append(s.tables, nt)
 				return false
 			}

internal/storage/storage.go

@@ -73,7 +73,7 @@ func (s *Storage) PutRaw(hkey uint64, value []byte) error {
 		err := t.putRaw(hkey, value)
 		if err == errNotEnoughSpace {
 			// Create a new table and put the new k/v pair in it.
-			nt := newTable(t.inuse * 2)
+			nt := newTable(s.Inuse() * 2)
 			s.tables = append(s.tables, nt)
 			res = ErrFragmented
 			// try again
@@ -102,7 +102,7 @@ func (s *Storage) Put(hkey uint64, value *VData) error {
 		err := t.put(hkey, value)
 		if err == errNotEnoughSpace {
 			// Create a new table and put the new k/v pair in it.
-			nt := newTable(t.inuse * 2)
+			nt := newTable(s.Inuse() * 2)
 			s.tables = append(s.tables, nt)
 			res = ErrFragmented
 			// try again
@@ -229,12 +229,7 @@ func (s *Storage) Delete(hkey uint64) error {
 	t := s.tables[0]
 	if float64(t.allocated)*maxGarbageRatio <= float64(t.garbage) {
 		// Create a new table here.
-		newSize := t.inuse * 2
-		if newSize > t.allocated {
-			// Don't grow up.
-			newSize = t.allocated
-		}
-		nt := newTable(newSize)
+		nt := newTable(s.Inuse() * 2)
 		s.tables = append(s.tables, nt)
 		return ErrFragmented
 	}

internal/storage/table.go

@@ -52,6 +52,14 @@ func newTable(size int) *table {
 		hkeys:     make(map[uint64]int),
 		allocated: size,
 	}
+	//  From builtin.go:
+	//
+	//  The size specifies the length. The capacity of the slice is
+	//	equal to its length. A second integer argument may be provided to
+	//	specify a different capacity; it must be no smaller than the
+	//	length. For example, make([]int, 0, 10) allocates an underlying array
+	//	of size 10 and returns a slice of length 0 and capacity 10 that is
+	//	backed by this underlying array.
 	t.memory = make([]byte, size)
 	return t
 }
@@ -127,7 +135,7 @@ func (t *table) getRaw(hkey uint64) ([]byte, bool) {
 
 	// In-memory structure:
 	// 1                 | klen       | 8           | 8                  | 4                    | vlen
-	// KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | Timestamp(uint64) | VALUE-LENGTH(uint32) | VALUE(bytes)
+	// KEY-LENGTH(uint8) | KEY(bytes) | TTL(uint64) | Timestamp(uint64)  | VALUE-LENGTH(uint32) | VALUE(bytes)
 	klen := int(uint8(t.memory[end]))
 	end++       // One byte to keep key length
 	end += klen // Key length