Use the new storethehash with index GC

gammazero · gammazero · commit 18dcc293501c · 2022-04-08T04:55:29.000-07:00
diff --git a/go.mod b/go.mod
@@ -7,8 +7,8 @@ require (
 	github.com/gammazero/keymutex v0.0.2
 	github.com/gammazero/radixtree v0.2.5
 	github.com/ipfs/go-cid v0.1.0
-	github.com/ipfs/go-log/v2 v2.5.0
-	github.com/ipld/go-storethehash v0.0.2
+	github.com/ipfs/go-log/v2 v2.5.1
+	github.com/ipld/go-storethehash v0.1.0
 	github.com/libp2p/go-libp2p-core v0.14.0
 	github.com/multiformats/go-multihash v0.1.0
 	go.opencensus.io v0.23.0
diff --git a/go.sum b/go.sum
@@ -128,8 +128,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC
 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k=
 github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc=
 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
-github.com/ipfs/go-log/v2 v2.5.0 h1:+MhAooFd9XZNvR0i9FriKW6HB0ql7HNXUuflWtc0dd4=
-github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
+github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY=
+github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI=
 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
@@ -140,8 +140,8 @@ github.com/ipld/go-car v0.1.0 h1:AaIEA5ITRnFA68uMyuIPYGM2XXllxsu8sNjFJP797us=
 github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g=
 github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w=
 github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU=
-github.com/ipld/go-storethehash v0.0.2 h1:PFAlcTG2RoS6Cwbes3J/X/NvuNTj+wgF1x0OED4x6bw=
-github.com/ipld/go-storethehash v0.0.2/go.mod h1:w8cQfWInks8lvvbQTiKbCPusU9v0sqiViBihTHbavpQ=
+github.com/ipld/go-storethehash v0.1.0 h1:HBa59pCuL33fLN8PekX+lq6vq1UlWCFxtfyLiN030ng=
+github.com/ipld/go-storethehash v0.1.0/go.mod h1:d3m79FZzARANjjBwZ1rNgyGUK+1M5m0SLHrarmJzcYo=
 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jbenet/go-cienv v0.0.0-20150120210510-1bb1476777ec/go.mod h1:rGaEvXB4uRSZMmzKNLoXvTu1sfx+1kv/DojUlPrSZGs=
diff --git a/store/storethehash/option.go b/store/storethehash/option.go
@@ -11,6 +11,7 @@ const (
 	defaultBurstRate     = 4 * 1024 * 1024
 	defaultIndexSizeBits = uint8(24)
 	defaultSyncInterval  = time.Second
+	defaultGCInterval    = 30 * time.Minute
 )
 
 // config contains all options for configuring storethehash valuestore.
@@ -44,6 +45,11 @@ func SyncInterval(syncInterval time.Duration) Option {
 func BurstRate(burstRate uint64) Option {
 	return func(cfg *config) {
 		cfg.burstRate = sthtypes.Work(burstRate)
+	}
+}
 
+func GCInterval(gcInterval time.Duration) Option {
+	return func(cfg *config) {
+		cfg.gcInterval = gcInterval
 	}
 }
diff --git a/store/storethehash/storethehash.go b/store/storethehash/storethehash.go
@@ -61,10 +61,11 @@ func New(dir string, options ...Option) (indexer.Interface, error) {
 		indexSizeBits: defaultIndexSizeBits,
 		syncInterval:  defaultSyncInterval,
 		burstRate:     defaultBurstRate,
+		gcInterval:    defaultGCInterval,
 	}
 	cfg.apply(options)
 
-	s, err := sth.OpenStore(indexPath, primary, cfg.indexSizeBits, cfg.syncInterval, cfg.burstRate)
+	s, err := sth.OpenStore(indexPath, primary, cfg.indexSizeBits, cfg.syncInterval, cfg.burstRate, cfg.gcInterval)
 	if err != nil {
 		return nil, err
 	}
@@ -180,22 +181,17 @@ func (s *sthStorage) RemoveProviderContext(providerID peer.ID, contextID []byte)
 }
 
 func (s *sthStorage) Size() (int64, error) {
-	var size int64
-	fi, err := os.Stat(filepath.Join(s.dir, "storethehash.data"))
-	if err != nil {
-		return 0, err
-	}
-	size += fi.Size()
-	fi, err = os.Stat(filepath.Join(s.dir, "storethehash.index"))
+	size, err := s.store.IndexStorageSize()
 	if err != nil {
 		return 0, err
 	}
-	size += fi.Size()
-	fi, err = os.Stat(filepath.Join(s.dir, "storethehash.index.free"))
+
+	fi, err := os.Stat(filepath.Join(s.dir, "storethehash.data"))
 	if err != nil {
 		return 0, err
 	}
 	size += fi.Size()
+
 	return size, nil
 }
 
diff --git a/version.json b/version.json
@@ -1,3 +1,3 @@
 {
-  "version": "v0.2.9"
+  "version": "v0.2.10"
 }

Original file line number	Diff line number	Diff line change
`@@ -11,6 +11,7 @@ const (`
`11`	`11`	`defaultBurstRate = 4 * 1024 * 1024`
`12`	`12`	`defaultIndexSizeBits = uint8(24)`
`13`	`13`	`defaultSyncInterval = time.Second`
	`14`	`+ defaultGCInterval = 30 * time.Minute`
`14`	`15`	`)`
`15`	`16`
`16`	`17`	`// config contains all options for configuring storethehash valuestore.`
`@@ -44,6 +45,11 @@ func SyncInterval(syncInterval time.Duration) Option {`
`44`	`45`	`func BurstRate(burstRate uint64) Option {`
`45`	`46`	`return func(cfg *config) {`
`46`	`47`	`cfg.burstRate = sthtypes.Work(burstRate)`
	`48`	`+ }`
	`49`	`+}`
`47`	`50`
	`51`	`+func GCInterval(gcInterval time.Duration) Option {`
	`52`	`+ return func(cfg *config) {`
	`53`	`+ cfg.gcInterval = gcInterval`
`48`	`54`	`}`
`49`	`55`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`{`
`2`		`- "version": "v0.2.9"`
	`2`	`+ "version": "v0.2.10"`
`3`	`3`	`}`