Skip to content

Commit 4149a93

Browse files
asutulaasutula
authored
bucket archive improvements and limits (#457)
* bucket archive improvements and limits Signed-off-by: Aaron Sutula <hi@asutula.com> * default rep factor limit of 4 Signed-off-by: Aaron Sutula <hi@asutula.com> * naming nit Signed-off-by: Aaron Sutula <hi@asutula.com> * set max rep factor for tests Signed-off-by: Aaron Sutula <hi@asutula.com>
1 parent 669a051 commit 4149a93

File tree

8 files changed

+320
-307
lines changed

8 files changed

+320
-307
lines changed

‎api/apitest/apitest.go‎

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -41,22 +41,23 @@ func DefaultTextileConfig(t util.TestingTWithCleanup) core.Config {
4141
require.NoError(t, err)
4242

4343
return core.Config{
44-
Hub: true,
45-
Debug: true,
46-
AddrAPI: util.MustParseAddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort)),
47-
AddrAPIProxy: util.MustParseAddr("/ip4/0.0.0.0/tcp/0"),
48-
AddrMongoURI: MongoUri,
49-
AddrMongoName: util.MakeToken(12),
50-
AddrThreadsHost: util.MustParseAddr("/ip4/0.0.0.0/tcp/0"),
51-
AddrIPFSAPI: IPFSApiAddr,
52-
AddrGatewayHost: util.MustParseAddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", gatewayPort)),
53-
AddrGatewayURL: fmt.Sprintf("http://127.0.0.1:%d", gatewayPort),
54-
CustomerioAPIKey: os.Getenv("CUSTOMERIO_API_KEY"),
55-
CustomerioConfirmTmpl: os.Getenv("CUSTOMERIO_CONFIRM_TMPL"),
56-
CustomerioInviteTmpl: os.Getenv("CUSTOMERIO_INVITE_TMPL"),
57-
EmailSessionSecret: SessionSecret,
58-
SegmentAPIKey: os.Getenv("SEGMENT_API_KEY"),
59-
SegmentPrefix: "test_",
44+
Hub: true,
45+
Debug: true,
46+
AddrAPI: util.MustParseAddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", apiPort)),
47+
AddrAPIProxy: util.MustParseAddr("/ip4/0.0.0.0/tcp/0"),
48+
AddrMongoURI: MongoUri,
49+
AddrMongoName: util.MakeToken(12),
50+
AddrThreadsHost: util.MustParseAddr("/ip4/0.0.0.0/tcp/0"),
51+
AddrIPFSAPI: IPFSApiAddr,
52+
AddrGatewayHost: util.MustParseAddr(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", gatewayPort)),
53+
AddrGatewayURL: fmt.Sprintf("http://127.0.0.1:%d", gatewayPort),
54+
CustomerioAPIKey: os.Getenv("CUSTOMERIO_API_KEY"),
55+
CustomerioConfirmTmpl: os.Getenv("CUSTOMERIO_CONFIRM_TMPL"),
56+
CustomerioInviteTmpl: os.Getenv("CUSTOMERIO_INVITE_TMPL"),
57+
EmailSessionSecret: SessionSecret,
58+
SegmentAPIKey: os.Getenv("SEGMENT_API_KEY"),
59+
SegmentPrefix: "test_",
60+
BucketArchiveMaxRepFactor: 4,
6061
}
6162
}
6263

‎api/bucketsd/pb/bucketsd.pb.go‎

Lines changed: 227 additions & 236 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/bucketsd/pb/bucketsd.proto‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,9 @@ message ArchiveConfig {
193193
repeated string trusted_miners = 4;
194194
repeated string country_codes = 5;
195195
ArchiveRenew renew = 6;
196-
string addr = 7;
197-
uint64 max_price = 8;
198-
bool fast_retrieval = 9;
199-
int64 deal_start_offset = 10;
196+
uint64 max_price = 7;
197+
bool fast_retrieval = 8;
198+
int64 deal_start_offset = 9;
200199
}
201200

202201
message Archive {

‎api/bucketsd/service.go‎

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -94,17 +94,18 @@ type ctxKey string
9494

9595
// Service is a gRPC service for buckets.
9696
type Service struct {
97-
Collections *mdb.Collections
98-
Buckets *tdb.Buckets
99-
GatewayURL string
100-
GatewayBucketsHost string
101-
IPFSClient iface.CoreAPI
102-
IPNSManager *ipns.Manager
103-
PowergateClient *pow.Client
104-
PowergateAdminToken string
105-
ArchiveTracker *archive.Tracker
106-
Semaphores *nutil.SemaphorePool
107-
MaxBucketSize int64
97+
Collections *mdb.Collections
98+
Buckets *tdb.Buckets
99+
GatewayURL string
100+
GatewayBucketsHost string
101+
IPFSClient iface.CoreAPI
102+
IPNSManager *ipns.Manager
103+
PowergateClient *pow.Client
104+
PowergateAdminToken string
105+
ArchiveTracker *archive.Tracker
106+
Semaphores *nutil.SemaphorePool
107+
MaxBucketSize int64
108+
MaxBucketArchiveRepFactor int
108109
}
109110

110111
var (
@@ -2256,7 +2257,12 @@ func (s *Service) SetDefaultArchiveConfig(
22562257
return nil, fmt.Errorf("getting bucket archive data: %s", err)
22572258
}
22582259

2259-
ba.DefaultArchiveConfig = fromPbArchiveConfig(req.ArchiveConfig)
2260+
c := fromPbArchiveConfig(req.ArchiveConfig)
2261+
if err := s.validateArchiveConfig(c); err != nil {
2262+
return nil, fmt.Errorf("validating archive config: %s", err)
2263+
}
2264+
2265+
ba.DefaultArchiveConfig = c
22602266
err = s.Collections.BucketArchives.Replace(ctx, ba)
22612267
if err != nil {
22622268
return nil, fmt.Errorf("saving default archive config: %s", err)
@@ -2349,14 +2355,14 @@ func (s *Service) Archive(ctx context.Context, req *pb.ArchiveRequest) (*pb.Arch
23492355
archiveConfig = &defaultDefaultArchiveConfig
23502356
}
23512357

2358+
if err := s.validateArchiveConfig(archiveConfig); err != nil {
2359+
return nil, fmt.Errorf("validating archive config: %s", err)
2360+
}
2361+
23522362
storageConfig := baseArchiveStorageConfig
23532363
storageConfig.Cold.Filecoin = toFilConfig(archiveConfig)
2354-
2355-
if storageConfig.Cold.Filecoin.Address == "" {
2356-
// We don't have an address to use, which is the case for a BucketArchive.DefaultArchiveConfig
2357-
// that is the default value, get the default address from the user
2358-
storageConfig.Cold.Filecoin.Address = defConfRes.DefaultStorageConfig.Cold.Filecoin.Address
2359-
}
2364+
// Get the address from the default storage config for this user.
2365+
storageConfig.Cold.Filecoin.Address = defConfRes.DefaultStorageConfig.Cold.Filecoin.Address
23602366

23612367
// Check that user wallet addr balance is > 0, if not, fail fast.
23622368
balRes, err := s.PowergateClient.Wallet.Balance(ctx, storageConfig.Cold.Filecoin.Address)
@@ -2604,7 +2610,6 @@ func toPbArchiveConfig(config *mdb.ArchiveConfig) *pb.ArchiveConfig {
26042610
Enabled: config.Renew.Enabled,
26052611
Threshold: int32(config.Renew.Threshold),
26062612
},
2607-
Addr: config.Addr,
26082613
MaxPrice: config.MaxPrice,
26092614
FastRetrieval: config.FastRetrieval,
26102615
DealStartOffset: config.DealStartOffset,
@@ -2622,7 +2627,6 @@ func fromPbArchiveConfig(pbConfig *pb.ArchiveConfig) *mdb.ArchiveConfig {
26222627
ExcludedMiners: pbConfig.ExcludedMiners,
26232628
TrustedMiners: pbConfig.TrustedMiners,
26242629
CountryCodes: pbConfig.CountryCodes,
2625-
Addr: pbConfig.Addr,
26262630
MaxPrice: pbConfig.MaxPrice,
26272631
FastRetrieval: pbConfig.FastRetrieval,
26282632
DealStartOffset: pbConfig.DealStartOffset,
@@ -2643,7 +2647,6 @@ func toFilConfig(config *mdb.ArchiveConfig) *userPb.FilConfig {
26432647
}
26442648
return &userPb.FilConfig{
26452649
ReplicationFactor: int64(config.RepFactor),
2646-
Address: config.Addr,
26472650
CountryCodes: config.CountryCodes,
26482651
DealMinDuration: config.DealMinDuration,
26492652
DealStartOffset: config.DealStartOffset,
@@ -2657,3 +2660,16 @@ func toFilConfig(config *mdb.ArchiveConfig) *userPb.FilConfig {
26572660
TrustedMiners: config.TrustedMiners,
26582661
}
26592662
}
2663+
2664+
func (s *Service) validateArchiveConfig(c *mdb.ArchiveConfig) error {
2665+
if c.RepFactor > s.MaxBucketArchiveRepFactor {
2666+
return fmt.Errorf("rep factor %d is greater than max allowed of %d", c.RepFactor, s.MaxBucketArchiveRepFactor)
2667+
}
2668+
if c.DealMinDuration < powUtil.MinDealDuration {
2669+
return fmt.Errorf("min deal duration %d is less than the allowed minimum of %d", c.DealMinDuration, powUtil.MinDealDuration)
2670+
}
2671+
if c.DealStartOffset <= 0 {
2672+
return fmt.Errorf("deal start offset of %d is less than required minimum of 1, and really should be higher than 1", c.DealStartOffset)
2673+
}
2674+
return nil
2675+
}

‎buckets/local/archive.go‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,41 +18,39 @@ type ArchiveConfig struct {
1818
// RepFactor (ignored in Filecoin mainnet) indicates the desired amount of active deals
1919
// with different miners to store the data. While making deals
2020
// the other attributes of FilConfig are considered for miner selection.
21-
RepFactor int
21+
RepFactor int `json:"repFactor"`
2222
// DealMinDuration indicates the duration to be used when making new deals.
23-
DealMinDuration int64
23+
DealMinDuration int64 `json:"dealMinDuration"`
2424
// ExcludedMiners (ignored in Filecoin mainnet) is a set of miner addresses won't be ever be selected
2525
// when making new deals, even if they comply to other filters.
26-
ExcludedMiners []string
26+
ExcludedMiners []string `json:"excludedMiners"`
2727
// TrustedMiners (ignored in Filecoin mainnet) is a set of miner addresses which will be forcibly used
2828
// when making new deals. An empty/nil list disables this feature.
29-
TrustedMiners []string
29+
TrustedMiners []string `json:"trustedMiners"`
3030
// CountryCodes (ignored in Filecoin mainnet) indicates that new deals should select miners on specific
3131
// countries.
32-
CountryCodes []string
32+
CountryCodes []string `json:"countryCodes"`
3333
// Renew indicates deal-renewal configuration.
34-
Renew ArchiveRenew
35-
// Addr is the wallet address used to store the data in filecoin
36-
Addr string
34+
Renew ArchiveRenew `json:"renew"`
3735
// MaxPrice is the maximum price that will be spent to store the data
38-
MaxPrice uint64
36+
MaxPrice uint64 `json:"maxPrice"`
3937
// FastRetrieval indicates that created deals should enable the
4038
// fast retrieval feature.
41-
FastRetrieval bool
39+
FastRetrieval bool `json:"fastRetrieval"`
4240
// DealStartOffset indicates how many epochs in the future impose a
4341
// deadline to new deals being active on-chain. This value might influence
4442
// if miners accept deals, since they should seal fast enough to satisfy
4543
// this constraint.
46-
DealStartOffset int64
44+
DealStartOffset int64 `json:"dealStartOffset"`
4745
}
4846

4947
// ArchiveRenew contains renew configuration for a ArchiveConfig.
5048
type ArchiveRenew struct {
5149
// Enabled indicates that deal-renewal is enabled for this Cid.
52-
Enabled bool
50+
Enabled bool `json:"enabled"`
5351
// Threshold indicates how many epochs before expiring should trigger
5452
// deal renewal. e.g: 100 epoch before expiring.
55-
Threshold int
53+
Threshold int `json:"threshold"`
5654
}
5755

5856
// DefaultArchiveConfig gets the default archive config for the specified Bucket.
@@ -81,7 +79,6 @@ func fromPbArchiveConfig(pbConfig *pb.ArchiveConfig) ArchiveConfig {
8179
ExcludedMiners: pbConfig.ExcludedMiners,
8280
TrustedMiners: pbConfig.TrustedMiners,
8381
CountryCodes: pbConfig.CountryCodes,
84-
Addr: pbConfig.Addr,
8582
MaxPrice: pbConfig.MaxPrice,
8683
FastRetrieval: pbConfig.FastRetrieval,
8784
DealStartOffset: pbConfig.DealStartOffset,
@@ -118,7 +115,6 @@ func toPbArchiveConfig(config ArchiveConfig) *pb.ArchiveConfig {
118115
Enabled: config.Renew.Enabled,
119116
Threshold: int32(config.Renew.Threshold),
120117
},
121-
Addr: config.Addr,
122118
MaxPrice: config.MaxPrice,
123119
FastRetrieval: config.FastRetrieval,
124120
DealStartOffset: config.DealStartOffset,

‎cmd/hubd/main.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ var (
9898
Key: "buckets.max_size",
9999
DefValue: int64(4 * gib),
100100
},
101+
"bucketsArchiveMaxRepFactor": {
102+
Key: "buckets.archive_max_rep_factor",
103+
DefValue: 4,
104+
},
101105

102106
// Threads
103107
"threadsMaxNumberPerOwner": {
@@ -253,6 +257,10 @@ func init() {
253257
"bucketsMaxSize",
254258
config.Flags["bucketsMaxSize"].DefValue.(int64),
255259
"Bucket max size in bytes")
260+
rootCmd.PersistentFlags().Int(
261+
"bucketsArchiveMaxRepFactor",
262+
config.Flags["bucketsArchiveMaxRepFactor"].DefValue.(int),
263+
"Bucket archive max replication factor")
256264

257265
// Threads
258266
rootCmd.PersistentFlags().Int(
@@ -378,6 +386,7 @@ var rootCmd = &cobra.Command{
378386

379387
// Buckets
380388
bucketsMaxSize := config.Viper.GetInt64("buckets.max_size")
389+
bucketsArchiveMaxRepFactor := config.Viper.GetInt("buckets.archive_max_rep_factor")
381390

382391
// Threads
383392
threadsMaxNumberPerOwner := config.Viper.GetInt("threads.max_number_per_owner")
@@ -434,7 +443,8 @@ var rootCmd = &cobra.Command{
434443
AddrBillingAPI: addrBillingApi,
435444
AddrPowergateAPI: addrPowergateApi,
436445
// Buckets
437-
MaxBucketSize: bucketsMaxSize,
446+
MaxBucketSize: bucketsMaxSize,
447+
BucketArchiveMaxRepFactor: bucketsArchiveMaxRepFactor,
438448
// Threads
439449
MaxNumberThreadsPerOwner: threadsMaxNumberPerOwner,
440450
// Powergate

‎core/core.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,8 @@ type Config struct {
167167
AddrPowergateAPI string
168168

169169
// Buckets
170-
MaxBucketSize int64
170+
MaxBucketSize int64
171+
BucketArchiveMaxRepFactor int
171172

172173
// Threads
173174
MaxNumberThreadsPerOwner int
@@ -383,17 +384,18 @@ func NewTextile(ctx context.Context, conf Config, opts ...Option) (*Textile, err
383384
}
384385
t.buckLocks = nutil.NewSemaphorePool(1)
385386
bs := &bucketsd.Service{
386-
Collections: t.collections,
387-
Buckets: t.bucks,
388-
GatewayURL: conf.AddrGatewayURL,
389-
GatewayBucketsHost: conf.DNSDomain,
390-
IPFSClient: ic,
391-
IPNSManager: t.ipnsm,
392-
PowergateClient: t.pc,
393-
PowergateAdminToken: conf.PowergateAdminToken,
394-
ArchiveTracker: t.archiveTracker,
395-
Semaphores: t.buckLocks,
396-
MaxBucketSize: conf.MaxBucketSize,
387+
Collections: t.collections,
388+
Buckets: t.bucks,
389+
GatewayURL: conf.AddrGatewayURL,
390+
GatewayBucketsHost: conf.DNSDomain,
391+
IPFSClient: ic,
392+
IPNSManager: t.ipnsm,
393+
PowergateClient: t.pc,
394+
PowergateAdminToken: conf.PowergateAdminToken,
395+
ArchiveTracker: t.archiveTracker,
396+
Semaphores: t.buckLocks,
397+
MaxBucketSize: conf.MaxBucketSize,
398+
MaxBucketArchiveRepFactor: conf.BucketArchiveMaxRepFactor,
397399
}
398400

399401
// Start serving

‎mongodb/bucketarchives.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,6 @@ type ArchiveConfig struct {
6666
CountryCodes []string `bson:"country_codes"`
6767
// Renew indicates deal-renewal configuration.
6868
Renew ArchiveRenew `bson:"renew"`
69-
// Addr is the wallet address used to store the data in filecoin
70-
Addr string `bson:"addr"`
7169
// MaxPrice is the maximum price that will be spent to store the data
7270
MaxPrice uint64 `bson:"max_price"`
7371
// FastRetrieval indicates that created deals should enable the

0 commit comments

Comments
 (0)