Skip to content

Commit a1fd85c

Browse files
committed
add CBT bitmap implementation
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
1 parent b182707 commit a1fd85c

10 files changed

Lines changed: 1326 additions & 24 deletions

File tree

go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ require (
99
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
1010
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
1111
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
12+
github.com/RoaringBitmap/roaring v1.9.4
1213
github.com/aws/aws-sdk-go-v2 v1.24.1
1314
github.com/aws/aws-sdk-go-v2/config v1.26.3
1415
github.com/aws/aws-sdk-go-v2/credentials v1.16.14
@@ -91,6 +92,7 @@ require (
9192
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
9293
github.com/aws/smithy-go v1.19.0 // indirect
9394
github.com/beorn7/perks v1.0.1 // indirect
95+
github.com/bits-and-blooms/bitset v1.12.0 // indirect
9496
github.com/blang/semver/v4 v4.0.0 // indirect
9597
github.com/cespare/xxhash/v2 v2.3.0 // indirect
9698
github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
@@ -148,6 +150,7 @@ require (
148150
github.com/moby/term v0.5.0 // indirect
149151
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
150152
github.com/modern-go/reflect2 v1.0.2 // indirect
153+
github.com/mschoch/smat v0.2.0 // indirect
151154
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
152155
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
153156
github.com/mxk/go-vss v1.2.0 // indirect

go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
113113
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
114114
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
115115
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
116+
github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
117+
github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
116118
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
117119
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
118120
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -168,6 +170,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
168170
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
169171
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
170172
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
173+
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
174+
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
171175
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
172176
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
173177
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -562,6 +566,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
562566
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
563567
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
564568
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
569+
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
570+
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
565571
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
566572
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
567573
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=

pkg/cbtservice/mocks/Service.go

Lines changed: 171 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/cbtservice/service.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ import "context"
2020

2121
// Range defines the range of a change
2222
type Range struct {
23-
Offset int64
24-
Length int64
23+
Offset uint64
24+
Length uint64
2525
}
2626

2727
// SourceInfo is the information provided to the uploader, the uploader calls CBT service with this information

pkg/uploader/cbt/bitmap.go

Lines changed: 107 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,29 @@ limitations under the License.
1616

1717
package cbt
1818

19-
import "github.com/vmware-tanzu/velero/pkg/cbtservice"
19+
import (
20+
"math/bits"
21+
22+
"github.com/RoaringBitmap/roaring"
23+
)
2024

2125
// Bitmap defines the methods to store and iterate the CBT bitmap
2226
type Bitmap interface {
2327
// Set sets bits within the provided range
24-
Set(cbtservice.Range)
28+
Set(uint64, uint64)
2529

2630
// SetFull sets all bits to the bitmap
2731
SetFull()
2832

2933
// Snapshot returns snapshot of the bitmap
30-
SourceID() string
34+
Snapshot() string
3135

3236
// ChangeID returns the changeID of the bitmap
3337
ChangeID() string
3438

39+
// VolumeID return ID of the volume from which the snapshot is taken
40+
VolumeID() string
41+
3542
// Iterator returns the iterator for the CBT Bitmap
3643
Iterator() Iterator
3744
}
@@ -44,12 +51,107 @@ type Iterator interface {
4451
// Snapshot returns snapshot of the bitmap
4552
Snapshot() string
4653

54+
// VolumeID return ID of the volume from which the snapshot is taken
55+
VolumeID() string
56+
4757
// BlockSize returns the granularity of the bitmap
48-
BlockSize() int
58+
BlockSize() uint
4959

5060
// Count returns the toal number of count in the bitmap
5161
Count() uint64
5262

5363
// Next returns the offset of the next set block and whether it comes to the end of the iteration
54-
Next() (int64, bool)
64+
Next() (uint64, bool)
65+
}
66+
67+
const (
68+
InvalidOffset64 = ^uint64(0)
69+
)
70+
71+
type bitmapImpl struct {
72+
bitmap *roaring.Bitmap
73+
blockSize uint
74+
blockSizeLog int
75+
length uint64
76+
snapshot string
77+
changeID string
78+
volumeID string
79+
}
80+
81+
type bitmapIterator struct {
82+
bitmapImpl
83+
iterator roaring.IntPeekable
84+
}
85+
86+
func NewBitmap(blockSize uint, length uint64, snapshot string, changeID string, volumeID string) Bitmap {
87+
return &bitmapImpl{
88+
bitmap: roaring.New(),
89+
blockSize: blockSize,
90+
blockSizeLog: bits.Len(blockSize) - 1,
91+
length: length,
92+
snapshot: snapshot,
93+
changeID: changeID,
94+
volumeID: volumeID,
95+
}
96+
}
97+
98+
func (c *bitmapImpl) Set(offset, length uint64) {
99+
if offset >= c.length {
100+
return
101+
}
102+
103+
if offset+length > c.length {
104+
length = c.length - offset
105+
}
106+
107+
start := offset >> c.blockSizeLog
108+
end := uint64((offset + length + uint64(c.blockSize) - 1) >> c.blockSizeLog)
109+
110+
c.bitmap.AddRange(start, end)
111+
}
112+
113+
func (c *bitmapImpl) SetFull() {
114+
start := uint64(0)
115+
end := uint64((c.length + uint64(c.blockSize) - 1) >> c.blockSizeLog)
116+
117+
c.bitmap.AddRange(start, end)
118+
}
119+
120+
func (c *bitmapImpl) Snapshot() string {
121+
return c.snapshot
122+
}
123+
124+
func (c *bitmapImpl) ChangeID() string {
125+
return c.changeID
126+
}
127+
128+
func (c *bitmapImpl) VolumeID() string {
129+
return c.volumeID
130+
}
131+
132+
func (c *bitmapImpl) Iterator() Iterator {
133+
if c.bitmap == nil {
134+
return nil
135+
}
136+
137+
return &bitmapIterator{
138+
bitmapImpl: *c,
139+
iterator: c.bitmap.Iterator(),
140+
}
141+
}
142+
143+
func (c *bitmapIterator) Next() (uint64, bool) {
144+
if !c.iterator.HasNext() {
145+
return InvalidOffset64, false
146+
}
147+
148+
return uint64(c.iterator.Next()) << c.blockSizeLog, true
149+
}
150+
151+
func (c *bitmapIterator) Count() uint64 {
152+
return c.bitmap.GetCardinality()
153+
}
154+
155+
func (c *bitmapIterator) BlockSize() uint {
156+
return c.blockSize
55157
}

0 commit comments

Comments
 (0)