Skip to content

Commit 47e6c33

Browse files
gaius-qigaius-qi
authored
feat: bump client to v0.2.26 and add CRC32FromStrings func (#4024)
Signed-off-by: Gaius <gaius.qi@gmail.com>
1 parent 1edd631 commit 47e6c33

File tree

7 files changed

+73
-18
lines changed

7 files changed

+73
-18
lines changed

‎.github/workflows/compatibility-e2e-v2.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,12 @@ jobs:
4141
skip: "Rate Limit | preheat files in cache"
4242
- module: client
4343
image: client
44-
image-tag: v0.2.18
44+
image-tag: v0.2.26
4545
chart-name: client
4646
skip: "Rate Limit"
4747
- module: seed-client
4848
image: client
49-
image-tag: v0.2.18
49+
image-tag: v0.2.26
5050
chart-name: seed-client
5151
skip: "Rate Limit"
5252

‎client-rs‎

Submodule client-rs updated 32 files

‎pkg/digest/digest.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"hash/crc32"
3030
"io"
3131
"os"
32+
"strconv"
3233
"strings"
3334

3435
"github.com/zeebo/blake3"
@@ -196,3 +197,19 @@ func SHA256FromBytes(bytes []byte) string {
196197
h.Write(bytes)
197198
return hex.EncodeToString(h.Sum(nil))
198199
}
200+
201+
// CRC32FromStrings computes the CRC32 checksum with multiple strings.
202+
func CRC32FromStrings(data ...string) string {
203+
if len(data) == 0 {
204+
return ""
205+
}
206+
207+
h := crc32.NewIEEE()
208+
for _, s := range data {
209+
if _, err := h.Write([]byte(s)); err != nil {
210+
return ""
211+
}
212+
}
213+
214+
return strconv.FormatUint(uint64(h.Sum32()), 10)
215+
}

‎pkg/digest/digest_test.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,7 @@ func TestDigest_SHA256FromStrings(t *testing.T) {
175175
func TestDigest_SHA256FromBytes(t *testing.T) {
176176
assert.Equal(t, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", SHA256FromBytes([]byte("hello")))
177177
}
178+
179+
func TestDigest_CRC32FromStrings(t *testing.T) {
180+
assert.Equal(t, "907060870", CRC32FromStrings("hello"))
181+
}

‎pkg/idgen/task_id.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,8 @@ func TaskIDV2ByURLBased(url string, pieceLength *uint64, tag, application string
110110
func TaskIDV2ByContent(content string) string {
111111
return pkgdigest.SHA256FromStrings(content)
112112
}
113+
114+
// PersistentCacheTaskIDByContent generates persistent cache task id by content.
115+
func PersistentCacheTaskIDByContent(content string) string {
116+
return pkgdigest.CRC32FromStrings(content)
117+
}

‎pkg/idgen/task_id_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,3 +207,26 @@ func TestTaskIDV2ByContent(t *testing.T) {
207207
})
208208
}
209209
}
210+
211+
func TestPersistentCacheTaskIDbyContent(t *testing.T) {
212+
tests := []struct {
213+
name string
214+
content string
215+
expect func(t *testing.T, d any)
216+
}{
217+
{
218+
name: "generate persistentCacheTaskID",
219+
content: "This is a test file",
220+
expect: func(t *testing.T, d any) {
221+
assert := assert.New(t)
222+
assert.Equal(d, "107352521")
223+
},
224+
},
225+
}
226+
227+
for _, tc := range tests {
228+
t.Run(tc.name, func(t *testing.T) {
229+
tc.expect(t, PersistentCacheTaskIDByContent(tc.content))
230+
})
231+
}
232+
}

‎test/e2e/v2/dfcache_test.go‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
. "github.com/onsi/ginkgo/v2" //nolint
2323
. "github.com/onsi/gomega" //nolint
2424

25+
"d7y.io/dragonfly/v2/pkg/idgen"
2526
"d7y.io/dragonfly/v2/test/e2e/v2/util"
2627
)
2728

@@ -60,19 +61,20 @@ var _ = Describe("Import and Export Using Dfcache", func() {
6061
Expect(err).NotTo(HaveOccurred())
6162
Expect(testFile.GetSha256()).To(Equal(sha256sum))
6263

63-
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
64+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --content-for-calculating-task-id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
6465
fmt.Println(string(importOut), err)
6566
Expect(err).NotTo(HaveOccurred())
6667

6768
seedClientPod, err := util.SeedClientExec(0)
6869
fmt.Println(err)
6970
Expect(err).NotTo(HaveOccurred())
7071

71-
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
72+
taskID := idgen.PersistentCacheTaskIDByContent(testFile.GetSha256())
73+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", taskID, testFile.GetOutputPath())).CombinedOutput()
7274
fmt.Println(string(exportOut), err)
7375
Expect(err).NotTo(HaveOccurred())
7476

75-
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
77+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, taskID)
7678
Expect(err).NotTo(HaveOccurred())
7779
Expect(testFile.GetSha256()).To(Equal(sha256sum))
7880

@@ -116,19 +118,20 @@ var _ = Describe("Import and Export Using Dfcache", func() {
116118
Expect(err).NotTo(HaveOccurred())
117119
Expect(testFile.GetSha256()).To(Equal(sha256sum))
118120

119-
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
121+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --content-for-calculating-task-id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
120122
fmt.Println(string(importOut), err)
121123
Expect(err).NotTo(HaveOccurred())
122124

123125
seedClientPod, err := util.SeedClientExec(0)
124126
fmt.Println(err)
125127
Expect(err).NotTo(HaveOccurred())
126128

127-
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
129+
taskID := idgen.PersistentCacheTaskIDByContent(testFile.GetSha256())
130+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", taskID, testFile.GetOutputPath())).CombinedOutput()
128131
fmt.Println(string(exportOut), err)
129132
Expect(err).NotTo(HaveOccurred())
130133

131-
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
134+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, taskID)
132135
Expect(err).NotTo(HaveOccurred())
133136
Expect(testFile.GetSha256()).To(Equal(sha256sum))
134137

@@ -172,19 +175,20 @@ var _ = Describe("Import and Export Using Dfcache", func() {
172175
Expect(err).NotTo(HaveOccurred())
173176
Expect(testFile.GetSha256()).To(Equal(sha256sum))
174177

175-
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
178+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --content-for-calculating-task-id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
176179
fmt.Println(string(importOut), err)
177180
Expect(err).NotTo(HaveOccurred())
178181

179182
seedClientPod, err := util.SeedClientExec(0)
180183
fmt.Println(err)
181184
Expect(err).NotTo(HaveOccurred())
182185

183-
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
186+
taskID := idgen.PersistentCacheTaskIDByContent(testFile.GetSha256())
187+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", taskID, testFile.GetOutputPath())).CombinedOutput()
184188
fmt.Println(string(exportOut), err)
185189
Expect(err).NotTo(HaveOccurred())
186190

187-
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
191+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, taskID)
188192
Expect(err).NotTo(HaveOccurred())
189193
Expect(testFile.GetSha256()).To(Equal(sha256sum))
190194

@@ -228,19 +232,20 @@ var _ = Describe("Import and Export Using Dfcache", func() {
228232
Expect(err).NotTo(HaveOccurred())
229233
Expect(testFile.GetSha256()).To(Equal(sha256sum))
230234

231-
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
235+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --content-for-calculating-task-id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
232236
fmt.Println(string(importOut), err)
233237
Expect(err).NotTo(HaveOccurred())
234238

235239
seedClientPod, err := util.SeedClientExec(0)
236240
fmt.Println(err)
237241
Expect(err).NotTo(HaveOccurred())
238242

239-
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
243+
taskID := idgen.PersistentCacheTaskIDByContent(testFile.GetSha256())
244+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", taskID, testFile.GetOutputPath())).CombinedOutput()
240245
fmt.Println(string(exportOut), err)
241246
Expect(err).NotTo(HaveOccurred())
242247

243-
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
248+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, taskID)
244249
Expect(err).NotTo(HaveOccurred())
245250
Expect(testFile.GetSha256()).To(Equal(sha256sum))
246251

@@ -284,19 +289,20 @@ var _ = Describe("Import and Export Using Dfcache", func() {
284289
Expect(err).NotTo(HaveOccurred())
285290
Expect(testFile.GetSha256()).To(Equal(sha256sum))
286291

287-
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
292+
importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --persistent-replica-count 1 --content-for-calculating-task-id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput()
288293
fmt.Println(string(importOut), err)
289294
Expect(err).NotTo(HaveOccurred())
290295

291296
seedClientPod, err := util.SeedClientExec(0)
292297
fmt.Println(err)
293298
Expect(err).NotTo(HaveOccurred())
294299

295-
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput()
300+
taskID := idgen.PersistentCacheTaskIDByContent(testFile.GetSha256())
301+
exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", taskID, testFile.GetOutputPath())).CombinedOutput()
296302
fmt.Println(string(exportOut), err)
297303
Expect(err).NotTo(HaveOccurred())
298304

299-
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256())
305+
sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, taskID)
300306
Expect(err).NotTo(HaveOccurred())
301307
Expect(testFile.GetSha256()).To(Equal(sha256sum))
302308

0 commit comments

Comments
 (0)