Skip to content

Commit 1c6a136

Browse files
gwillemgwillem
authored
Merge pull request #16 from sansecio/wdg/xxhash
Replace crc32 with xxhash
2 parents 5bf6450 + 7fbf53d commit 1c6a136

File tree

8 files changed

+101
-40
lines changed

8 files changed

+101
-40
lines changed

‎corediff.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,17 @@ import (
88
"log"
99
"os"
1010
"path/filepath"
11+
"runtime"
12+
13+
"github.com/gwillem/go-buildversion"
14+
"github.com/gwillem/go-selfupdate"
1115
)
1216

13-
var placeholder = struct{}{}
17+
var (
18+
selfUpdateURL = fmt.Sprintf("https://sansec.io/downloads/%s-%s/corediff", runtime.GOOS, runtime.GOARCH)
19+
placeholder = struct{}{}
20+
corediffVersion = buildversion.String()
21+
)
1422

1523
func loadDB(path string) hashDB {
1624
m := make(hashDB)
@@ -23,7 +31,7 @@ func loadDB(path string) hashDB {
2331
defer f.Close()
2432
reader := bufio.NewReader(f)
2533
for {
26-
var b uint32
34+
var b uint64
2735
err = binary.Read(reader, binary.LittleEndian, &b)
2836
if err == io.EOF {
2937
break
@@ -187,10 +195,14 @@ func addPath(root string, db hashDB, args *baseArgs) {
187195

188196
func main() {
189197

198+
if restarted, err := selfupdate.UpdateRestart(selfUpdateURL); restarted || err != nil {
199+
logVerbose("Restarted new version", restarted, "with error:", err)
200+
}
201+
190202
args := setup()
191203
db := loadDB(args.Database)
192204

193-
logInfo(boldwhite("Corediff loaded ", len(db), " precomputed hashes. (C) 2020-2023 [email protected]"))
205+
logInfo(boldwhite("Corediff ", corediffVersion, " loaded ", len(db), " precomputed hashes. (C) 2020-2023 [email protected]"))
194206
logInfo("Using database:", args.Database, "\n")
195207

196208
if args.Merge {

‎corediff_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/stretchr/testify/assert"
1111
)
1212

13-
func digest(b uint32) string {
13+
func digest(b uint64) string {
1414
return fmt.Sprintf("%x", b)
1515
}
1616

@@ -28,7 +28,7 @@ func Test_hash(t *testing.T) {
2828
args []byte
2929
want string
3030
}{
31-
{[]byte("banaan"), "14ac6691"},
31+
{[]byte("banaan"), "acfb1ff4438e39f3"},
3232
}
3333
for _, tt := range tests {
3434
t.Run(string(tt.args), func(t *testing.T) {
@@ -42,7 +42,7 @@ func Test_hash(t *testing.T) {
4242
func Test_vendor_bug(t *testing.T) {
4343
db := loadDB("fixture/sample.db")
4444
assert.Len(t, db, 238)
45-
wantHash := uint32(3333369281)
45+
wantHash := uint64(3900178074848893275)
4646
if _, ok := db[wantHash]; !ok {
4747
t.Error("hash not in db")
4848
}

‎fixture/sample.db‎

952 Bytes
Binary file not shown.

‎go.mod‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,26 @@
1-
module github.com/gwillem/magento-corediff
1+
module github.com/sansecio/corediff
22

33
go 1.20
44

55
require (
6-
github.com/cespare/xxhash/v2 v2.2.0
7-
github.com/fatih/color v1.12.0
6+
github.com/fatih/color v1.15.0
87
github.com/gobwas/glob v0.2.3
9-
github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d
8+
github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764
9+
github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292
10+
github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13
1011
github.com/jessevdk/go-flags v1.5.0
11-
github.com/stretchr/testify v1.6.1
12+
github.com/stretchr/testify v1.8.2
1213
)
1314

1415
require (
15-
github.com/adrg/xdg v0.2.3 // indirect
16-
github.com/davecgh/go-spew v1.1.0 // indirect
17-
github.com/mattn/go-colorable v0.1.8 // indirect
18-
github.com/mattn/go-isatty v0.0.12 // indirect
16+
github.com/adrg/xdg v0.4.0 // indirect
17+
github.com/cespare/xxhash/v2 v2.2.0 // indirect
18+
github.com/davecgh/go-spew v1.1.1 // indirect
19+
github.com/mattn/go-colorable v0.1.13 // indirect
20+
github.com/mattn/go-isatty v0.0.18 // indirect
1921
github.com/pmezard/go-difflib v1.0.0 // indirect
20-
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect
21-
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
22+
golang.org/x/sys v0.6.0 // indirect
23+
gopkg.in/yaml.v3 v3.0.1 // indirect
2224
)
25+
26+
//replace github.com/gwillem/go-buildversion => ../go-buildversion

‎go.sum‎

Lines changed: 33 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,47 @@
1-
github.com/adrg/xdg v0.2.3 h1:GxXngdYxNDkoUvZXjNJGwqZxWXi43MKbOOlA/00qZi4=
2-
github.com/adrg/xdg v0.2.3/go.mod h1:7I2hH/IT30IsupOpKZ5ue7/qNi3CoKzD6tL3HwpaRMQ=
1+
github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls=
2+
github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E=
33
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
44
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
5-
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
65
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7-
github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
8-
github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
6+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
7+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
8+
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
9+
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
910
github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
1011
github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
11-
github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d h1:SUYp/bOtCvpddj20LJRLoU/YFnoQhXLGG3S+Ro8uWP0=
12-
github.com/gwillem/urlfilecache v0.0.0-20201121123616-8e4f7e58333d/go.mod h1:wcspENTUwvMi317k0q22DWfx6hIXAlEjz5LO6pjJWQ0=
12+
github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5 h1:tMpRA020WZ+HsqrRFiflnC0Pj9gseMrG8jWimXw9WWQ=
13+
github.com/gwillem/go-buildversion v0.0.0-20230402113215-08d191152cc5/go.mod h1:mT3drjAmZ/WgzCwPqf270xC9absLnjSMbDSPXPRiGk8=
14+
github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764 h1:GL/gS9rv5F1oBgQwbT+pl0HefNbDZLjbcWmDNT/pUOU=
15+
github.com/gwillem/go-buildversion v0.0.0-20230402114832-b1addefa8764/go.mod h1:yf2A6rXn1ptErQTutA0vpEDZJG6E0b/aUnMnXelY5tU=
16+
github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292 h1:AoF/jIYBBxpLJkyf204k6ArAFDFVKitkcOGnyYuWHco=
17+
github.com/gwillem/go-selfupdate v0.0.0-20230402110315-648e078de292/go.mod h1:Zv9vnDD/HDN/WCidxG2QdCEDtYIaIeUTX9yHspWZfEM=
18+
github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13 h1:Kuu6BZRS1uaSatweU/S5pWKkwt+QFCMb2QjJrWcvTWA=
19+
github.com/gwillem/urlfilecache v0.0.0-20230402105623-8ef3b7b67c13/go.mod h1:vEuNDp+GXXq7N5Anwiu1f4VQ08IGuSTGs/PaFCEno/U=
1320
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
1421
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
15-
github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
16-
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
17-
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
18-
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
22+
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
23+
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
24+
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
25+
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
26+
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
1927
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
2028
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
2129
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
22-
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
23-
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
24-
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
25-
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
26-
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc=
30+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
31+
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
32+
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
33+
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
34+
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
35+
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
36+
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
2737
golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
38+
golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
39+
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
40+
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
41+
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
42+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
2843
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
2944
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
30-
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
3145
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
46+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
47+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎helpers.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@ package main
33
import (
44
"bytes"
55
"fmt"
6-
"hash/crc32"
76
"os"
87
"path/filepath"
98
"regexp"
109
"strings"
1110

11+
"github.com/cespare/xxhash/v2"
1212
"github.com/gobwas/glob"
1313
)
1414

@@ -78,11 +78,11 @@ func check(err error) {
7878
}
7979
}
8080

81-
func hash(b []byte) uint32 {
82-
return crc32.ChecksumIEEE(b)
81+
func hash(b []byte) uint64 {
82+
return xxhash.Sum64(b)
8383
}
8484

85-
func pathHash(p string) uint32 {
85+
func pathHash(p string) uint64 {
8686
return hash([]byte("path:" + p))
8787
}
8888

‎needle_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"testing"
6+
)
7+
8+
func Test_Needle(t *testing.T) {
9+
10+
needles := []string{
11+
"path:app/code/Magedelight/GeoIp/Controller/Adminhtml/Currencymapping/Delete.php",
12+
}
13+
dbpath := "m2.db"
14+
15+
db := loadDB(dbpath)
16+
fmt.Println("Loaded entries:", len(db))
17+
18+
for k, _ := range db {
19+
fmt.Println("first entry", k)
20+
break
21+
}
22+
23+
for _, needle := range needles {
24+
checksum := hash([]byte(needle))
25+
hash := fmt.Sprintf("%x", checksum)
26+
_, ok := db[checksum]
27+
fmt.Println(ok, hash)
28+
}
29+
}

‎setup.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
)
1212

1313
type (
14-
hashDB map[uint32]struct{}
14+
hashDB map[uint64]struct{}
1515

1616
walkStats struct {
1717
totalFiles int
@@ -121,7 +121,7 @@ func setup() *baseArgs {
121121
fmt.Println("Can't merge without given --database file")
122122
os.Exit(1)
123123
}
124-
fmt.Println("Downloading default hash database from", hashDBURL)
124+
// fmt.Println("Using default hash database from", hashDBURL)
125125
args.Database = urlfilecache.ToPath(hashDBURL)
126126
}
127127

0 commit comments

Comments
 (0)