Skip to content

Commit 8852bd3

Browse files
authored
[fix] use less memory / speed-up parsing (#5)
* [fix] use less memory / speed-up parsing * [clean] * [parser] lazy approach * [review] * [fix] * [review] * [review] * [review] * [review] * [review]
1 parent 42648ea commit 8852bd3

File tree

15 files changed

+1307
-161
lines changed

15 files changed

+1307
-161
lines changed

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
.idea/
2-
vendor/
2+
vendor/
3+
*.pprof
4+
coverage.*

README.md

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ Library provides convenient way to check if:
1414

1515
go get -v github.com/travelaudience/go-iabtcf
1616

17-
### Example
17+
### Example - Normal Parsing
1818

1919
package main
2020

@@ -34,6 +34,27 @@ Library provides convenient way to check if:
3434
sf := s.EverySpecialFeatureAllowed([]int{1})
3535
va := s.VendorAllowed(1)
3636
}
37+
38+
### Example - Lazy Parsing
39+
40+
package main
41+
42+
import (
43+
"fmt"
44+
45+
"github.com/travelaudience/go-iabtcf"
46+
)
47+
48+
func main() {
49+
var s, err = iabtcf.LazyParseCoreString("COwIsAvOwIsAvBIAAAENAPCMAP_AAP_AAAAAFoQBQABAAGAAQAAwACQAAAAA.IFoEUQQgAIQwgIwQABAEAAAAOIAACAIAAAAQAIAgEAACEAAAAAgAQBAAAAAAAGBAAgAAAAAAAFAAECAAAgAAQARAEQAAAAAJAAIAAgAAAYQEAAAQmAgBC3ZAYzUw")
50+
if err != nil {
51+
panic(err)
52+
}
53+
54+
pa := s.EveryPurposeAllowed([]int{1})
55+
sf := s.EverySpecialFeatureAllowed([]int{1})
56+
va := s.VendorAllowed(1)
57+
}
3758

3859
## Contributing
3960

bits.go

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
package iabtcf
2+
3+
import (
4+
"fmt"
5+
"time"
6+
)
7+
8+
// //////////////////////////////////////////////////
9+
// bits
10+
11+
// Bits represents a bitset with some helpers to read int, bool, string and time fields
12+
//
13+
// Bits are stored in a byte slice
14+
// First byte will store the first 8 bits, second byte the next 8 bits, and so on
15+
//
16+
// note: the last byte may contain less than 8 bits. Those bits are left aligned.
17+
type Bits []byte
18+
19+
// HasBit checks if the bit number is set
20+
//
21+
// note: number is not the index and it starts at 1.
22+
func (b Bits) HasBit(number int) bool {
23+
return b.ReadBoolField(number - 1)
24+
}
25+
26+
// Length returns the number of bits in the bitset
27+
func (b Bits) Length() int {
28+
return len(b) * nbBitInByte
29+
}
30+
31+
const (
32+
nbBitInByte = 8
33+
lastBitIndex = nbBitInByte - 1
34+
)
35+
36+
var (
37+
bitMasks = [nbBitInByte]byte{
38+
1 << 7,
39+
1 << 6,
40+
1 << 5,
41+
1 << 4,
42+
1 << 3,
43+
1 << 2,
44+
1 << 1,
45+
1,
46+
}
47+
)
48+
49+
// ReadInt64Field reads an int64 field of nbBits bits starting at offset
50+
//
51+
// note: if offset is negative, the result will be zero
52+
// note: if offset + nbBits is out of bound, the result will be the same if we were adding trailing zeros
53+
// example: 00101 > read with offset 2 and nbBits 5 > equivalent to reading 10100 = 20
54+
func (b Bits) ReadInt64Field(offset, nbBits int) int64 {
55+
if offset < 0 {
56+
return 0
57+
}
58+
var result int64
59+
byteIndex := offset / nbBitInByte
60+
if byteIndex >= len(b) {
61+
return result
62+
}
63+
bitIndex := offset % nbBitInByte
64+
for i := 0; i < nbBits; i++ {
65+
mask := bitMasks[bitIndex]
66+
if b[byteIndex]&mask == mask {
67+
result |= 1 << (nbBits - 1 - i)
68+
}
69+
if bitIndex == lastBitIndex {
70+
byteIndex++
71+
if byteIndex >= len(b) {
72+
return result
73+
}
74+
bitIndex = 0
75+
} else {
76+
bitIndex++
77+
}
78+
}
79+
return result
80+
}
81+
82+
// ReadIntField reads an int field of nbBits bits starting at offset
83+
func (b *Bits) ReadIntField(offset, nbBits int) int {
84+
return int(b.ReadInt64Field(offset, nbBits))
85+
}
86+
87+
const (
88+
timeNbBits = 36
89+
)
90+
91+
// ReadTimeField reads a time field of 36 bits starting at offset
92+
func (b *Bits) ReadTimeField(offset int) time.Time {
93+
ds := b.ReadInt64Field(offset, timeNbBits)
94+
return time.Unix(ds/dsPerSec, (ds%dsPerSec)*nsPerDs).UTC()
95+
}
96+
97+
const (
98+
characterNbBits = 6
99+
)
100+
101+
// ReadStringField reads a string field of nbBits bits starting at offset
102+
//
103+
// note: each character is represented by 6 bits, so the number of bits must be a multiple of 6
104+
// note: the characters are represented by the uppercase alphabet starting from 'A'
105+
func (b *Bits) ReadStringField(offset, nbBits int) string {
106+
length := nbBits / characterNbBits
107+
var buf = make([]byte, 0, length)
108+
nextOffset := offset
109+
for i := 0; i < length; i++ {
110+
value := b.ReadInt64Field(nextOffset, characterNbBits)
111+
buf = append(buf, byte(value)+'A')
112+
nextOffset += characterNbBits
113+
}
114+
return string(buf)
115+
}
116+
117+
const (
118+
boolNbBits = 1
119+
)
120+
121+
// ReadBoolField reads a bool field of 1 bit starting at offset
122+
func (b *Bits) ReadBoolField(offset int) bool {
123+
return b.ReadInt64Field(offset, boolNbBits) == 1
124+
}
125+
126+
// ToBitString returns the bitset as a string of bits ( human readable 0s and 1s )
127+
func (bits Bits) ToBitString() string {
128+
if bits == nil {
129+
return ""
130+
}
131+
132+
result := ""
133+
134+
for i, b := range bits {
135+
if i != 0 {
136+
result += " "
137+
}
138+
result += fmt.Sprintf("%08b", b)
139+
}
140+
141+
return result
142+
}
143+
144+
// //////////////////////////////////////////////////
145+
// bit string helper
146+
147+
// BitStringToBits converts a bit string to a Bits struct
148+
func BitStringToBits(value string) Bits {
149+
return Bits(BitStringToBytes(value))
150+
}
151+
152+
// BitStringToBytes converts a bit string to a byte slice
153+
func BitStringToBytes(value string) []byte {
154+
bytes := make([]byte, 0, len(value)/nbBitInByte)
155+
156+
position := lastBitIndex
157+
var lastByte byte
158+
for i := 0; i < len(value); i++ {
159+
if value[i] == ' ' {
160+
continue
161+
}
162+
if value[i] == '1' {
163+
lastByte |= 1 << position
164+
}
165+
if position == 0 {
166+
position = lastBitIndex
167+
bytes = append(bytes, lastByte)
168+
lastByte = 0
169+
} else {
170+
position--
171+
}
172+
}
173+
if position != nbBitInByte-1 {
174+
bytes = append(bytes, lastByte)
175+
}
176+
return bytes
177+
}

bits_test.go

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package iabtcf
2+
3+
import (
4+
"encoding/base64"
5+
"fmt"
6+
"strings"
7+
"testing"
8+
9+
"github.com/stretchr/testify/require"
10+
)
11+
12+
func TestBits(t *testing.T) {
13+
14+
wantHasBit := func(number int, expected []int) bool {
15+
for _, e := range expected {
16+
if number == e {
17+
return true
18+
}
19+
}
20+
return false
21+
}
22+
23+
type TestCase struct {
24+
Base64 string
25+
WantBitString string
26+
WantHasBit []int
27+
}
28+
29+
values := map[string]*TestCase{
30+
"101": {
31+
Base64: "oA",
32+
WantBitString: "10100000",
33+
WantHasBit: []int{1, 3},
34+
},
35+
"00000001": {
36+
Base64: "AQ",
37+
WantBitString: "00000001",
38+
WantHasBit: []int{8},
39+
},
40+
"00000101": {
41+
Base64: "BQ",
42+
WantBitString: "00000101",
43+
WantHasBit: []int{6, 8},
44+
},
45+
"10000101": {
46+
Base64: "hQ",
47+
WantBitString: "10000101",
48+
WantHasBit: []int{1, 6, 8},
49+
},
50+
"00000001 00000101": {
51+
Base64: "AQU",
52+
WantBitString: "00000001 00000101",
53+
WantHasBit: []int{8, 14, 16},
54+
},
55+
"00000001 101": {
56+
Base64: "AaA",
57+
WantBitString: "00000001 10100000",
58+
WantHasBit: []int{8, 9, 11},
59+
},
60+
"00000001 00000000": {
61+
Base64: "AQA",
62+
WantBitString: "00000001 00000000",
63+
WantHasBit: []int{8},
64+
},
65+
"00000001 00000000 1": {
66+
Base64: "AQCA",
67+
WantBitString: "00000001 00000000 10000000",
68+
WantHasBit: []int{8, 17},
69+
},
70+
"00000001 0000001": {
71+
Base64: "AQI",
72+
WantBitString: "00000001 00000010",
73+
WantHasBit: []int{8, 15},
74+
},
75+
}
76+
77+
for bitString, tc := range values {
78+
t.Run(bitString, func(t *testing.T) {
79+
t.Helper()
80+
fmt.Printf("\n[test] ---------- %s ---------- \n", bitString)
81+
var wantBytes, err = base64.RawURLEncoding.DecodeString(tc.Base64)
82+
fmt.Printf("[test] base64: %s >>> bytes: %v \n", tc.Base64, wantBytes)
83+
require.NoError(t, err, "unexpected base64 error")
84+
85+
gotBits := BitStringToBits(bitString)
86+
fmt.Printf("[test] bits: %s >>> bytes: %v \n", bitString, gotBits)
87+
require.Equal(t, wantBytes, []byte(gotBits))
88+
89+
fmt.Printf("[test] Bits: %v \n", gotBits)
90+
91+
fmt.Printf("[test] bytes: %v >>> bits: %s \n", gotBits, gotBits.ToBitString())
92+
require.Equal(t, tc.WantBitString, gotBits.ToBitString())
93+
94+
fmt.Printf("[test] WantHasBit: %v \n", tc.WantHasBit)
95+
length := len(strings.ReplaceAll(bitString, " ", ""))
96+
for number := 1; number <= length; number++ {
97+
gotHasBit := gotBits.HasBit(number)
98+
wantHasBit := wantHasBit(number, tc.WantHasBit)
99+
require.Equal(t, wantHasBit, gotHasBit)
100+
}
101+
})
102+
}
103+
}

0 commit comments

Comments
 (0)