Skip to content

Commit 594601e

Browse files
pboothepboothe
authored
Move rawfile to contentprovider. (#31)
* Move rawfile to content.Provider. * Create tarreader for the stuff that just reads tar files.
1 parent 89d847f commit 594601e

File tree

13 files changed

+86
-652
lines changed

13 files changed

+86
-652
lines changed

‎asnannotator/asn.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ import (
66
"net"
77
"sync"
88

9+
"github.com/m-lab/go/content"
910
"github.com/m-lab/go/rtx"
1011

1112
"github.com/m-lab/tcp-info/inetdiag"
1213
"github.com/m-lab/uuid-annotator/annotator"
1314
"github.com/m-lab/uuid-annotator/ipinfo"
14-
"github.com/m-lab/uuid-annotator/rawfile"
1515
"github.com/m-lab/uuid-annotator/routeview"
16+
"github.com/m-lab/uuid-annotator/tarreader"
1617
)
1718

1819
// ASNAnnotator is just a regular annotator with a Reload method and an AnnotateIP method.
@@ -26,17 +27,17 @@ type ASNAnnotator interface {
2627
type asnAnnotator struct {
2728
m sync.RWMutex
2829
localIPs []net.IP
29-
as4 rawfile.Provider
30-
as6 rawfile.Provider
31-
asnamedata rawfile.Provider
30+
as4 content.Provider
31+
as6 content.Provider
32+
asnamedata content.Provider
3233
asn4 routeview.Index
3334
asn6 routeview.Index
3435
asnames ipinfo.ASNames
3536
}
3637

3738
// New makes a new Annotator that uses IP addresses to lookup ASN metadata for
3839
// that IP based on the current copy of RouteViews data stored in the given providers.
39-
func New(ctx context.Context, as4 rawfile.Provider, as6 rawfile.Provider, asnamedata rawfile.Provider, localIPs []net.IP) ASNAnnotator {
40+
func New(ctx context.Context, as4 content.Provider, as6 content.Provider, asnamedata content.Provider, localIPs []net.IP) ASNAnnotator {
4041
a := &asnAnnotator{
4142
as4: as4,
4243
as6: as6,
@@ -139,9 +140,9 @@ func (a *asnAnnotator) Reload(ctx context.Context) {
139140
a.asnames = newnames
140141
}
141142

142-
func load(ctx context.Context, src rawfile.Provider, oldvalue routeview.Index) (routeview.Index, error) {
143+
func load(ctx context.Context, src content.Provider, oldvalue routeview.Index) (routeview.Index, error) {
143144
gz, err := src.Get(ctx)
144-
if err == rawfile.ErrNoChange {
145+
if err == content.ErrNoChange {
145146
return oldvalue, nil
146147
}
147148
if err != nil {
@@ -151,16 +152,16 @@ func load(ctx context.Context, src rawfile.Provider, oldvalue routeview.Index) (
151152
}
152153

153154
func loadGZ(gz []byte) (routeview.Index, error) {
154-
data, err := rawfile.FromGZ(gz)
155+
data, err := tarreader.FromGZ(gz)
155156
if err != nil {
156157
return nil, err
157158
}
158159
return routeview.ParseRouteView(data), nil
159160
}
160161

161-
func loadNames(ctx context.Context, src rawfile.Provider, oldvalue ipinfo.ASNames) (ipinfo.ASNames, error) {
162+
func loadNames(ctx context.Context, src content.Provider, oldvalue ipinfo.ASNames) (ipinfo.ASNames, error) {
162163
data, err := src.Get(ctx)
163-
if err == rawfile.ErrNoChange {
164+
if err == content.ErrNoChange {
164165
return oldvalue, nil
165166
}
166167
if err != nil {

‎asnannotator/asn_test.go‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,42 +10,41 @@ import (
1010
"testing"
1111

1212
"github.com/go-test/deep"
13+
"github.com/m-lab/go/content"
1314
"github.com/m-lab/go/rtx"
1415
"github.com/m-lab/tcp-info/inetdiag"
1516
"github.com/m-lab/uuid-annotator/annotator"
16-
"github.com/m-lab/uuid-annotator/rawfile"
1717
)
1818

19-
var local4Rawfile rawfile.Provider
20-
var local6Rawfile rawfile.Provider
21-
var localASNamesfile rawfile.Provider
22-
var corruptFile rawfile.Provider
19+
var local4Rawfile content.Provider
20+
var local6Rawfile content.Provider
21+
var localASNamesfile content.Provider
22+
var corruptFile content.Provider
2323
var localIPs []net.IP
2424

2525
func setUp() {
2626
var err error
2727
u4, err := url.Parse("file:../testdata/RouteViewIPv4.pfx2as.gz")
2828
rtx.Must(err, "Could not parse URL")
29-
local4Rawfile, err = rawfile.FromURL(context.Background(), u4)
30-
rtx.Must(err, "Could not create rawfile.Provider")
29+
local4Rawfile, err = content.FromURL(context.Background(), u4)
30+
rtx.Must(err, "Could not create content.Provider")
3131

3232
u6, err := url.Parse("file:../testdata/RouteViewIPv6.pfx2as.gz")
3333
rtx.Must(err, "Could not parse URL")
34-
local6Rawfile, err = rawfile.FromURL(context.Background(), u6)
35-
rtx.Must(err, "Could not create rawfile.Provider")
34+
local6Rawfile, err = content.FromURL(context.Background(), u6)
35+
rtx.Must(err, "Could not create content.Provider")
3636

3737
asn, err := url.Parse("file:../data/asnames.ipinfo.csv")
3838
rtx.Must(err, "Could not parse URL")
39-
localASNamesfile, err = rawfile.FromURL(context.Background(), asn)
40-
rtx.Must(err, "Could not create rawfile.Provider")
39+
localASNamesfile, err = content.FromURL(context.Background(), asn)
40+
rtx.Must(err, "Could not create content.Provider")
4141

4242
cor, err := url.Parse("file:../testdata/corrupt.gz")
4343
rtx.Must(err, "Could not parse URL")
44-
corruptFile, err = rawfile.FromURL(context.Background(), cor)
45-
rtx.Must(err, "Could not create rawfile.Provider")
46-
47-
log.SetFlags(0) // log.Lshortfile | log.LstdFlags)
44+
corruptFile, err = content.FromURL(context.Background(), cor)
45+
rtx.Must(err, "Could not create content.Provider")
4846

47+
log.SetFlags(log.Lshortfile | log.LstdFlags)
4948
}
5049

5150
func Test_asnAnnotator_Annotate(t *testing.T) {
@@ -209,9 +208,9 @@ func Test_asnAnnotator_Reload(t *testing.T) {
209208
setUp()
210209
tests := []struct {
211210
name string
212-
as4 rawfile.Provider
213-
as6 rawfile.Provider
214-
asnamedata rawfile.Provider
211+
as4 content.Provider
212+
as6 content.Provider
213+
asnamedata content.Provider
215214
}{
216215
{
217216
name: "success",
@@ -227,7 +226,7 @@ func Test_asnAnnotator_Reload(t *testing.T) {
227226
},
228227
{
229228
name: "v4-no-change",
230-
as4: badProvider{rawfile.ErrNoChange},
229+
as4: badProvider{content.ErrNoChange},
231230
as6: local6Rawfile,
232231
asnamedata: localASNamesfile,
233232
},
@@ -240,7 +239,7 @@ func Test_asnAnnotator_Reload(t *testing.T) {
240239
{
241240
name: "v6-no-change",
242241
as4: local4Rawfile,
243-
as6: badProvider{rawfile.ErrNoChange},
242+
as6: badProvider{content.ErrNoChange},
244243
asnamedata: localASNamesfile,
245244
},
246245
{
@@ -253,7 +252,7 @@ func Test_asnAnnotator_Reload(t *testing.T) {
253252
name: "names-no-change",
254253
as4: local4Rawfile,
255254
as6: local6Rawfile,
256-
asnamedata: badProvider{rawfile.ErrNoChange},
255+
asnamedata: badProvider{content.ErrNoChange},
257256
},
258257
{
259258
name: "names-not-a-csv",

‎geoannotator/ipannotator.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import (
88
"net"
99
"sync"
1010

11+
"github.com/m-lab/go/content"
1112
"github.com/m-lab/go/rtx"
1213
"github.com/oschwald/geoip2-golang"
1314

1415
"github.com/m-lab/tcp-info/inetdiag"
1516
"github.com/m-lab/uuid-annotator/annotator"
16-
"github.com/m-lab/uuid-annotator/rawfile"
17+
"github.com/m-lab/uuid-annotator/tarreader"
1718
)
1819

1920
// GeoAnnotator is just a regular annotator with a Reload method and an AnnotateIP method.
@@ -27,7 +28,7 @@ type GeoAnnotator interface {
2728
type geoannotator struct {
2829
mut sync.RWMutex
2930
localIPs []net.IP
30-
backingDataSource rawfile.Provider
31+
backingDataSource content.Provider
3132
maxmind *geoip2.Reader
3233
}
3334

@@ -137,13 +138,13 @@ func (g *geoannotator) Reload(ctx context.Context) {
137138
// load unconditionally loads datasets and returns them.
138139
func (g *geoannotator) load(ctx context.Context) (*geoip2.Reader, error) {
139140
tgz, err := g.backingDataSource.Get(ctx)
140-
if err == rawfile.ErrNoChange {
141+
if err == content.ErrNoChange {
141142
return g.maxmind, nil
142143
}
143144
if err != nil {
144145
return nil, err
145146
}
146-
data, err := rawfile.FromTarGZ(tgz, "GeoLite2-City.mmdb")
147+
data, err := tarreader.FromTarGZ(tgz, "GeoLite2-City.mmdb")
147148
if err != nil {
148149
return nil, err
149150
}
@@ -153,7 +154,7 @@ func (g *geoannotator) load(ctx context.Context) (*geoip2.Reader, error) {
153154
// New makes a new Annotator that uses IP addresses to generate geolocation and
154155
// ASNumber metadata for that IP based on the current copy of MaxMind data
155156
// stored in GCS.
156-
func New(ctx context.Context, geo rawfile.Provider, localIPs []net.IP) GeoAnnotator {
157+
func New(ctx context.Context, geo content.Provider, localIPs []net.IP) GeoAnnotator {
157158
g := &geoannotator{
158159
backingDataSource: geo,
159160
localIPs: localIPs,

‎geoannotator/ipannotator_test.go‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,19 @@ import (
1010
"testing"
1111

1212
"github.com/go-test/deep"
13+
"github.com/m-lab/go/content"
1314
"github.com/m-lab/go/pretty"
1415
"github.com/m-lab/go/rtx"
1516
"github.com/m-lab/tcp-info/inetdiag"
1617
"github.com/oschwald/geoip2-golang"
1718

1819
"github.com/m-lab/uuid-annotator/annotator"
19-
"github.com/m-lab/uuid-annotator/rawfile"
20+
"github.com/m-lab/uuid-annotator/tarreader"
2021
)
2122

22-
var localRawfile rawfile.Provider
23-
var localWrongType rawfile.Provider
24-
var localEmpty rawfile.Provider
23+
var localRawfile content.Provider
24+
var localWrongType content.Provider
25+
var localEmpty content.Provider
2526

2627
// Networks taken from https://github.com/maxmind/MaxMind-DB/blob/master/source-data/GeoIP2-City-Test.json
2728
var localIP = "175.16.199.3"
@@ -31,18 +32,18 @@ func setUp() {
3132
var err error
3233
u, err := url.Parse("file:../testdata/fake.tar.gz")
3334
rtx.Must(err, "Could not parse URL")
34-
localRawfile, err = rawfile.FromURL(context.Background(), u)
35-
rtx.Must(err, "Could not create rawfile.Provider")
35+
localRawfile, err = content.FromURL(context.Background(), u)
36+
rtx.Must(err, "Could not create content.Provider")
3637

3738
u, err = url.Parse("file:../testdata/wrongtype.tar.gz")
3839
rtx.Must(err, "Could not parse URL")
39-
localWrongType, err = rawfile.FromURL(context.Background(), u)
40-
rtx.Must(err, "Could not create rawfile.Provider")
40+
localWrongType, err = content.FromURL(context.Background(), u)
41+
rtx.Must(err, "Could not create content.Provider")
4142

4243
u, err = url.Parse("file:../testdata/empty.tar.gz")
4344
rtx.Must(err, "Could not parse URL")
44-
localEmpty, err = rawfile.FromURL(context.Background(), u)
45-
rtx.Must(err, "Could not create rawfile.Provider")
45+
localEmpty, err = content.FromURL(context.Background(), u)
46+
rtx.Must(err, "Could not create content.Provider")
4647

4748
log.SetFlags(log.Lshortfile | log.LstdFlags)
4849
}
@@ -219,7 +220,7 @@ func TestIPAnnotationLoadNoChange(t *testing.T) {
219220
ctx := context.Background()
220221
fakeReader := geoip2.Reader{}
221222
g := geoannotator{
222-
backingDataSource: badProvider{rawfile.ErrNoChange},
223+
backingDataSource: badProvider{content.ErrNoChange},
223224
localIPs: []net.IP{net.ParseIP(localIP)},
224225
maxmind: &fakeReader, // NOTE: fake pointer just to verify return value below.
225226
}
@@ -294,8 +295,8 @@ func TestIPAnnotationMissingCityDB(t *testing.T) {
294295
}
295296

296297
mm, err := g.load(ctx)
297-
if err != rawfile.ErrFileNotFound {
298-
t.Errorf("geoannotator.load() returned wrong error; got %q, want %q", err, rawfile.ErrFileNotFound)
298+
if err != tarreader.ErrFileNotFound {
299+
t.Errorf("geoannotator.load() returned wrong error; got %q, want %q", err, tarreader.ErrFileNotFound)
299300
}
300301
if mm != nil {
301302
t.Errorf("geoannotator.load() return non-nil ptr; got %v, want nil", mm)

‎ipservice/ipservice_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ import (
1414
"sync"
1515
"testing"
1616

17+
"github.com/m-lab/go/content"
1718
"github.com/m-lab/go/rtx"
1819
"github.com/m-lab/uuid-annotator/annotator"
1920
"github.com/m-lab/uuid-annotator/asnannotator"
2021
"github.com/m-lab/uuid-annotator/geoannotator"
21-
"github.com/m-lab/uuid-annotator/rawfile"
2222
)
2323

2424
var (
@@ -32,16 +32,16 @@ func init() {
3232
// Set up ASN annotator.
3333
u4, err := url.Parse("file:../testdata/RouteViewIPv4.pfx2as.gz")
3434
rtx.Must(err, "Could not parse URL")
35-
local4Rawfile, err := rawfile.FromURL(context.Background(), u4)
36-
rtx.Must(err, "Could not create rawfile.Provider")
35+
local4Rawfile, err := content.FromURL(context.Background(), u4)
36+
rtx.Must(err, "Could not create content.Provider")
3737
u6, err := url.Parse("file:../testdata/RouteViewIPv6.pfx2as.gz")
3838
rtx.Must(err, "Could not parse URL")
39-
local6Rawfile, err := rawfile.FromURL(context.Background(), u6)
40-
rtx.Must(err, "Could not create rawfile.Provider")
39+
local6Rawfile, err := content.FromURL(context.Background(), u6)
40+
rtx.Must(err, "Could not create content.Provider")
4141
as, err := url.Parse("file:../data/asnames.ipinfo.csv")
4242
rtx.Must(err, "Could not parse URL")
43-
localASNamesfile, err := rawfile.FromURL(context.Background(), as)
44-
rtx.Must(err, "Could not create rawfile.Provider")
43+
localASNamesfile, err := content.FromURL(context.Background(), as)
44+
rtx.Must(err, "Could not create content.Provider")
4545

4646
localIPs := []net.IP{
4747
net.ParseIP("9.0.0.9"),
@@ -52,8 +52,8 @@ func init() {
5252
// Set up geo annotator.
5353
u, err := url.Parse("file:../testdata/fake.tar.gz")
5454
rtx.Must(err, "Could not parse URL")
55-
localRawfile, err := rawfile.FromURL(context.Background(), u)
56-
rtx.Must(err, "Could not create rawfile.Provider")
55+
localRawfile, err := content.FromURL(context.Background(), u)
56+
rtx.Must(err, "Could not create content.Provider")
5757
geo = geoannotator.New(ctx, localRawfile, localIPs)
5858
}
5959

‎main.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"sync"
99
"time"
1010

11+
"github.com/m-lab/go/content"
1112
"github.com/m-lab/go/flagx"
1213
"github.com/m-lab/go/memoryless"
1314
"github.com/m-lab/go/prometheusx"
@@ -19,7 +20,6 @@ import (
1920
"github.com/m-lab/uuid-annotator/geoannotator"
2021
"github.com/m-lab/uuid-annotator/handler"
2122
"github.com/m-lab/uuid-annotator/ipservice"
22-
"github.com/m-lab/uuid-annotator/rawfile"
2323
"github.com/m-lab/uuid-annotator/siteannotator"
2424
)
2525

@@ -36,7 +36,7 @@ var (
3636
// Reloading relatively frequently should be fine as long as (a) download
3737
// failure is non-fatal for reloads and (b) cache-checking actually works so
3838
// that we don't re-download the data until it is new. The first condition is
39-
// enforced in the geoannotator package, and the second in rawfile.
39+
// enforced in the geoannotator package, and the second in content.
4040
reloadMin = flag.Duration("reloadmin", time.Hour, "Minimum time to wait between reloads of backing data")
4141
reloadTime = flag.Duration("reloadtime", 5*time.Hour, "Expected time to wait between reloads of backing data")
4242
reloadMax = flag.Duration("reloadmax", 24*time.Hour, "Maximum time to wait between reloads of backing data")
@@ -87,19 +87,19 @@ func main() {
8787
localAddrs, err := net.InterfaceAddrs()
8888
rtx.Must(err, "Could not read local addresses")
8989
localIPs := findLocalIPs(localAddrs)
90-
p, err := rawfile.FromURL(mainCtx, maxmindurl.URL)
90+
p, err := content.FromURL(mainCtx, maxmindurl.URL)
9191
rtx.Must(err, "Could not get maxmind data from url")
9292
geo := geoannotator.New(mainCtx, p, localIPs)
9393

94-
p4, err := rawfile.FromURL(mainCtx, routeviewv4.URL)
94+
p4, err := content.FromURL(mainCtx, routeviewv4.URL)
9595
rtx.Must(err, "Could not load routeview v4 URL")
96-
p6, err := rawfile.FromURL(mainCtx, routeviewv6.URL)
96+
p6, err := content.FromURL(mainCtx, routeviewv6.URL)
9797
rtx.Must(err, "Could not load routeview v6 URL")
98-
asnames, err := rawfile.FromURL(mainCtx, asnameurl.URL)
98+
asnames, err := content.FromURL(mainCtx, asnameurl.URL)
9999
rtx.Must(err, "Could not load AS names URL")
100100
asn := asnannotator.New(mainCtx, p4, p6, asnames, localIPs)
101101

102-
js, err := rawfile.FromURL(mainCtx, siteinfo.URL)
102+
js, err := content.FromURL(mainCtx, siteinfo.URL)
103103
rtx.Must(err, "Could not load siteinfo URL")
104104
site := siteannotator.New(mainCtx, *hostname, js, localIPs)
105105

0 commit comments

Comments
 (0)