Require geofeeds to be valid UTF-8

mm-david · mm-david · commit 9d3c6de68bfa · 2026-02-16T18:22:35.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 ## CHANGELOG
 
+## 4.0.0 (2026-02-16)
+
+* Require that geofeeds be encoded as valid UTF-8.
+
 ## 3.1.0 (2024-11-05)
 
 * Empty geofeeds will now be considered invalid, unless the (new) EmptyOK option
diff --git a/go.mod b/go.mod
@@ -1,19 +1,15 @@
-module github.com/maxmind/mm-geofeed-verifier/v3
+module github.com/maxmind/mm-geofeed-verifier/v4
 
 go 1.24.0
 
 require (
-	github.com/TomOnTime/utfutil v1.0.0
-	github.com/oschwald/geoip2-golang v1.13.0
+	github.com/oschwald/maxminddb-golang/v2 v2.1.0
 	github.com/stretchr/testify v1.11.1
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/oschwald/maxminddb-golang v1.13.0 // indirect
-	github.com/oschwald/maxminddb-golang/v2 v2.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -1,23 +1,13 @@
-github.com/TomOnTime/utfutil v1.0.0 h1:/0Ivgo2OjXJxo8i7zgvs7ewSFZMLwCRGm3P5Umowb90=
-github.com/TomOnTime/utfutil v1.0.0/go.mod h1:l9lZmOniizVSuIliSkEf87qivMRlSNzbdBFKjuLRg1c=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/oschwald/geoip2-golang v1.13.0 h1:Q44/Ldc703pasJeP5V9+aFSZFmBN7DKHbNsSFzQATJI=
-github.com/oschwald/geoip2-golang v1.13.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo=
-github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU=
-github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o=
 github.com/oschwald/maxminddb-golang/v2 v2.1.0 h1:2Iv7lmG9XtxuZA/jFAsd7LnZaC1E59pFsj5O/nU15pw=
 github.com/oschwald/maxminddb-golang/v2 v2.1.0/go.mod h1:gG4V88LsawPEqtbL1Veh1WRh+nVSYwXzJ1P5Fcn77g0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
 golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
-golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/main.go b/main.go
@@ -18,7 +18,7 @@ import (
 	"sort" //nolint:depguard // preexisting
 	"strings"
 
-	"github.com/maxmind/mm-geofeed-verifier/v3/verify"
+	"github.com/maxmind/mm-geofeed-verifier/v4/verify"
 )
 
 // This value is set by build scripts. Changing the name of
diff --git a/verify/errors.go b/verify/errors.go
@@ -3,6 +3,11 @@ package verify
 import "errors"
 
 var (
+	// ErrNotUTF8 indicates a file encoding that is not valid UTF-8 (with
+	// optional BOM). RFC 8805 says that "feeds MUST use UTF-8 character
+	// encoding". This is a separate error from ErrInvalidGeofeed, because we
+	// can't confidently read anything from the file if it's not UTF-8.
+	ErrNotUTF8 = errors.New("geofeed is not valid UTF-8")
 	// ErrInvalidGeofeed represents error that is returned in case of incomplete
 	// compliance with RFC 8805 standards and the mode in which the program is
 	// run.
diff --git a/verify/test_data/geofeed-valid-lax.csv b/verify/test_data/geofeed-valid-lax.csv
diff --git a/verify/test_data/geofeed-valid-optional-fields.csv b/verify/test_data/geofeed-valid-optional-fields.csv
diff --git a/verify/test_data/geofeed-valid-shiftjis.csv b/verify/test_data/geofeed-valid-shiftjis.csv
@@ -0,0 +1,4 @@
+ 2a02:ecc0::/29 ,US,US-NJ, Parsippany ,
+202.196.224.5/32,AT,AT-9,Vienna,1060,/22
+# Verifier should compare most specific subdivision, i.e. GB-WBK instead of GB-ENG
+2.125.160.216/29,GB,GB-WBK,����,
diff --git a/verify/test_data/geofeed-valid-utf16le.csv b/verify/test_data/geofeed-valid-utf16le.csv
diff --git a/verify/test_data/geofeed-valid-utf8-bom.csv b/verify/test_data/geofeed-valid-utf8-bom.csv
@@ -0,0 +1,4 @@
+﻿ 2a02:ecc0::/29 ,US,US-NJ, Parsippany ,
+202.196.224.5/32,AT,AT-9,Vienna,1060,/22
+# Verifier should compare most specific subdivision, i.e. GB-WBK instead of GB-ENG
+2.125.160.216/29,GB,GB-WBK,Boxford,
diff --git a/verify/test_data/geofeed-valid.csv b/verify/test_data/geofeed-valid.csv
diff --git a/verify/verify.go b/verify/verify.go
@@ -3,16 +3,17 @@
 package verify
 
 import (
+	"bytes"
 	"encoding/csv"
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/netip"
+	"os"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
-	"github.com/TomOnTime/utfutil"
 	"github.com/oschwald/maxminddb-golang/v2"
 )
 
@@ -62,20 +63,20 @@ func ProcessGeofeed(
 	c := NewCheckResult()
 	var diffLines []string
 
-	// Use utfutil to remove a BOM, if present (common on files from Windows).
-	// See https://github.com/golang/go/issues/33887.
-	geofeedFH, err := utfutil.OpenFile(filepath.Clean(geofeedFilename), utfutil.UTF8)
+	geofeedData, err := os.ReadFile(filepath.Clean(geofeedFilename))
 	if err != nil {
 		if opts.HideFilePathsInErrorMessages {
 			return c, diffLines, nil, fmt.Errorf("unable to open file: %w", err)
 		}
 		return c, diffLines, nil, fmt.Errorf("unable to open %s: %w", geofeedFilename, err)
 	}
-	defer func() {
-		if err := geofeedFH.Close(); err != nil {
-			log.Println(err)
-		}
-	}()
+
+	// Strip UTF-8 BOM if present (common on files from Windows).
+	geofeedData = bytes.TrimPrefix(geofeedData, []byte{0xEF, 0xBB, 0xBF})
+
+	if !utf8.Valid(geofeedData) {
+		return c, diffLines, nil, ErrNotUTF8
+	}
 
 	db, err := maxminddb.Open(filepath.Clean(mmdbFilename))
 	if err != nil {
@@ -99,7 +100,7 @@ func ProcessGeofeed(
 	}
 	asnCounts := map[uint]int{}
 
-	csvReader := csv.NewReader(geofeedFH)
+	csvReader := csv.NewReader(bytes.NewReader(geofeedData))
 	csvReader.ReuseRecord = true
 	csvReader.Comment = '#'
 	csvReader.FieldsPerRecord = -1
diff --git a/verify/verify_test.go b/verify/verify_test.go
@@ -75,6 +75,20 @@ func TestProcessGeofeed_Valid(t *testing.T) {
 			},
 			laxMode: false,
 		},
+		{
+			gf: "test_data/geofeed-valid-utf8-bom.csv",
+			db: "test_data/GeoIP2-City-Test.mmdb",
+			dl: []string{
+				"Found a potential improvement: '2a02:ecc0::/29",
+				"current postal code: '34021'\t\tsuggested postal code: '1060'",
+			},
+			c: CheckResult{
+				Total:             3,
+				Differences:       2,
+				SampleInvalidRows: map[RowInvalidity]string{},
+			},
+			laxMode: false,
+		},
 		{
 			gf: "test_data/empty.csv",
 			db: "test_data/GeoIP2-City-Test.mmdb",
@@ -211,3 +225,31 @@ func TestProcessGeofeed_Invalid(t *testing.T) {
 		)
 	}
 }
+
+func TestProcessGeofeed_NonUTF8(t *testing.T) {
+	tests := []struct {
+		gf   string
+		desc string
+	}{
+		{
+			gf:   "test_data/geofeed-valid-utf16le.csv",
+			desc: "UTF-16 LE encoded geofeed",
+		},
+		{
+			gf:   "test_data/geofeed-valid-shiftjis.csv",
+			desc: "Shift-JIS encoded geofeed",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			_, _, _, err := ProcessGeofeed(
+				test.gf,
+				"test_data/GeoIP2-City-Test.mmdb",
+				"",
+				Options{},
+			)
+			require.ErrorIs(t, err, ErrNotUTF8)
+		})
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ import (`
`18`	`18`	`"sort" //nolint:depguard // preexisting`
`19`	`19`	`"strings"`
`20`	`20`
`21`		`- "github.com/maxmind/mm-geofeed-verifier/v3/verify"`
	`21`	`+ "github.com/maxmind/mm-geofeed-verifier/v4/verify"`
`22`	`22`	`)`
`23`	`23`
`24`	`24`	`// This value is set by build scripts. Changing the name of`
-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 + 2a02:ecc0::/29 ,US,US-NJ, Parsippany ,
 +202.196.224.5/32,AT,AT-9,Vienna,1060,/22
 +# Verifier should compare most specific subdivision, i.e. GB-WBK instead of GB-ENG
 +2.125.160.216/29,GB,GB-WBK,“Œ‹ž,