MaxMind GeoIP

.env.example

@@ -32,3 +32,4 @@ CH_BATCH_TIMEOUT=5s
 REDIS_URL=redis://localhost:6379
 ANONYMIZATION_SALT=change-me
 LOG_LEVEL=info
+GEOIP_DB_PATH=data/GeoLite2-City.mmdb

.github/workflows/ci.yml

@@ -18,6 +18,14 @@ jobs:
           go-version-file: go.mod
           cache: true
 
+      - name: Download GeoLite2 City database
+        env:
+          MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }}
+        run: |
+          mkdir -p data
+          wget -nv -O- "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=$MAXMIND_LICENSE_KEY&suffix=tar.gz" | tar xz -C data
+          cp data/GeoLite2-City_*/GeoLite2-City.mmdb data/GeoLite2-City.mmdb
+
       - name: go vet
         run: go vet ./...
 

.gitignore

@@ -32,3 +32,5 @@ go.work.sum
 # Editor/IDE
 # .idea/
 # .vscode/
+
+data/

Dockerfile

@@ -21,6 +21,9 @@ WORKDIR /app
 
 COPY --from=builder /app/bin/processor/ /bin/processor
 
+# GeoIP Database
+COPY data/GeoLite2-City.mmdb /data/GeoLite2-City.mmdb
+
 RUN chown -R appuser:appuser /app
 
 USER appuser

cmd/processor/main.go

@@ -134,7 +134,16 @@ func main() {
 	app := server.NewServer(cfg.Port)
 
 	// 4. Initialize event consumer
-	anonymizer := processors.NewAnonymizer(cfg.AnonymizationSalt)
+	geoIP, err := processors.NewGeoIPResolver(cfg.GeoIPDBPath)
+	if err != nil {
+		logger.Error().Err(err).Msg("failed to initialize geoip resolver")
+		return
+	}
+	defer func() {
+		//nolint:errcheck
+		geoIP.Close()
+	}()
+	anonymizer := processors.NewAnonymizer(cfg.AnonymizationSalt, geoIP)
 	uaParser := processors.NewUserAgentParser()
 	redisClient := newRedisClient(ctx, cfg.RedisURL, logger)
 	redisStore := storage.NewRedisStore(redisClient)

go.mod

@@ -4,8 +4,10 @@ go 1.26.0
 
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.43.0
+	github.com/go-redis/redismock/v9 v9.2.0
 	github.com/google/uuid v1.6.0
 	github.com/mileusna/useragent v1.3.5
+	github.com/oschwald/geoip2-golang/v2 v2.1.0
 	github.com/redis/go-redis/v9 v9.18.0
 	github.com/rs/zerolog v1.34.0
 	github.com/stretchr/testify v1.11.1
@@ -20,11 +22,11 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/go-faster/city v1.0.1 // indirect
 	github.com/go-faster/errors v0.7.1 // indirect
-	github.com/go-redis/redismock/v9 v9.2.0 // indirect
 	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/oschwald/maxminddb-golang/v2 v2.1.1 // indirect
 	github.com/paulmach/orb v0.12.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.26 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -17,6 +17,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
 github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
 github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
@@ -56,6 +58,16 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws=
 github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
+github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
+github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
+github.com/oschwald/geoip2-golang/v2 v2.1.0 h1:DjnLhNJu9WHwTrmoiQFvgmyJoczhdnm7LB23UBI2Amo=
+github.com/oschwald/geoip2-golang/v2 v2.1.0/go.mod h1:qdVmcPgrTJ4q2eP9tHq/yldMTdp2VMr33uVdFbHBiBc=
+github.com/oschwald/maxminddb-golang/v2 v2.1.1 h1:lA8FH0oOrM4u7mLvowq8IT6a3Q/qEnqRzLQn9eH5ojc=
+github.com/oschwald/maxminddb-golang/v2 v2.1.1/go.mod h1:PLdx6PR+siSIoXqqy7C7r3SB3KZnhxWr1Dp6g0Hacl8=
 github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s=
 github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
@@ -118,6 +130,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -138,6 +152,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -152,6 +168,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/config/config.go

@@ -47,8 +47,9 @@ type ProcessorConfig struct {
 	ClickhouseMaxIdleConns         int
 	ClickhouseConnOpenStrategy     string
 	RedisURL                       string
-	AnonymizationSalt              string
 	LogLevel                       string
+	AnonymizationSalt              string
+	GeoIPDBPath                    string
 }
 
 func LoadProcessorConfig() ProcessorConfig {
@@ -90,6 +91,7 @@ func LoadProcessorConfig() ProcessorConfig {
 		RedisURL:                     getEnv("REDIS_URL"),
 		LogLevel:                     getEnv("LOG_LEVEL"),
 		AnonymizationSalt:            getEnv("ANONYMIZATION_SALT"),
+		GeoIPDBPath:                  getEnv("GEOIP_DB_PATH"),
 	}
 }
 

internal/consumer/consumer_test.go

@@ -527,7 +527,7 @@ func testPipeline() *pipeline.Pipeline {
 	}
 	logger := zerolog.Nop()
 	return pipeline.NewPipeline(
-		processors.NewAnonymizer("test-salt"),
+		processors.NewAnonymizer("test-salt", nil),
 		processors.NewUserAgentParser(),
 		nil,
 		writer,

internal/pipeline/pipeline_test.go

@@ -55,7 +55,7 @@ func TestProcessEventDerivesUUIDAndClampsScreenWidth(t *testing.T) {
 	}
 	logger := zerolog.Nop()
 	p := NewPipeline(
-		processors.NewAnonymizer("test-salt"),
+		processors.NewAnonymizer("test-salt", nil),
 		processors.NewUserAgentParser(),
 		nil,
 		writer,

internal/processors/anonymizer.go

@@ -9,29 +9,26 @@ import (
 
 type Anonymizer struct {
 	dailySalt string // use a static salt for simplicity. will add rotation later
+	geoIP     *GeoIPResolver
 }
 
-func NewAnonymizer(salt string) *Anonymizer {
-	return &Anonymizer{dailySalt: salt}
+func NewAnonymizer(salt string, geoIP *GeoIPResolver) *Anonymizer {
+	return &Anonymizer{dailySalt: salt, geoIP: geoIP}
 }
 
 // Anonymize truncates the IP and returns a hash + geo data.
 // geo lookup is stubbed. will add MaxMind GeoIP later
 func (a *Anonymizer) Anonymize(rawIP string) (ipHash, country, region string) {
 	truncated := truncateIP(rawIP)
-	if truncated == "" {
-		// Keep IP as unknown when parsing fails.
-		country = ""
-		region = ""
-		return
+
+	// Geo lookup on truncated IP (before discarding)
+	if a.geoIP != nil {
+		country, region = a.geoIP.Resolve(truncated)
 	}
 
 	hash := sha256.Sum256([]byte(truncated + a.dailySalt))
 	ipHash = fmt.Sprintf("%x", hash[:16]) // 32-char hex
 
-	// returns empty strings
-	country = ""
-	region = ""
 	return
 }
 

internal/processors/anonymizer_test.go

@@ -27,7 +27,7 @@ func TestTruncateIP(t *testing.T) {
 }
 
 func TestAnonymizeUsesTruncatedIPv4Range(t *testing.T) {
-	a := NewAnonymizer("test-salt")
+	a := NewAnonymizer("test-salt", nil)
 
 	firstHash, firstCountry, firstRegion := a.Anonymize("192.168.1.123")
 	secondHash, secondCountry, secondRegion := a.Anonymize("192.168.1.200")
@@ -51,8 +51,8 @@ func TestAnonymizeUsesTruncatedIPv4Range(t *testing.T) {
 }
 
 func TestAnonymizeIncludesSaltInHash(t *testing.T) {
-	first := NewAnonymizer("salt-a")
-	second := NewAnonymizer("salt-b")
+	first := NewAnonymizer("salt-a", nil)
+	second := NewAnonymizer("salt-b", nil)
 
 	firstHash, _, _ := first.Anonymize("203.0.113.7")
 	secondHash, _, _ := second.Anonymize("203.0.113.7")

internal/processors/geoip.go

@@ -0,0 +1,44 @@
+package processors
+
+import (
+	"fmt"
+	"net/netip"
+
+	"github.com/oschwald/geoip2-golang/v2"
+)
+
+type GeoIPResolver struct {
+	db *geoip2.Reader
+}
+
+func NewGeoIPResolver(dbPath string) (*GeoIPResolver, error) {
+	db, err := geoip2.Open(dbPath)
+	if err != nil {
+		return nil, fmt.Errorf("open GeoIP database: %w", err)
+	}
+	return &GeoIPResolver{db: db}, nil
+}
+
+// Resolve returns country and region for an IP address.
+// Pass the truncated IP as GeoLite2 still works with /24 truncation for country-level.
+func (g *GeoIPResolver) Resolve(ip string) (country, region string) {
+	parsed, err := netip.ParseAddr(ip)
+	if err != nil {
+		return "", ""
+	}
+
+	record, err := g.db.City(parsed)
+	if err != nil {
+		return "", ""
+	}
+
+	country = record.Country.Names.English
+	if len(record.Subdivisions) > 0 {
+		region = record.Subdivisions[0].Names.English
+	}
+	return
+}
+
+func (g *GeoIPResolver) Close() error {
+	return g.db.Close()
+}

internal/processors/geoip_test.go

@@ -0,0 +1,57 @@
+package processors
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func newTestGeoIPResolver(t *testing.T) *GeoIPResolver {
+	t.Helper()
+
+	dbPath := filepath.Join("..", "..", "data", "GeoLite2-City.mmdb")
+	resolver, err := NewGeoIPResolver(dbPath)
+	if err != nil {
+		t.Fatalf("NewGeoIPResolver(%q) error = %v", dbPath, err)
+	}
+
+	t.Cleanup(func() {
+		if err := resolver.Close(); err != nil {
+			t.Fatalf("Close() error = %v", err)
+		}
+	})
+
+	return resolver
+}
+
+func TestGeoIPResolverResolvePublicIP(t *testing.T) {
+	resolver := newTestGeoIPResolver(t)
+
+	country, region := resolver.Resolve("81.2.69.142")
+
+	if country != "United Kingdom" {
+		t.Fatalf("country = %q, want %q", country, "United Kingdom")
+	}
+	if region != "England" {
+		t.Fatalf("region = %q, want %q", region, "England")
+	}
+}
+
+func TestGeoIPResolverResolveInvalidIP(t *testing.T) {
+	resolver := newTestGeoIPResolver(t)
+
+	country, region := resolver.Resolve("not-an-ip")
+
+	if country != "" || region != "" {
+		t.Fatalf("expected empty result for invalid ip, got %q/%q", country, region)
+	}
+}
+
+func TestGeoIPResolverResolvePrivateIP(t *testing.T) {
+	resolver := newTestGeoIPResolver(t)
+
+	country, region := resolver.Resolve("192.168.1.1")
+
+	if country != "" || region != "" {
+		t.Fatalf("expected empty result for private ip, got %q/%q", country, region)
+	}
+}