diff --git a/.env.example b/.env.example index 53e873d..1ca6c07 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,4 @@ CH_BATCH_TIMEOUT=5s REDIS_URL=redis://localhost:6379 ANONYMIZATION_SALT=change-me LOG_LEVEL=info +GEOIP_DB_PATH=data/GeoLite2-City.mmdb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe0928..278d7ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,14 @@ jobs: go-version-file: go.mod cache: true + - name: Download GeoLite2 City database + env: + MAXMIND_LICENSE_KEY: ${{ secrets.MAXMIND_LICENSE_KEY }} + run: | + mkdir -p data + wget -nv -O- "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=$MAXMIND_LICENSE_KEY&suffix=tar.gz" | tar xz -C data + cp data/GeoLite2-City_*/GeoLite2-City.mmdb data/GeoLite2-City.mmdb + - name: go vet run: go vet ./... diff --git a/.gitignore b/.gitignore index 810c24e..cb93e54 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ go.work.sum # Editor/IDE # .idea/ # .vscode/ + +data/ diff --git a/Dockerfile b/Dockerfile index 2f7ea75..9749ea2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,9 @@ WORKDIR /app COPY --from=builder /app/bin/processor/ /bin/processor +# GeoIP Database +COPY data/GeoLite2-City.mmdb /data/GeoLite2-City.mmdb + RUN chown -R appuser:appuser /app USER appuser diff --git a/cmd/processor/main.go b/cmd/processor/main.go index e97b233..096ccb4 100644 --- a/cmd/processor/main.go +++ b/cmd/processor/main.go @@ -134,7 +134,16 @@ func main() { app := server.NewServer(cfg.Port) // 4. Initialize event consumer - anonymizer := processors.NewAnonymizer(cfg.AnonymizationSalt) + geoIP, err := processors.NewGeoIPResolver(cfg.GeoIPDBPath) + if err != nil { + logger.Error().Err(err).Msg("failed to initialize geoip resolver") + return + } + defer func() { + //nolint:errcheck + geoIP.Close() + }() + anonymizer := processors.NewAnonymizer(cfg.AnonymizationSalt, geoIP) uaParser := processors.NewUserAgentParser() redisClient := newRedisClient(ctx, cfg.RedisURL, logger) redisStore := storage.NewRedisStore(redisClient) diff --git a/go.mod b/go.mod index e2dfb3e..7bde3f4 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.26.0 require ( github.com/ClickHouse/clickhouse-go/v2 v2.43.0 + github.com/go-redis/redismock/v9 v9.2.0 github.com/google/uuid v1.6.0 github.com/mileusna/useragent v1.3.5 + github.com/oschwald/geoip2-golang/v2 v2.1.0 github.com/redis/go-redis/v9 v9.18.0 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 @@ -20,11 +22,11 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/go-redis/redismock/v9 v9.2.0 // indirect github.com/klauspost/compress v1.18.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/oschwald/maxminddb-golang/v2 v2.1.1 // indirect github.com/paulmach/orb v0.12.0 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index a0794e0..61f8f72 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= @@ -56,6 +58,16 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws= github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= +github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/oschwald/geoip2-golang/v2 v2.1.0 h1:DjnLhNJu9WHwTrmoiQFvgmyJoczhdnm7LB23UBI2Amo= +github.com/oschwald/geoip2-golang/v2 v2.1.0/go.mod h1:qdVmcPgrTJ4q2eP9tHq/yldMTdp2VMr33uVdFbHBiBc= +github.com/oschwald/maxminddb-golang/v2 v2.1.1 h1:lA8FH0oOrM4u7mLvowq8IT6a3Q/qEnqRzLQn9eH5ojc= +github.com/oschwald/maxminddb-golang/v2 v2.1.1/go.mod h1:PLdx6PR+siSIoXqqy7C7r3SB3KZnhxWr1Dp6g0Hacl8= github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= @@ -118,6 +130,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -138,6 +152,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -152,6 +168,8 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index 6a3c93d..12cea8a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,8 +47,9 @@ type ProcessorConfig struct { ClickhouseMaxIdleConns int ClickhouseConnOpenStrategy string RedisURL string - AnonymizationSalt string LogLevel string + AnonymizationSalt string + GeoIPDBPath string } func LoadProcessorConfig() ProcessorConfig { @@ -90,6 +91,7 @@ func LoadProcessorConfig() ProcessorConfig { RedisURL: getEnv("REDIS_URL"), LogLevel: getEnv("LOG_LEVEL"), AnonymizationSalt: getEnv("ANONYMIZATION_SALT"), + GeoIPDBPath: getEnv("GEOIP_DB_PATH"), } } diff --git a/internal/consumer/consumer_test.go b/internal/consumer/consumer_test.go index 78e2e72..c2cae23 100644 --- a/internal/consumer/consumer_test.go +++ b/internal/consumer/consumer_test.go @@ -527,7 +527,7 @@ func testPipeline() *pipeline.Pipeline { } logger := zerolog.Nop() return pipeline.NewPipeline( - processors.NewAnonymizer("test-salt"), + processors.NewAnonymizer("test-salt", nil), processors.NewUserAgentParser(), nil, writer, diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go index eb39d3e..673951c 100644 --- a/internal/pipeline/pipeline_test.go +++ b/internal/pipeline/pipeline_test.go @@ -55,7 +55,7 @@ func TestProcessEventDerivesUUIDAndClampsScreenWidth(t *testing.T) { } logger := zerolog.Nop() p := NewPipeline( - processors.NewAnonymizer("test-salt"), + processors.NewAnonymizer("test-salt", nil), processors.NewUserAgentParser(), nil, writer, diff --git a/internal/processors/anonymizer.go b/internal/processors/anonymizer.go index 9f77b86..697870c 100644 --- a/internal/processors/anonymizer.go +++ b/internal/processors/anonymizer.go @@ -9,29 +9,26 @@ import ( type Anonymizer struct { dailySalt string // use a static salt for simplicity. will add rotation later + geoIP *GeoIPResolver } -func NewAnonymizer(salt string) *Anonymizer { - return &Anonymizer{dailySalt: salt} +func NewAnonymizer(salt string, geoIP *GeoIPResolver) *Anonymizer { + return &Anonymizer{dailySalt: salt, geoIP: geoIP} } // Anonymize truncates the IP and returns a hash + geo data. // geo lookup is stubbed. will add MaxMind GeoIP later func (a *Anonymizer) Anonymize(rawIP string) (ipHash, country, region string) { truncated := truncateIP(rawIP) - if truncated == "" { - // Keep IP as unknown when parsing fails. - country = "" - region = "" - return + + // Geo lookup on truncated IP (before discarding) + if a.geoIP != nil { + country, region = a.geoIP.Resolve(truncated) } hash := sha256.Sum256([]byte(truncated + a.dailySalt)) ipHash = fmt.Sprintf("%x", hash[:16]) // 32-char hex - // returns empty strings - country = "" - region = "" return } diff --git a/internal/processors/anonymizer_test.go b/internal/processors/anonymizer_test.go index e557f79..0d5c69a 100644 --- a/internal/processors/anonymizer_test.go +++ b/internal/processors/anonymizer_test.go @@ -27,7 +27,7 @@ func TestTruncateIP(t *testing.T) { } func TestAnonymizeUsesTruncatedIPv4Range(t *testing.T) { - a := NewAnonymizer("test-salt") + a := NewAnonymizer("test-salt", nil) firstHash, firstCountry, firstRegion := a.Anonymize("192.168.1.123") secondHash, secondCountry, secondRegion := a.Anonymize("192.168.1.200") @@ -51,8 +51,8 @@ func TestAnonymizeUsesTruncatedIPv4Range(t *testing.T) { } func TestAnonymizeIncludesSaltInHash(t *testing.T) { - first := NewAnonymizer("salt-a") - second := NewAnonymizer("salt-b") + first := NewAnonymizer("salt-a", nil) + second := NewAnonymizer("salt-b", nil) firstHash, _, _ := first.Anonymize("203.0.113.7") secondHash, _, _ := second.Anonymize("203.0.113.7") diff --git a/internal/processors/geoip.go b/internal/processors/geoip.go new file mode 100644 index 0000000..099eb63 --- /dev/null +++ b/internal/processors/geoip.go @@ -0,0 +1,44 @@ +package processors + +import ( + "fmt" + "net/netip" + + "github.com/oschwald/geoip2-golang/v2" +) + +type GeoIPResolver struct { + db *geoip2.Reader +} + +func NewGeoIPResolver(dbPath string) (*GeoIPResolver, error) { + db, err := geoip2.Open(dbPath) + if err != nil { + return nil, fmt.Errorf("open GeoIP database: %w", err) + } + return &GeoIPResolver{db: db}, nil +} + +// Resolve returns country and region for an IP address. +// Pass the truncated IP as GeoLite2 still works with /24 truncation for country-level. +func (g *GeoIPResolver) Resolve(ip string) (country, region string) { + parsed, err := netip.ParseAddr(ip) + if err != nil { + return "", "" + } + + record, err := g.db.City(parsed) + if err != nil { + return "", "" + } + + country = record.Country.Names.English + if len(record.Subdivisions) > 0 { + region = record.Subdivisions[0].Names.English + } + return +} + +func (g *GeoIPResolver) Close() error { + return g.db.Close() +} diff --git a/internal/processors/geoip_test.go b/internal/processors/geoip_test.go new file mode 100644 index 0000000..6eb66ce --- /dev/null +++ b/internal/processors/geoip_test.go @@ -0,0 +1,57 @@ +package processors + +import ( + "path/filepath" + "testing" +) + +func newTestGeoIPResolver(t *testing.T) *GeoIPResolver { + t.Helper() + + dbPath := filepath.Join("..", "..", "data", "GeoLite2-City.mmdb") + resolver, err := NewGeoIPResolver(dbPath) + if err != nil { + t.Fatalf("NewGeoIPResolver(%q) error = %v", dbPath, err) + } + + t.Cleanup(func() { + if err := resolver.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + }) + + return resolver +} + +func TestGeoIPResolverResolvePublicIP(t *testing.T) { + resolver := newTestGeoIPResolver(t) + + country, region := resolver.Resolve("81.2.69.142") + + if country != "United Kingdom" { + t.Fatalf("country = %q, want %q", country, "United Kingdom") + } + if region != "England" { + t.Fatalf("region = %q, want %q", region, "England") + } +} + +func TestGeoIPResolverResolveInvalidIP(t *testing.T) { + resolver := newTestGeoIPResolver(t) + + country, region := resolver.Resolve("not-an-ip") + + if country != "" || region != "" { + t.Fatalf("expected empty result for invalid ip, got %q/%q", country, region) + } +} + +func TestGeoIPResolverResolvePrivateIP(t *testing.T) { + resolver := newTestGeoIPResolver(t) + + country, region := resolver.Resolve("192.168.1.1") + + if country != "" || region != "" { + t.Fatalf("expected empty result for private ip, got %q/%q", country, region) + } +}