From ee63113b3b5da34ac0be94df9d9e7f375ab07f23 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 08:51:12 +0200 Subject: [PATCH 01/25] feat: recover go-spatial attribute dictionary building func --- tile/go_spatial_feature.go | 214 +++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/tile/go_spatial_feature.go b/tile/go_spatial_feature.go index 7d348a6..7ce21c2 100644 --- a/tile/go_spatial_feature.go +++ b/tile/go_spatial_feature.go @@ -12,6 +12,7 @@ import ( "errors" "fmt" "log" + "slices" "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/wkt" @@ -407,6 +408,219 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er } } +// keyvalMapsFromFeatures returns a key map and value map, to help with the translation +// to mapbox tile format. In the Tile format, the Tile contains a mapping of all the unique +// keys and values, and then each feature contains a vector map to these two. This is an +// intermediate data structure to help with the construction of the three mappings. +func keyvalMapsFromFeatures(features []Feature) (keyMap []string, valMap []any, err error) { + var didFind bool + for _, f := range features { + for k, v := range f.Tags { + didFind = slices.Contains(keyMap, k) + if !didFind { + keyMap = append(keyMap, k) + } + didFind = false + + switch vt := v.(type) { + default: + if vt == nil { + // ignore nil types + continue + } + return keyMap, valMap, fmt.Errorf("unsupported type for value(%v) with key(%v) in tags for feature %v", vt, k, f) + + case string: + for _, mv := range valMap { + tmv, ok := mv.(string) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case fmt.Stringer: + for _, mv := range valMap { + tmv, ok := mv.(fmt.Stringer) + if !ok { + continue + } + if tmv.String() == vt.String() { + didFind = true + break + } + } + + case int: + for _, mv := range valMap { + tmv, ok := mv.(int) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case int8: + for _, mv := range valMap { + tmv, ok := mv.(int8) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case int16: + for _, mv := range valMap { + tmv, ok := mv.(int16) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case int32: + for _, mv := range valMap { + tmv, ok := mv.(int32) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case int64: + for _, mv := range valMap { + tmv, ok := mv.(int64) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case uint: + for _, mv := range valMap { + tmv, ok := mv.(uint) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case uint8: + for _, mv := range valMap { + tmv, ok := mv.(uint8) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case uint16: + for _, mv := range valMap { + tmv, ok := mv.(uint16) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case uint32: + for _, mv := range valMap { + tmv, ok := mv.(uint32) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case uint64: + for _, mv := range valMap { + tmv, ok := mv.(uint64) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case float32: + for _, mv := range valMap { + tmv, ok := mv.(float32) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case float64: + for _, mv := range valMap { + tmv, ok := mv.(float64) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + case bool: + for _, mv := range valMap { + tmv, ok := mv.(bool) + if !ok { + continue + } + if tmv == vt { + didFind = true + break + } + } + + } // value type switch + + if !didFind { + valMap = append(valMap, v) + } + + } // For f.Tags + } // for features + return keyMap, valMap, nil +} + // keyvalTagsMap will return the tags map as expected by the mapbox tile spec. It takes // a keyMap and a valueMap that list the order of the expected keys and values. It will // return a vector map that refers to these two maps. From c877203658202539ee6661ee2a21ac03ffecb18f Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 10:46:36 +0200 Subject: [PATCH 02/25] feat: read fids based on tile coords --- processing/gpkg/encoded.go | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go index a6b410b..c283034 100644 --- a/processing/gpkg/encoded.go +++ b/processing/gpkg/encoded.go @@ -7,6 +7,7 @@ import ( "github.com/go-spatial/geom/encoding/gpkg" "github.com/pdok/texel/processing" + "github.com/pdok/texel/tile" ) func (t Table) EncodedName() string { @@ -19,6 +20,11 @@ func (t Table) insertSQLEncoded() string { ` VALUES (?, ?, ?, ?, ?)` } +// selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile +func (t Table) selectEncodedSQL() string { + return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?` +} + func serializeToBytes(intslice []uint32) []byte { bytes := make([]byte, len(intslice)*4) @@ -29,6 +35,56 @@ func serializeToBytes(intslice []uint32) []byte { return bytes } +// deserializeFromBytes is the inverse of serializeToBytes: it turns a BLOB of +// little-endian uint32s back into the encoded geometry command/coordinate stream. +func deserializeFromBytes(data []byte) []uint32 { + intslice := make([]uint32, len(data)/4) + for i := range intslice { + intslice[i] = binary.LittleEndian.Uint32(data[i*4:]) + } + return intslice +} + +// EncodedFeatureRow pairs a feature's identifier with its encoded, tile-relative geometry +// as read back from the "_encoded" table. +type EncodedFeatureRow struct { + FeatureID int64 + Geom tile.EncodedGeometry +} + +// GetFeaturesForTile returns every encoded feature stored for the given tile, +// i.e. the equivalent of the tile-features table's rows for (tileX, tileY). +func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]EncodedFeatureRow, error) { + rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) + if err != nil { + return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) + } + defer rows.Close() + + var features []EncodedFeatureRow + for rows.Next() { + var featureID int64 + var geometryType int32 + var data []byte + if err := rows.Scan(&featureID, &geometryType, &data); err != nil { + return nil, fmt.Errorf("scanning encoded feature row for tile (%d,%d): %w", tileX, tileY, err) + } + features = append(features, EncodedFeatureRow{ + FeatureID: featureID, + Geom: tile.EncodedGeometry{ + Encoding: deserializeFromBytes(data), + GeometryType: geometryType, + XTile: tileX, + YTile: tileY, + }, + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating encoded feature rows for tile (%d,%d): %w", tileX, tileY, err) + } + return features, nil +} + func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.FeatureForTileMatrix) { tx, err := target.handle.Begin() if err != nil { From 23ceeb776050e72e42dd93a7952b45c78bd81ae0 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 11:39:44 +0200 Subject: [PATCH 03/25] feat: retrieve feature attributes based on fid list --- processing/gpkg/attributes.go | 130 ++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 processing/gpkg/attributes.go diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go new file mode 100644 index 0000000..2cd17a3 --- /dev/null +++ b/processing/gpkg/attributes.go @@ -0,0 +1,130 @@ +package gpkg + +import ( + "fmt" + "strings" + "time" +) + +// Batch value for reading features from the geopackage table +const maxSQLiteBatchParams = 500 + +// selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. +// The fid is assumed to be the table's first column. +func (t Table) selectByFIDsSQL(batchSize int) string { + var csql []string //nolint:prealloc + for _, c := range t.columns { + csql = append(csql, c.name) + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", batchSize), ",") + fidColumn := t.columns[0].name + return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` +} + +// GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) +// for the given feature IDs, batching the query to respect SQLite's parameter limits. +// The fid is assumed to be the table's first column. +func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (map[int64]map[string]any, error) { + attributes := make(map[int64]map[string]any, len(featureIDs)) + fidColumn := source.Table.columns[0].name + + for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { + batch := featureIDs[start:min(start+maxSQLiteBatchParams, len(featureIDs))] + + batchAsAny := make([]any, len(batch)) + for i, fid := range batch { + batchAsAny[i] = fid + } + + if err := source.queryAttributesBatch(fidColumn, batchAsAny, attributes); err != nil { + return nil, err + } + } + + return attributes, nil +} + +func (source SourceGeopackage) getColNames() []string { + cols := source.Table.columns + colNames := make([]string, len(cols)) + for i, col := range cols { + colNames[i] = col.name + } + return colNames +} + +// Read from source.Table, store rows as attributes[fid][columnName] = value for all fids +func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes map[int64]map[string]any) error { + // Read relevant rows + rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) + if err != nil { + return fmt.Errorf("querying attributes for %d feature(s): %w", len(fids), err) + } + defer rows.Close() + + cols := source.getColNames() + + // Process rows + for rows.Next() { + // Read data with helper vars + vals := make([]any, len(cols)) + valPtrs := make([]any, len(cols)) + for i := range cols { + valPtrs[i] = &vals[i] + } + if err := rows.Scan(valPtrs...); err != nil { + return fmt.Errorf("scanning attribute row: %w", err) + } + + // Interpret data + row := make(map[string]any, len(cols)) + var fid int64 + var fidFound bool + for i, colName := range cols { + switch colName { + case fidColumn: + // Read fid + id, ok := vals[i].(int64) + if !ok { + return fmt.Errorf("expected int64 fid for column %s, got %T", colName, vals[i]) + } + fid, fidFound = id, true + case source.Table.gcolumn: + // Skip geometry column + default: + // Interpret data + v, ok := cleanData(vals[i]) + if !ok { + return fmt.Errorf("unexpected type for sqlite column data: %v: %T", colName, v) + } + row[colName] = v + } + } + if !fidFound { + return fmt.Errorf("row did not contain fid column %s", fidColumn) + } + attributes[fid] = row + } + return rows.Err() +} + +func cleanData(v any) (any, bool) { + switch v := v.(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + return v, true + case int64: + return v, true + case float64: + return v, true + case time.Time: + return v.Format(time.RFC3339), true + case string: + return v, true + case nil: + return nil, true + default: + return nil, false + } +} From b150751b96d11cf8c9141eb49e04710659345368 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 11:47:10 +0200 Subject: [PATCH 04/25] feat: extract info from data structure for a single feature --- processing/gpkg/attributes.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go index 2cd17a3..696ace4 100644 --- a/processing/gpkg/attributes.go +++ b/processing/gpkg/attributes.go @@ -21,6 +21,14 @@ func (t Table) selectByFIDsSQL(batchSize int) string { return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` } +// Extract features from the map construct. +func AttributesForFeature(attributes map[int64]map[string]any, featureID int64) map[string]any { + if attrs, ok := attributes[featureID]; ok { + return attrs + } + return map[string]any{} +} + // GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) // for the given feature IDs, batching the query to respect SQLite's parameter limits. // The fid is assumed to be the table's first column. From 4688d936ba5f47ad9bc0d23bbaf17a1273c60619 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 13:28:41 +0200 Subject: [PATCH 05/25] refactor: name for internal representation attribute table --- processing/gpkg/attributes.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go index 696ace4..8b281f3 100644 --- a/processing/gpkg/attributes.go +++ b/processing/gpkg/attributes.go @@ -9,6 +9,10 @@ import ( // Batch value for reading features from the geopackage table const maxSQLiteBatchParams = 500 +// Data structure to store attribute info before processing. +// This could perhaps also be map[int64][]any for performance +type InternalAttributeTable map[int64]map[string]any + // selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. // The fid is assumed to be the table's first column. func (t Table) selectByFIDsSQL(batchSize int) string { @@ -22,7 +26,7 @@ func (t Table) selectByFIDsSQL(batchSize int) string { } // Extract features from the map construct. -func AttributesForFeature(attributes map[int64]map[string]any, featureID int64) map[string]any { +func AttributesForFeature(attributes InternalAttributeTable, featureID int64) map[string]any { if attrs, ok := attributes[featureID]; ok { return attrs } @@ -32,8 +36,8 @@ func AttributesForFeature(attributes map[int64]map[string]any, featureID int64) // GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) // for the given feature IDs, batching the query to respect SQLite's parameter limits. // The fid is assumed to be the table's first column. -func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (map[int64]map[string]any, error) { - attributes := make(map[int64]map[string]any, len(featureIDs)) +func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (InternalAttributeTable, error) { + attributes := make(InternalAttributeTable, len(featureIDs)) fidColumn := source.Table.columns[0].name for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { @@ -62,7 +66,7 @@ func (source SourceGeopackage) getColNames() []string { } // Read from source.Table, store rows as attributes[fid][columnName] = value for all fids -func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes map[int64]map[string]any) error { +func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes InternalAttributeTable) error { // Read relevant rows rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) if err != nil { From aa636f6a9daa48ca5b829db4b5647135f3e088f7 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 13:50:08 +0200 Subject: [PATCH 06/25] feat: construct keys dictionary --- processing/gpkg/encoded.go | 13 +++++++++++++ tile/keysvalues.go | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 tile/keysvalues.go diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go index c283034..c0215db 100644 --- a/processing/gpkg/encoded.go +++ b/processing/gpkg/encoded.go @@ -14,6 +14,19 @@ func (t Table) EncodedName() string { return t.Name + "_encoded" } +// List all columns except fid and geometry column +// Assumptions: fid column has index 0 +func (t Table) AttributeColumnNames() []string { + names := make([]string, 0, len(t.columns)) + for i, c := range t.columns { + if i == 0 || c.name == t.gcolumn { + continue // skip the fid and geometry columns, they are not attributes + } + names = append(names, c.name) + } + return names +} + func (t Table) insertSQLEncoded() string { return `INSERT INTO "` + t.EncodedName() + `"` + ` (tile_x, tile_y, feature_id, geometry_type, data)` + diff --git a/tile/keysvalues.go b/tile/keysvalues.go new file mode 100644 index 0000000..7c780fe --- /dev/null +++ b/tile/keysvalues.go @@ -0,0 +1,13 @@ +package tile + +// Convert slice to map for easy access of indices +func BuildKeyDictionary(attributeNames []string) map[string]uint32 { + index := make(map[string]uint32, len(attributeNames)) + for i, name := range attributeNames { + if i > int(^uint32(0)) { + panic("Number of keys does not fit in uint32") + } + index[name] = uint32(i) //nolint:gosec // G115 this is guarded + } + return index +} From b85a6d6ec7e2dcc7fbbbbb0f99e2cd57dbbccb57 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 14:07:32 +0200 Subject: [PATCH 07/25] feat: generate value dictionary --- processing/gpkg/attributes.go | 14 ++++++-------- tile/keysvalues.go | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go index 8b281f3..8c562c6 100644 --- a/processing/gpkg/attributes.go +++ b/processing/gpkg/attributes.go @@ -4,15 +4,13 @@ import ( "fmt" "strings" "time" + + "github.com/pdok/texel/tile" ) // Batch value for reading features from the geopackage table const maxSQLiteBatchParams = 500 -// Data structure to store attribute info before processing. -// This could perhaps also be map[int64][]any for performance -type InternalAttributeTable map[int64]map[string]any - // selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. // The fid is assumed to be the table's first column. func (t Table) selectByFIDsSQL(batchSize int) string { @@ -26,7 +24,7 @@ func (t Table) selectByFIDsSQL(batchSize int) string { } // Extract features from the map construct. -func AttributesForFeature(attributes InternalAttributeTable, featureID int64) map[string]any { +func AttributesForFeature(attributes tile.InternalAttributeTable, featureID int64) map[string]any { if attrs, ok := attributes[featureID]; ok { return attrs } @@ -36,8 +34,8 @@ func AttributesForFeature(attributes InternalAttributeTable, featureID int64) ma // GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) // for the given feature IDs, batching the query to respect SQLite's parameter limits. // The fid is assumed to be the table's first column. -func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (InternalAttributeTable, error) { - attributes := make(InternalAttributeTable, len(featureIDs)) +func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) { + attributes := make(tile.InternalAttributeTable, len(featureIDs)) fidColumn := source.Table.columns[0].name for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { @@ -66,7 +64,7 @@ func (source SourceGeopackage) getColNames() []string { } // Read from source.Table, store rows as attributes[fid][columnName] = value for all fids -func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes InternalAttributeTable) error { +func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes tile.InternalAttributeTable) error { // Read relevant rows rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) if err != nil { diff --git a/tile/keysvalues.go b/tile/keysvalues.go index 7c780fe..5346dd7 100644 --- a/tile/keysvalues.go +++ b/tile/keysvalues.go @@ -1,5 +1,9 @@ package tile +// InternalAttributeTable maps feature ID to a map of attribute name to value, as +// produced by reading a source table's attribute columns for a set of features. +type InternalAttributeTable map[int64]map[string]any + // Convert slice to map for easy access of indices func BuildKeyDictionary(attributeNames []string) map[string]uint32 { index := make(map[string]uint32, len(attributeNames)) @@ -11,3 +15,24 @@ func BuildKeyDictionary(attributeNames []string) map[string]uint32 { } return index } + +// BuildValueDictionary scans a tile's feature attributes and returns an index from +// value to a stable dictionary position. Values are compared by (dynamic type, value) - +// matching the MVT tag value semantics used by keyvalTagsMap - which comparable Go types +// (string, the int/uint/float variants, bool) support directly as map keys. nil values +// are skipped, matching the MVT spec's handling of absent tag values (see keyvalTagsMap). +func BuildValueDictionary(attributes InternalAttributeTable) map[any]uint32 { + index := make(map[any]uint32) + for _, attrs := range attributes { + for _, v := range attrs { + if v == nil { + continue + } + if _, ok := index[v]; ok { + continue + } + index[v] = uint32(len(index)) //nolint:gosec // G115 a tile cannot realistically hold more than 2^32 distinct values + } + } + return index +} From 1d688ab7e0ba30f4199a2fcc9e4a6c4ed5efe90f Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 14:27:06 +0200 Subject: [PATCH 08/25] feat: construct feature struct from go-spatial --- tile/build_feature.go | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 tile/build_feature.go diff --git a/tile/build_feature.go b/tile/build_feature.go new file mode 100644 index 0000000..c14ab23 --- /dev/null +++ b/tile/build_feature.go @@ -0,0 +1,46 @@ +package tile + +import "fmt" + +// Given attribute map and indices for keys and values, build tag list. +// Panic if key or value is not present. +func buildTags(attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) ([]uint32, error) { + tags := make([]uint32, 0, 2*len(attributes)) + for key, val := range attributes { + if val == nil { + continue + } + + kidx, ok := keyIndex[key] + if !ok { + return nil, fmt.Errorf("did not find key (%v) in key dictionary", key) + } + + vidx, ok := valueIndex[val] + if !ok { + return nil, fmt.Errorf("did not find value (%v) in value dictionary", val) + } + + tags = append(tags, kidx, vidx) + } + return tags, nil +} + +// Build go-spatial compatible feature that can be marshalled. +func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*GSTileFeature, error) { + tags, err := buildTags(attributes, keyIndex, valueIndex) + if err != nil { + return nil, fmt.Errorf("feature %d: %w", featureID, err) + } + + //nolint:gosec // G115 feature ids are expected to be non-negative + id := uint64(featureID) + gtype := GSTileGeomType(geom.GeometryType) + + return &GSTileFeature{ + Id: &id, + Tags: tags, + Type: >ype, + Geometry: geom.Encoding, + }, nil +} From 0fb1880f8a238ebc61bae61a03a3d48d8eb5d641 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 14:51:07 +0200 Subject: [PATCH 09/25] feat: encode data into layer --- go.mod | 2 +- tile/build_layer.go | 97 +++++++++++++++++++++++++++++++++++++ tile/build_layer_test.go | 101 +++++++++++++++++++++++++++++++++++++++ tile/go_spatial_layer.go | 51 ++++++++++++++++++++ 4 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 tile/build_layer.go create mode 100644 tile/build_layer_test.go create mode 100644 tile/go_spatial_layer.go diff --git a/go.mod b/go.mod index 49ff554..24ab543 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/creasty/defaults v1.8.0 github.com/go-playground/validator/v10 v10.30.3 github.com/go-spatial/geom v0.1.0 + github.com/golang/protobuf v1.5.3 github.com/iancoleman/strcase v0.3.0 github.com/muesli/reflow v0.3.0 github.com/perimeterx/marshmallow v1.1.5 @@ -29,7 +30,6 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-spatial/proj v0.3.0 // indirect github.com/go-test/deep v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/tile/build_layer.go b/tile/build_layer.go new file mode 100644 index 0000000..d021924 --- /dev/null +++ b/tile/build_layer.go @@ -0,0 +1,97 @@ +package tile + +import "fmt" + +// mvtVersion is the MVT spec version this codebase produces (matches go-spatial/geom's +// mvt.Version). +const mvtVersion = 2 + +// orderedByIndex inverts a dictionary (as built by BuildKeyDictionary/BuildValueDictionary) +// back into a slice, ordered by index. Position i in the result is dictionary index i, +// which is exactly what the Layer.Keys/Layer.Values fields require: a feature's Tags +// reference keys/values by that same position. +func orderedByIndex[K comparable](index map[K]uint32) []K { + ordered := make([]K, len(index)) + for k, i := range index { + ordered[i] = k + } + return ordered +} + +// toGSTileValue converts a Go attribute value into the MVT Value oneof, mirroring +// go-spatial/geom/encoding/mvt's vectorTileValue: the exact numeric type determines which +// oneof field is set (e.g. a Go int32 becomes a zigzag-encoded "sint" value, matching the +// original library, even though our attribute values are currently only ever string, +// int64 or float64). +func toGSTileValue(v any) *GSTileValue { + tv := new(GSTileValue) + switch t := v.(type) { + case string: + tv.StringValue = &t + case fmt.Stringer: + str := t.String() + tv.StringValue = &str + case bool: + tv.BoolValue = &t + case int8: + iv := int64(t) + tv.SintValue = &iv + case int16: + iv := int64(t) + tv.SintValue = &iv + case int32: + iv := int64(t) + tv.SintValue = &iv + case int64: + tv.IntValue = &t + case uint8: + iv := int64(t) + tv.SintValue = &iv + case uint16: + iv := int64(t) + tv.SintValue = &iv + case uint32: + iv := int64(t) + tv.SintValue = &iv + case uint64: + tv.UintValue = &t + case float32: + tv.FloatValue = &t + case float64: + tv.DoubleValue = &t + } + return tv +} + +// BuildLayer assembles a GSTileLayer for one tile: the layer name (assumed to be the +// source table name), the already-built features (see BuildFeature), and the key/value +// dictionaries used to build those features' Tags. The dictionaries are inverted back +// into the ordered Keys/Values arrays the features' Tags indices refer to. +func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint32, valueIndex map[any]uint32) *GSTileLayer { + keys := orderedByIndex(keyIndex) + + rawValues := orderedByIndex(valueIndex) + values := make([]*GSTileValue, len(rawValues)) + for i, v := range rawValues { + values[i] = toGSTileValue(v) + } + + version := uint32(mvtVersion) + extent := uint32(precision) + layerName := name + + return &GSTileLayer{ + Version: &version, + Name: &layerName, + Features: features, + Keys: keys, + Values: values, + Extent: &extent, + } +} + +// BuildTile wraps one or more layers into the final GSTile, ready to be serialized with +// (github.com/golang/protobuf/proto).Marshal. +func BuildTile(layers ...*GSTileLayer) *GSTile { + return &GSTile{Layers: layers} +} diff --git a/tile/build_layer_test.go b/tile/build_layer_test.go new file mode 100644 index 0000000..eb4c3d4 --- /dev/null +++ b/tile/build_layer_test.go @@ -0,0 +1,101 @@ +package tile_test + +import ( + "testing" + + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" + oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on + "github.com/pdok/texel/tile" +) + +// TestBuildLayerIsWireCompatibleWithVectorTile verifies that our hand-copied, +// GS-prefixed protobuf types (GSTile/GSTileLayer/GSTileValue/GSTileFeature in +// go_spatial_layer.go and go_spatial_feature.go) produce bytes on the wire that are +// indistinguishable from the real, code-generated go-spatial/geom vector_tile types. +// +// This matters because our types are not registered/generated protobuf messages: they +// only implement the minimal proto.Message interface (Reset/String/ProtoMessage) and +// rely on the same `protobuf:"..."` struct tags as the original vector_tile.pb.go for +// the classic reflection-based (github.com/golang/protobuf/proto) marshaler to encode +// them correctly. A typo in a tag (wrong field number/wire type) would silently produce +// an invalid or misinterpreted MVT tile, so this test marshals with our types and +// unmarshals with the real, generated ones to prove the two are wire-compatible. +func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { + keyIndex := tile.BuildKeyDictionary([]string{"name", "count"}) + attributes := tile.InternalAttributeTable{ + 1: {"name": "foo", "count": int64(3)}, + 2: {"name": "bar", "count": int64(3)}, // shares the "count" value with feature 1 + } + valueIndex := tile.BuildValueDictionary(attributes) + + feature1, err := tile.BuildFeature(1, tile.EncodedGeometry{Encoding: []uint32{9, 2, 2}, GeometryType: 1}, attributes[1], keyIndex, valueIndex) + if err != nil { + t.Fatalf("BuildFeature(1): %v", err) + } + feature2, err := tile.BuildFeature(2, tile.EncodedGeometry{Encoding: []uint32{9, 4, 4}, GeometryType: 1}, attributes[2], keyIndex, valueIndex) + if err != nil { + t.Fatalf("BuildFeature(2): %v", err) + } + + layer := tile.BuildLayer("mytable", []*tile.GSTileFeature{feature1, feature2}, keyIndex, valueIndex) + gsTile := tile.BuildTile(layer) + + b, err := oldproto.Marshal(gsTile) + if err != nil { + t.Fatalf("marshaling our GSTile: %v", err) + } + + var vt vectorTile.Tile + if err := oldproto.Unmarshal(b, &vt); err != nil { + t.Fatalf("unmarshaling with the real vectorTile.Tile: %v", err) + } + + if len(vt.Layers) != 1 { + t.Fatalf("expected 1 layer, got %d", len(vt.Layers)) + } + l := vt.Layers[0] + + if got := l.GetName(); got != "mytable" { + t.Errorf("layer name = %q, want %q", got, "mytable") + } + if got := l.GetVersion(); got != 2 { + t.Errorf("layer version = %d, want 2", got) + } + if got := l.GetExtent(); got != 4096 { + t.Errorf("layer extent = %d, want 4096", got) + } + if want := []string{"name", "count"}; !equalStrings(l.Keys, want) { + t.Errorf("layer keys = %v, want %v", l.Keys, want) + } + // "foo", "bar" and the shared int64(3) => 3 distinct values. + if len(l.Values) != 3 { + t.Fatalf("expected 3 distinct values, got %d: %+v", len(l.Values), l.Values) + } + if len(l.Features) != 2 { + t.Fatalf("expected 2 features, got %d", len(l.Features)) + } + + for i, feat := range l.Features { + if feat.GetType() != vectorTile.Tile_POINT { + t.Errorf("feature %d type = %v, want POINT", i, feat.GetType()) + } + if len(feat.Tags)%2 != 0 { + t.Errorf("feature %d tags = %v, expected an even number of (key,value) indices", i, feat.Tags) + } + } + if feat0Geo, feat1Geo := l.Features[0].Geometry, l.Features[1].Geometry; len(feat0Geo) == 0 || len(feat1Geo) == 0 { + t.Errorf("expected both features to keep their encoded geometry, got %v and %v", feat0Geo, feat1Geo) + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/tile/go_spatial_layer.go b/tile/go_spatial_layer.go new file mode 100644 index 0000000..3bd61f3 --- /dev/null +++ b/tile/go_spatial_layer.go @@ -0,0 +1,51 @@ +package tile + +// This file was copied from go-spatial/geom/encoding/mvt/vector_tile/vector_tile.pb.go. +// Modifications made: +// - Renamed Tile/Tile_Layer/Tile_Value to GSTile/GSTileLayer/GSTileValue, and dropped the +// GeomType/Feature declarations that already live in go_spatial_feature.go. +// - Dropped proto.XXX_InternalExtensions/ExtensionRangeArray/Descriptor: extensions and +// the file descriptor are not used by this codebase. +// - String() uses fmt instead of proto.CompactTextString, to avoid pulling in the +// proto package just for debug printing. + +import "fmt" + +// GSTileValue mirrors vector_tile.pb.go's Tile_Value. Exactly one of these fields +// should be set for a valid value. +type GSTileValue struct { + StringValue *string `protobuf:"bytes,1,opt,name=string_value" json:"string_value,omitempty"` + FloatValue *float32 `protobuf:"fixed32,2,opt,name=float_value" json:"float_value,omitempty"` + DoubleValue *float64 `protobuf:"fixed64,3,opt,name=double_value" json:"double_value,omitempty"` + IntValue *int64 `protobuf:"varint,4,opt,name=int_value" json:"int_value,omitempty"` + UintValue *uint64 `protobuf:"varint,5,opt,name=uint_value" json:"uint_value,omitempty"` + SintValue *int64 `protobuf:"zigzag64,6,opt,name=sint_value" json:"sint_value,omitempty"` + BoolValue *bool `protobuf:"varint,7,opt,name=bool_value" json:"bool_value,omitempty"` +} + +func (m *GSTileValue) Reset() { *m = GSTileValue{} } +func (m *GSTileValue) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTileValue) ProtoMessage() {} + +// GSTileLayer mirrors vector_tile.pb.go's Tile_Layer. +type GSTileLayer struct { + Version *uint32 `protobuf:"varint,15,req,name=version,def=1" json:"version,omitempty"` + Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` + Features []*GSTileFeature `protobuf:"bytes,2,rep,name=features" json:"features,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=keys" json:"keys,omitempty"` + Values []*GSTileValue `protobuf:"bytes,4,rep,name=values" json:"values,omitempty"` + Extent *uint32 `protobuf:"varint,5,opt,name=extent,def=4096" json:"extent,omitempty"` +} + +func (m *GSTileLayer) Reset() { *m = GSTileLayer{} } +func (m *GSTileLayer) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTileLayer) ProtoMessage() {} + +// GSTile mirrors vector_tile.pb.go's Tile. +type GSTile struct { + Layers []*GSTileLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` +} + +func (m *GSTile) Reset() { *m = GSTile{} } +func (m *GSTile) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTile) ProtoMessage() {} From 53bcb657d1e7a65e6d4c60f3db635818fed8064a Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 15:11:48 +0200 Subject: [PATCH 10/25] feat: wire functionality to encode a single table --- processing/gpkg/mvttile.go | 123 +++++++++++++++++++++++++++++++++++++ tile/marshal.go | 9 +++ 2 files changed, 132 insertions(+) create mode 100644 processing/gpkg/mvttile.go create mode 100644 tile/marshal.go diff --git a/processing/gpkg/mvttile.go b/processing/gpkg/mvttile.go new file mode 100644 index 0000000..05d99f7 --- /dev/null +++ b/processing/gpkg/mvttile.go @@ -0,0 +1,123 @@ +package gpkg + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + + "github.com/pdok/texel/tile" +) + +// TileCoord identifies a tile by its column/row in the tile matrix. +type TileCoord struct { + X, Y uint +} + +// selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at +// least one encoded feature. +func (t Table) selectDistinctTilesSQL() string { + return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y` +} + +// ListTiles returns every distinct tile present in the "
_encoded" table. +func (source SourceGeopackage) ListTiles() ([]TileCoord, error) { + rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL()) + if err != nil { + return nil, fmt.Errorf("listing tiles: %w", err) + } + defer rows.Close() + + var tiles []TileCoord + for rows.Next() { + var x, y int64 + if err := rows.Scan(&x, &y); err != nil { + return nil, fmt.Errorf("scanning tile row: %w", err) + } + tiles = append(tiles, TileCoord{X: uint(x), Y: uint(y)}) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating tile rows: %w", err) + } + return tiles, nil +} + +// BuildMVTTile builds and serializes the MVT tile for (tileX, tileY): it fetches the +// tile's encoded features and their attributes, builds the per-tile value dictionary, +// assembles a single layer named layerName using the given (already built once, +// fixed-schema) key dictionary, and marshals the result to MVT protobuf bytes. A tile +// with no encoded features still yields a valid, empty layer rather than an error. +func (source SourceGeopackage) BuildMVTTile(layerName string, keyIndex map[string]uint32, tileX, tileY uint) ([]byte, error) { + rows, err := source.GetFeaturesForTile(tileX, tileY) + if err != nil { + return nil, fmt.Errorf("getting features for tile (%d,%d): %w", tileX, tileY, err) + } + + featureIDs := make([]int64, len(rows)) + for i, row := range rows { + featureIDs[i] = row.FeatureID + } + + attributes, err := source.GetAttributesForFeatures(featureIDs) + if err != nil { + return nil, fmt.Errorf("getting attributes for tile (%d,%d): %w", tileX, tileY, err) + } + valueIndex := tile.BuildValueDictionary(attributes) + + features := make([]*tile.GSTileFeature, 0, len(rows)) + for _, row := range rows { + attrs := AttributesForFeature(attributes, row.FeatureID) + feature, err := tile.BuildFeature(row.FeatureID, row.Geom, attrs, keyIndex, valueIndex) + if err != nil { + return nil, fmt.Errorf("building tile (%d,%d): %w", tileX, tileY, err) + } + features = append(features, feature) + } + + layer := tile.BuildLayer(layerName, features, keyIndex, valueIndex) + + data, err := tile.MarshalTile(tile.BuildTile(layer)) + if err != nil { + return nil, fmt.Errorf("marshaling tile (%d,%d): %w", tileX, tileY, err) + } + return data, nil +} + +// WriteTileFile writes one tile's serialized bytes to //.mvt, +// creating directories as needed. +func WriteTileFile(baseDir string, tileX, tileY uint, data []byte) error { + dir := filepath.Join(baseDir, strconv.FormatUint(uint64(tileX), 10)) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating tile directory %s: %w", dir, err) + } + + path := filepath.Join(dir, strconv.FormatUint(uint64(tileY), 10)+".mvt") + if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions + return fmt.Errorf("writing tile file %s: %w", path, err) + } + return nil +} + +// ProcessTilesToMVT enumerates every tile with encoded features for source.Table, +// builds each tile's MVT bytes (see BuildMVTTile) and writes it to outDir via +// WriteTileFile. The key dictionary is built once, up front, since the attribute +// schema is fixed for the whole table (see tile.BuildKeyDictionary). +func (source SourceGeopackage) ProcessTilesToMVT(outDir string) error { + keyIndex := tile.BuildKeyDictionary(source.Table.AttributeColumnNames()) + + tiles, err := source.ListTiles() + if err != nil { + return err + } + + for _, tc := range tiles { + data, err := source.BuildMVTTile(source.Table.Name, keyIndex, tc.X, tc.Y) + if err != nil { + return fmt.Errorf("processing tile (%d,%d): %w", tc.X, tc.Y, err) + } + if err := WriteTileFile(outDir, tc.X, tc.Y, data); err != nil { + return fmt.Errorf("processing tile (%d,%d): %w", tc.X, tc.Y, err) + } + } + return nil +} diff --git a/tile/marshal.go b/tile/marshal.go new file mode 100644 index 0000000..3d4a0ee --- /dev/null +++ b/tile/marshal.go @@ -0,0 +1,9 @@ +package tile + +import oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on + +// MarshalTile serializes a GSTile into MVT protobuf bytes. Kept in this package so +// callers never need to depend on the protobuf library directly. +func MarshalTile(t *GSTile) ([]byte, error) { + return oldproto.Marshal(t) +} From 4ddebe26367713550a277ff7a64dcff0aa39a511 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 20 Jul 2026 16:27:32 +0200 Subject: [PATCH 11/25] feat: add 'texel mvt' command for tiling --- main.go | 332 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 195 insertions(+), 137 deletions(-) diff --git a/main.go b/main.go index a7dd19e..7ec8bfe 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,9 @@ const ( IGNOREOUTSIDEGRID string = `ignoreoutsidegrid` REVERSEWINDINGORDER string = `reversewindingorder` ENCODETILES string = `encodetiles` + + MVTSOURCE string = `mvtSourceGpkg` + MVTOUTDIR string = `mvtOutDir` ) //nolint:funlen @@ -45,152 +48,184 @@ func main() { app.Usage = "A Golang Polygon Snapping application" app.Version = versioninfo.Short() - app.Flags = []cli.Flag{ - &cli.StringFlag{ - Name: SOURCE, - Aliases: []string{"s"}, - Usage: "Source GPKG", - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(SOURCE)}, - }, - &cli.StringFlag{ - Name: TARGET, - Aliases: []string{"t"}, - Usage: "Target GPKG (prefix). One GPKG per tile matrix cq zoom level will be created and the filename will be suffixed. E.g. target_6.gpkg", - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TARGET)}, - }, - &cli.BoolFlag{ - Name: OVERWRITE, - Aliases: []string{"o"}, - Usage: "Overwrite a target GPKG if it exists", - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(OVERWRITE)}, - }, - &cli.StringFlag{ - Name: TILEMATRIXSET, - Aliases: []string{"tms"}, - Usage: `ID of a (built-in) tile matrix set. E.g.: NetherlandsRDNewQuad`, - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TILEMATRIXSET)}, - }, - &cli.StringFlag{ - Name: TILEMATRICES, - Aliases: []string{"z"}, - Usage: `IDs (usually the same as the zoom levels) of the tile matrices in the tile matrix set that should be processed for. JSON array of integers. E.g.: [4,5,6,7,8]`, - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TILEMATRICES)}, - }, - &cli.IntFlag{ - Name: PAGESIZE, - Aliases: []string{"p"}, - Usage: "Page Size, how many features are written per transaction to a target GPKG", - Value: 1000, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(PAGESIZE)}, - }, - &cli.BoolFlag{ - Name: KEEPPOINTSANDLINES, - Aliases: []string{"pl"}, - Usage: "Parts of polygons are reduced to points and lines after texel, keep these details or not.", - // TODO: This results in broken tiles, we should change this to a collapse option (optionally collapse rings from polygons to lines and points; grow lines as long as possible and add an option for line-length-to-keep) - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(KEEPPOINTSANDLINES)}, - }, - &cli.BoolFlag{ - Name: IGNOREOUTSIDEGRID, - Aliases: []string{"iog"}, - Usage: "Ignore polygons that fall (partly) outside the grid, instead of panicking", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(IGNOREOUTSIDEGRID)}, - }, - &cli.BoolFlag{ - Name: REVERSEWINDINGORDER, - Aliases: []string{"rwo"}, - Usage: "Reverse the winding order of rings in polygons, contrary to the OGC simple features spec", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(REVERSEWINDINGORDER)}, - }, - &cli.BoolFlag{ - Name: ENCODETILES, - Aliases: []string{"enc"}, - Usage: "Add tables with mapbox-encoded geometries.", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(ENCODETILES)}, - }, - } + // bare invocation (no subcommand) is routed to "snap", so its Required flags + // are still enforced as before. + app.DefaultCommand = "snap" - app.Action = func(c *cli.Context) error { - tileMatrixSet, err := tms20.LoadEmbeddedTileMatrixSet(c.String(TILEMATRIXSET)) - if err != nil { - return err - } - var tileMatrixIDs []int - err = json.Unmarshal([]byte(c.String(TILEMATRICES)), &tileMatrixIDs) - if err != nil { - return err - } - if err = validateTileMatrixSet(tileMatrixSet, tileMatrixIDs); err != nil { - return err - } + app.Commands = []*cli.Command{ + { + Name: "snap", + Usage: "Snap polygons in a source GeoPackage to a tile grid and write the result to a target GeoPackage", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: SOURCE, + Aliases: []string{"s"}, + Usage: "Source GPKG", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(SOURCE)}, + }, + &cli.StringFlag{ + Name: TARGET, + Aliases: []string{"t"}, + Usage: "Target GPKG (prefix). One GPKG per tile matrix cq zoom level will be created and the filename will be suffixed. E.g. target_6.gpkg", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TARGET)}, + }, + &cli.BoolFlag{ + Name: OVERWRITE, + Aliases: []string{"o"}, + Usage: "Overwrite a target GPKG if it exists", + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(OVERWRITE)}, + }, + &cli.StringFlag{ + Name: TILEMATRIXSET, + Aliases: []string{"tms"}, + Usage: `ID of a (built-in) tile matrix set. E.g.: NetherlandsRDNewQuad`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TILEMATRIXSET)}, + }, + &cli.StringFlag{ + Name: TILEMATRICES, + Aliases: []string{"z"}, + Usage: `IDs (usually the same as the zoom levels) of the tile matrices in the tile matrix set that should be processed for. JSON array of integers. E.g.: [4,5,6,7,8]`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TILEMATRICES)}, + }, + &cli.IntFlag{ + Name: PAGESIZE, + Aliases: []string{"p"}, + Usage: "Page Size, how many features are written per transaction to a target GPKG", + Value: 1000, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(PAGESIZE)}, + }, + &cli.BoolFlag{ + Name: KEEPPOINTSANDLINES, + Aliases: []string{"pl"}, + Usage: "Parts of polygons are reduced to points and lines after texel, keep these details or not.", + // TODO: This results in broken tiles, we should change this to a collapse option (optionally collapse rings from polygons to lines and points; grow lines as long as possible and add an option for line-length-to-keep) + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(KEEPPOINTSANDLINES)}, + }, + &cli.BoolFlag{ + Name: IGNOREOUTSIDEGRID, + Aliases: []string{"iog"}, + Usage: "Ignore polygons that fall (partly) outside the grid, instead of panicking", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(IGNOREOUTSIDEGRID)}, + }, + &cli.BoolFlag{ + Name: REVERSEWINDINGORDER, + Aliases: []string{"rwo"}, + Usage: "Reverse the winding order of rings in polygons, contrary to the OGC simple features spec", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(REVERSEWINDINGORDER)}, + }, + &cli.BoolFlag{ + Name: ENCODETILES, + Aliases: []string{"enc"}, + Usage: "Add tables with mapbox-encoded geometries.", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(ENCODETILES)}, + }, + }, + Action: func(c *cli.Context) error { + tileMatrixSet, err := tms20.LoadEmbeddedTileMatrixSet(c.String(TILEMATRIXSET)) + if err != nil { + return err + } + var tileMatrixIDs []int + err = json.Unmarshal([]byte(c.String(TILEMATRICES)), &tileMatrixIDs) + if err != nil { + return err + } + if err = validateTileMatrixSet(tileMatrixSet, tileMatrixIDs); err != nil { + return err + } - _, err = os.Stat(c.String(SOURCE)) - if os.IsNotExist(err) { - log.Fatalf("error opening source GeoPackage: %s", err) - } + _, err = os.Stat(c.String(SOURCE)) + if os.IsNotExist(err) { + log.Fatalf("error opening source GeoPackage: %s", err) + } - source := gpkg.SourceGeopackage{} - source.Init(c.String(SOURCE)) - defer source.Close() + source := gpkg.SourceGeopackage{} + source.Init(c.String(SOURCE)) + defer source.Close() - targetPathFmt := injectSuffixIntoPath(c.String(TARGET)) + targetPathFmt := injectSuffixIntoPath(c.String(TARGET)) - gpkgTargets := make(map[int]*gpkg.TargetGeopackage, len(tileMatrixIDs)) - overwrite := c.Bool(OVERWRITE) - pagesize := c.Int(PAGESIZE) // TODO divide by tile matrices count - snapConfig := snap.Config{ - KeepPointsAndLines: c.Bool(KEEPPOINTSANDLINES), - IgnoreOutsideGrid: c.Bool(IGNOREOUTSIDEGRID), - ReverseWindingOrder: c.Bool(REVERSEWINDINGORDER), - EncodeTiles: c.Bool(ENCODETILES), - } - for _, tmID := range tileMatrixIDs { - gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize) - defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func - } + gpkgTargets := make(map[int]*gpkg.TargetGeopackage, len(tileMatrixIDs)) + overwrite := c.Bool(OVERWRITE) + pagesize := c.Int(PAGESIZE) // TODO divide by tile matrices count + snapConfig := snap.Config{ + KeepPointsAndLines: c.Bool(KEEPPOINTSANDLINES), + IgnoreOutsideGrid: c.Bool(IGNOREOUTSIDEGRID), + ReverseWindingOrder: c.Bool(REVERSEWINDINGORDER), + EncodeTiles: c.Bool(ENCODETILES), + } + for _, tmID := range tileMatrixIDs { + gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize) + defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func + } - tables := source.GetTableInfo() - for _, target := range gpkgTargets { - err = target.CreateTables(tables) - if err != nil { - log.Fatalf("error initialization the target GeoPackage: %s", err) - } - } + tables := source.GetTableInfo() + for _, target := range gpkgTargets { + err = target.CreateTables(tables) + if err != nil { + log.Fatalf("error initialization the target GeoPackage: %s", err) + } + } - log.Println("=== start snapping ===") + log.Println("=== start snapping ===") - // need a copied map because of type difference processing.Target vs gpkg.TargetGeopackage - targets := make(map[int]processing.Target, len(gpkgTargets)) - for tmID, target := range gpkgTargets { - targets[tmID] = target - } - // Process the tables sequentially - for _, table := range tables { - log.Printf(" snapping %s", table.Name) - for _, target := range gpkgTargets { - source.Table = table - target.Table = table - } - processBySnapping(source, targets, tileMatrixSet, snapConfig) - log.Printf(" finished %s", table.Name) - } + // need a copied map because of type difference processing.Target vs gpkg.TargetGeopackage + targets := make(map[int]processing.Target, len(gpkgTargets)) + for tmID, target := range gpkgTargets { + targets[tmID] = target + } + // Process the tables sequentially + for _, table := range tables { + log.Printf(" snapping %s", table.Name) + for _, target := range gpkgTargets { + source.Table = table + target.Table = table + } + processBySnapping(source, targets, tileMatrixSet, snapConfig) + log.Printf(" finished %s", table.Name) + } - log.Println("=== done snapping ===") - return nil + log.Println("=== done snapping ===") + return nil + }, + }, + { + Name: "mvt", + Usage: "Build MVT tiles from an already-encoded target GeoPackage (see --" + ENCODETILES + ")", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: MVTSOURCE, + Aliases: []string{"s"}, + Usage: `GeoPackage containing the encoded ("
_encoded") and attribute tables`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(MVTSOURCE)}, + }, + &cli.StringFlag{ + Name: MVTOUTDIR, + Aliases: []string{"o"}, + Usage: "Directory to write the generated /.mvt files to", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(MVTOUTDIR)}, + }, + }, + Action: func(c *cli.Context) error { + return runBuildMVTTiles(c.String(MVTSOURCE), c.String(MVTOUTDIR)) + }, + }, } err := app.Run(os.Args) @@ -239,3 +274,26 @@ func processBySnapping(source processing.Source, targets map[tms20.TMID]processi return snap.SnapPolygon(p, tileMatrixSet, tmIDs, snapConfig) }) } + +// runBuildMVTTiles builds and writes MVT tiles for every table in the given source +// GeoPackage (a GeoPackage previously produced with --encodetiles), using +// gpkg.SourceGeopackage.ProcessTilesToMVT for each table. +func runBuildMVTTiles(sourcePath, outDir string) error { + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath) + } + + source := gpkg.SourceGeopackage{} + source.Init(sourcePath) + defer source.Close() + + tables := source.GetTableInfo() + for _, table := range tables { + source.Table = table + log.Printf(" building MVT tiles for %s", table.Name) + if err := source.ProcessTilesToMVT(outDir); err != nil { + return fmt.Errorf("building MVT tiles for %s: %w", table.Name, err) + } + } + return nil +} From 7ddb79a8fdfd76c8d9e1d9e6e2b6f1326bd943bb Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 21 Jul 2026 10:48:55 +0200 Subject: [PATCH 12/25] refactor: isolate go-spatial types --- processing/gpkg/mvttile.go | 2 +- tile/build_layer.go | 27 ++++------ tile/build_layer_test.go | 26 +++++----- tile/go_spatial_feature.go | 56 +++------------------ tile/go_spatial_layer.go | 51 ------------------- tile/go_spatial_types.go | 100 +++++++++++++++++++++++++++++++++++++ 6 files changed, 131 insertions(+), 131 deletions(-) delete mode 100644 tile/go_spatial_layer.go create mode 100644 tile/go_spatial_types.go diff --git a/processing/gpkg/mvttile.go b/processing/gpkg/mvttile.go index 05d99f7..7c8b7cf 100644 --- a/processing/gpkg/mvttile.go +++ b/processing/gpkg/mvttile.go @@ -34,7 +34,7 @@ func (source SourceGeopackage) ListTiles() ([]TileCoord, error) { if err := rows.Scan(&x, &y); err != nil { return nil, fmt.Errorf("scanning tile row: %w", err) } - tiles = append(tiles, TileCoord{X: uint(x), Y: uint(y)}) + tiles = append(tiles, TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterating tile rows: %w", err) diff --git a/tile/build_layer.go b/tile/build_layer.go index d021924..ae38552 100644 --- a/tile/build_layer.go +++ b/tile/build_layer.go @@ -2,14 +2,11 @@ package tile import "fmt" -// mvtVersion is the MVT spec version this codebase produces (matches go-spatial/geom's -// mvt.Version). const mvtVersion = 2 -// orderedByIndex inverts a dictionary (as built by BuildKeyDictionary/BuildValueDictionary) -// back into a slice, ordered by index. Position i in the result is dictionary index i, -// which is exactly what the Layer.Keys/Layer.Values fields require: a feature's Tags -// reference keys/values by that same position. +// Convert tag map to slice. Assumes all indices 0, 1, ..., len(index)-1 are +// exactly used once. Necessary since iteration over maps is in arbitrary order. +// This is a generic function since we need it with K = string and K = any. func orderedByIndex[K comparable](index map[K]uint32) []K { ordered := make([]K, len(index)) for k, i := range index { @@ -18,11 +15,8 @@ func orderedByIndex[K comparable](index map[K]uint32) []K { return ordered } -// toGSTileValue converts a Go attribute value into the MVT Value oneof, mirroring -// go-spatial/geom/encoding/mvt's vectorTileValue: the exact numeric type determines which -// oneof field is set (e.g. a Go int32 becomes a zigzag-encoded "sint" value, matching the -// original library, even though our attribute values are currently only ever string, -// int64 or float64). +// Convert value to GSTileValue. We do not support XXX_unrecognized and panic +// if an unknown type is encountered. This should never happen. func toGSTileValue(v any) *GSTileValue { tv := new(GSTileValue) switch t := v.(type) { @@ -59,14 +53,14 @@ func toGSTileValue(v any) *GSTileValue { tv.FloatValue = &t case float64: tv.DoubleValue = &t + default: + // We should never encounter this. + panic("Unknown type for GSTypeValue.") } return tv } -// BuildLayer assembles a GSTileLayer for one tile: the layer name (assumed to be the -// source table name), the already-built features (see BuildFeature), and the key/value -// dictionaries used to build those features' Tags. The dictionaries are inverted back -// into the ordered Keys/Values arrays the features' Tags indices refer to. +// Assemble layer struct func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint32, valueIndex map[any]uint32) *GSTileLayer { keys := orderedByIndex(keyIndex) @@ -90,8 +84,7 @@ func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint } } -// BuildTile wraps one or more layers into the final GSTile, ready to be serialized with -// (github.com/golang/protobuf/proto).Marshal. +// Assemble layers into GSTile. This struct can be marshalled by protobuf. func BuildTile(layers ...*GSTileLayer) *GSTile { return &GSTile{Layers: layers} } diff --git a/tile/build_layer_test.go b/tile/build_layer_test.go index eb4c3d4..0c8e8bc 100644 --- a/tile/build_layer_test.go +++ b/tile/build_layer_test.go @@ -50,10 +50,10 @@ func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { t.Fatalf("unmarshaling with the real vectorTile.Tile: %v", err) } - if len(vt.Layers) != 1 { - t.Fatalf("expected 1 layer, got %d", len(vt.Layers)) + if len(vt.GetLayers()) != 1 { + t.Fatalf("expected 1 layer, got %d", len(vt.GetLayers())) } - l := vt.Layers[0] + l := vt.GetLayers()[0] if got := l.GetName(); got != "mytable" { t.Errorf("layer name = %q, want %q", got, "mytable") @@ -64,26 +64,26 @@ func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { if got := l.GetExtent(); got != 4096 { t.Errorf("layer extent = %d, want 4096", got) } - if want := []string{"name", "count"}; !equalStrings(l.Keys, want) { - t.Errorf("layer keys = %v, want %v", l.Keys, want) + if want := []string{"name", "count"}; !equalStrings(l.GetKeys(), want) { + t.Errorf("layer keys = %v, want %v", l.GetKeys(), want) } // "foo", "bar" and the shared int64(3) => 3 distinct values. - if len(l.Values) != 3 { - t.Fatalf("expected 3 distinct values, got %d: %+v", len(l.Values), l.Values) + if len(l.GetValues()) != 3 { + t.Fatalf("expected 3 distinct values, got %d: %+v", len(l.GetValues()), l.GetValues()) } - if len(l.Features) != 2 { - t.Fatalf("expected 2 features, got %d", len(l.Features)) + if len(l.GetFeatures()) != 2 { + t.Fatalf("expected 2 features, got %d", len(l.GetFeatures())) } - for i, feat := range l.Features { + for i, feat := range l.GetFeatures() { if feat.GetType() != vectorTile.Tile_POINT { t.Errorf("feature %d type = %v, want POINT", i, feat.GetType()) } - if len(feat.Tags)%2 != 0 { - t.Errorf("feature %d tags = %v, expected an even number of (key,value) indices", i, feat.Tags) + if len(feat.GetTags())%2 != 0 { + t.Errorf("feature %d tags = %v, expected an even number of (key,value) indices", i, feat.GetTags()) } } - if feat0Geo, feat1Geo := l.Features[0].Geometry, l.Features[1].Geometry; len(feat0Geo) == 0 || len(feat1Geo) == 0 { + if feat0Geo, feat1Geo := l.GetFeatures()[0].GetGeometry(), l.GetFeatures()[1].GetGeometry(); len(feat0Geo) == 0 || len(feat1Geo) == 0 { t.Errorf("expected both features to keep their encoded geometry, got %v and %v", feat0Geo, feat1Geo) } } diff --git a/tile/go_spatial_feature.go b/tile/go_spatial_feature.go index 7ce21c2..37ffbf2 100644 --- a/tile/go_spatial_feature.go +++ b/tile/go_spatial_feature.go @@ -19,55 +19,6 @@ import ( "github.com/go-spatial/geom/winding" ) -// START copied from vector_tile.pb.go - -type GSTileGeomType int32 - -const ( - TileUNKNOWN GSTileGeomType = 0 - TilePOINT GSTileGeomType = 1 - TileLINESTRING GSTileGeomType = 2 - TilePOLYGON GSTileGeomType = 3 -) - -var TileGeomTypeName = map[int32]string{ - 0: "UNKNOWN", - 1: "POINT", - 2: "LINESTRING", - 3: "POLYGON", -} - -var TileGeomTypeValue = map[string]int32{ - "UNKNOWN": 0, - "POINT": 1, - "LINESTRING": 2, - "POLYGON": 3, -} - -var ( - ErrNilFeature = errors.New("feature is nil") - ErrUnknownGeometryType = errors.New("unknown geometry type") - ErrNilGeometryType = errors.New("geometry is nil") -) - -type GSTileFeature struct { - Id *uint64 `protobuf:"varint,1,opt,name=id,def=0" json:"id,omitempty"` - // Tags of this feature are encoded as repeated pairs of - // integers. - // A detailed description of tags is located in sections - // 4.2 and 4.4 of the specification - Tags []uint32 `protobuf:"varint,2,rep,packed,name=tags" json:"tags,omitempty"` - // The type of geometry stored in this feature. - Type *GSTileGeomType `protobuf:"varint,3,opt,name=type,enum=vector_tile.Tile_GeomType,def=0" json:"type,omitempty"` - // Contains a stream of commands and parameters (vertices). - // A detailed description on geometry encoding is located in - // section 4.3 of the specification. - Geometry []uint32 `protobuf:"varint,4,rep,packed,name=geometry" json:"geometry,omitempty"` - XXXunrecognized []byte `json:"-"` -} - -// END copied from vector_tile.pb.go - // Definition needed for compilation const debug = false @@ -81,6 +32,13 @@ const debug = false // Feature describes a feature of a Layer. A layer will contain multiple features // each of which has a geometry describing the interesting thing, and the metadata // associated with it. + +var ( + ErrNilFeature = errors.New("feature is nil") + ErrUnknownGeometryType = errors.New("unknown geometry type") + ErrNilGeometryType = errors.New("geometry is nil") +) + type Feature struct { //nolint:recvcheck ID *uint64 Tags map[string]any diff --git a/tile/go_spatial_layer.go b/tile/go_spatial_layer.go deleted file mode 100644 index 3bd61f3..0000000 --- a/tile/go_spatial_layer.go +++ /dev/null @@ -1,51 +0,0 @@ -package tile - -// This file was copied from go-spatial/geom/encoding/mvt/vector_tile/vector_tile.pb.go. -// Modifications made: -// - Renamed Tile/Tile_Layer/Tile_Value to GSTile/GSTileLayer/GSTileValue, and dropped the -// GeomType/Feature declarations that already live in go_spatial_feature.go. -// - Dropped proto.XXX_InternalExtensions/ExtensionRangeArray/Descriptor: extensions and -// the file descriptor are not used by this codebase. -// - String() uses fmt instead of proto.CompactTextString, to avoid pulling in the -// proto package just for debug printing. - -import "fmt" - -// GSTileValue mirrors vector_tile.pb.go's Tile_Value. Exactly one of these fields -// should be set for a valid value. -type GSTileValue struct { - StringValue *string `protobuf:"bytes,1,opt,name=string_value" json:"string_value,omitempty"` - FloatValue *float32 `protobuf:"fixed32,2,opt,name=float_value" json:"float_value,omitempty"` - DoubleValue *float64 `protobuf:"fixed64,3,opt,name=double_value" json:"double_value,omitempty"` - IntValue *int64 `protobuf:"varint,4,opt,name=int_value" json:"int_value,omitempty"` - UintValue *uint64 `protobuf:"varint,5,opt,name=uint_value" json:"uint_value,omitempty"` - SintValue *int64 `protobuf:"zigzag64,6,opt,name=sint_value" json:"sint_value,omitempty"` - BoolValue *bool `protobuf:"varint,7,opt,name=bool_value" json:"bool_value,omitempty"` -} - -func (m *GSTileValue) Reset() { *m = GSTileValue{} } -func (m *GSTileValue) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTileValue) ProtoMessage() {} - -// GSTileLayer mirrors vector_tile.pb.go's Tile_Layer. -type GSTileLayer struct { - Version *uint32 `protobuf:"varint,15,req,name=version,def=1" json:"version,omitempty"` - Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` - Features []*GSTileFeature `protobuf:"bytes,2,rep,name=features" json:"features,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=keys" json:"keys,omitempty"` - Values []*GSTileValue `protobuf:"bytes,4,rep,name=values" json:"values,omitempty"` - Extent *uint32 `protobuf:"varint,5,opt,name=extent,def=4096" json:"extent,omitempty"` -} - -func (m *GSTileLayer) Reset() { *m = GSTileLayer{} } -func (m *GSTileLayer) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTileLayer) ProtoMessage() {} - -// GSTile mirrors vector_tile.pb.go's Tile. -type GSTile struct { - Layers []*GSTileLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` -} - -func (m *GSTile) Reset() { *m = GSTile{} } -func (m *GSTile) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTile) ProtoMessage() {} diff --git a/tile/go_spatial_types.go b/tile/go_spatial_types.go new file mode 100644 index 0000000..ed5b483 --- /dev/null +++ b/tile/go_spatial_types.go @@ -0,0 +1,100 @@ +package tile + +// These are the types obtained from go-spatial/encoding/mvt/vectortile needed +// for encoding. Modifications made: +// - Renamed types for linter purposes. +// - Dropped some protobuf functionality not used. +// - Use fmt for debug printing rather than proto. + +import ( + "fmt" +) + +// Geometry types + +type GSTileGeomType int32 + +const ( + TileUNKNOWN GSTileGeomType = 0 + TilePOINT GSTileGeomType = 1 + TileLINESTRING GSTileGeomType = 2 + TilePOLYGON GSTileGeomType = 3 +) + +var TileGeomTypeName = map[int32]string{ + 0: "UNKNOWN", + 1: "POINT", + 2: "LINESTRING", + 3: "POLYGON", +} + +var TileGeomTypeValue = map[string]int32{ + "UNKNOWN": 0, + "POINT": 1, + "LINESTRING": 2, + "POLYGON": 3, +} + +// GSTileFeature mirrors vector_tile.pb.go's Tile_Feature. + +type GSTileFeature struct { + Id *uint64 `protobuf:"varint,1,opt,name=id,def=0" json:"id,omitempty"` + // Tags of this feature are encoded as repeated pairs of + // integers. + // A detailed description of tags is located in sections + // 4.2 and 4.4 of the specification + Tags []uint32 `protobuf:"varint,2,rep,packed,name=tags" json:"tags,omitempty"` + // The type of geometry stored in this feature. + Type *GSTileGeomType `protobuf:"varint,3,opt,name=type,enum=vector_tile.Tile_GeomType,def=0" json:"type,omitempty"` + // Contains a stream of commands and parameters (vertices). + // A detailed description on geometry encoding is located in + // section 4.3 of the specification. + Geometry []uint32 `protobuf:"varint,4,rep,packed,name=geometry" json:"geometry,omitempty"` + XXXunrecognized []byte `json:"-"` +} + +func (m *GSTileFeature) Reset() { *m = GSTileFeature{} } +func (m *GSTileFeature) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTileFeature) ProtoMessage() {} + +// GSTileValue mirrors vector_tile.pb.go's Tile_Value. +// Exactly one of these fields should be set for a valid value. + +type GSTileValue struct { + StringValue *string `protobuf:"bytes,1,opt,name=string_value" json:"stringValue,omitempty"` + FloatValue *float32 `protobuf:"fixed32,2,opt,name=float_value" json:"floatValue,omitempty"` + DoubleValue *float64 `protobuf:"fixed64,3,opt,name=double_value" json:"doubleValue,omitempty"` + IntValue *int64 `protobuf:"varint,4,opt,name=int_value" json:"intValue,omitempty"` + UintValue *uint64 `protobuf:"varint,5,opt,name=uint_value" json:"uintValue,omitempty"` + SintValue *int64 `protobuf:"zigzag64,6,opt,name=sint_value" json:"sintValue,omitempty"` + BoolValue *bool `protobuf:"varint,7,opt,name=bool_value" json:"boolValue,omitempty"` +} + +func (m *GSTileValue) Reset() { *m = GSTileValue{} } +func (m *GSTileValue) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTileValue) ProtoMessage() {} + +// GSTileLayer mirrors vector_tile.pb.go's Tile_Layer. + +type GSTileLayer struct { + Version *uint32 `protobuf:"varint,15,req,name=version,def=1" json:"version,omitempty"` + Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` + Features []*GSTileFeature `protobuf:"bytes,2,rep,name=features" json:"features,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=keys" json:"keys,omitempty"` + Values []*GSTileValue `protobuf:"bytes,4,rep,name=values" json:"values,omitempty"` + Extent *uint32 `protobuf:"varint,5,opt,name=extent,def=4096" json:"extent,omitempty"` +} + +func (m *GSTileLayer) Reset() { *m = GSTileLayer{} } +func (m *GSTileLayer) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTileLayer) ProtoMessage() {} + +// GSTile mirrors vector_tile.pb.go's Tile. + +type GSTile struct { + Layers []*GSTileLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` +} + +func (m *GSTile) Reset() { *m = GSTile{} } +func (m *GSTile) String() string { return fmt.Sprintf("%+v", *m) } +func (*GSTile) ProtoMessage() {} From 2a73ee4b43de3ef335670df9ae651d5b7206b05a Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 21 Jul 2026 11:02:11 +0200 Subject: [PATCH 13/25] feat: add overflow guard --- tile/keysvalues.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tile/keysvalues.go b/tile/keysvalues.go index 5346dd7..7449ae6 100644 --- a/tile/keysvalues.go +++ b/tile/keysvalues.go @@ -31,7 +31,11 @@ func BuildValueDictionary(attributes InternalAttributeTable) map[any]uint32 { if _, ok := index[v]; ok { continue } - index[v] = uint32(len(index)) //nolint:gosec // G115 a tile cannot realistically hold more than 2^32 distinct values + l := len(index) + if l > int(^uint32(0)) { + panic("Number of values does not fit in uint32") + } + index[v] = uint32(l) } } return index From b5110c4842e119f8f4526c0d488eac09fec66607 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 21 Jul 2026 11:12:07 +0200 Subject: [PATCH 14/25] refactor: removed unused code copied from go-spatial --- tile/go_spatial_feature.go | 420 +------------------------------------ 1 file changed, 6 insertions(+), 414 deletions(-) diff --git a/tile/go_spatial_feature.go b/tile/go_spatial_feature.go index 37ffbf2..f19324c 100644 --- a/tile/go_spatial_feature.go +++ b/tile/go_spatial_feature.go @@ -1,18 +1,16 @@ package tile -// This file was copied from go-spatial/geom/encoding/mvt/feature.go -// Modifications made: -// - Made EncodeGeometry an exported function. -// - Made file self-contained by adding type definitions from -// go-spatial/geom/encoding/mvt/vector_tile/vector_tile.pb.go -// - Made file self-contained by defining the debug variable -// - Removed unused parameter "context" from all functions. +// This is the geometry encoding logic copied from go-spatial/geom/encoding/mvt +// It was necessary to take this step, since usually EncodeGeometry is not an +// exported function. Modifications made: +// - Added `const debug = false` which is usually present in another file. +// - Addressed linter issues +// - Removed `context` parameter. import ( "errors" "fmt" "log" - "slices" "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/wkt" @@ -39,71 +37,6 @@ var ( ErrNilGeometryType = errors.New("geometry is nil") ) -type Feature struct { //nolint:recvcheck - ID *uint64 - Tags map[string]any - Geometry geom.Geometry -} - -func (f Feature) String() string { - g, err := wkt.EncodeString(f.Geometry) - if err != nil { - return fmt.Sprintf("encoding error for geom geom, %v", err) - } - - if f.ID != nil { - return fmt.Sprintf("{Feature: %v, GEO: %v, Tags: %+v}", *f.ID, g, f.Tags) - } - - return fmt.Sprintf("{Feature: GEO: %v, Tags: %+v}", g, f.Tags) -} - -// NewFeatures returns one or more features for the given Geometry -func NewFeatures(geo geom.Geometry, tags map[string]any) (f []Feature) { - if geo == nil { - return f // return empty feature set for a nil geometry - } - - if g, ok := geo.(geom.Collection); ok { - geos := g.Geometries() - for i := range geos { - f = append(f, NewFeatures(geos[i], tags)...) - } - return f - } - - f = append(f, Feature{ - Tags: tags, - Geometry: geo, - }) - - return f -} - -// VTileFeature will return a vectorTile.Feature that would represent the Feature -func (f *Feature) VTileFeature(keys []string, vals []any) (tf *GSTileFeature, err error) { - tf = new(GSTileFeature) - tf.Id = f.ID - - if tf.Tags, err = keyvalTagsMap(keys, vals, f); err != nil { - return tf, err - } - - geo, gtype, err := EncodeGeometry(f.Geometry) - if err != nil { - return tf, err - } - - if len(geo) == 0 { - return nil, nil - } - - tf.Geometry = geo - tf.Type = >ype - - return tf, nil -} - // These values came from: https://github.com/mapbox/vector-tile-spec/tree/master/2.1 const ( cmdMoveTo uint32 = 1 @@ -365,344 +298,3 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er return nil, TileUNKNOWN, ErrUnknownGeometryType } } - -// keyvalMapsFromFeatures returns a key map and value map, to help with the translation -// to mapbox tile format. In the Tile format, the Tile contains a mapping of all the unique -// keys and values, and then each feature contains a vector map to these two. This is an -// intermediate data structure to help with the construction of the three mappings. -func keyvalMapsFromFeatures(features []Feature) (keyMap []string, valMap []any, err error) { - var didFind bool - for _, f := range features { - for k, v := range f.Tags { - didFind = slices.Contains(keyMap, k) - if !didFind { - keyMap = append(keyMap, k) - } - didFind = false - - switch vt := v.(type) { - default: - if vt == nil { - // ignore nil types - continue - } - return keyMap, valMap, fmt.Errorf("unsupported type for value(%v) with key(%v) in tags for feature %v", vt, k, f) - - case string: - for _, mv := range valMap { - tmv, ok := mv.(string) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case fmt.Stringer: - for _, mv := range valMap { - tmv, ok := mv.(fmt.Stringer) - if !ok { - continue - } - if tmv.String() == vt.String() { - didFind = true - break - } - } - - case int: - for _, mv := range valMap { - tmv, ok := mv.(int) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case int8: - for _, mv := range valMap { - tmv, ok := mv.(int8) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case int16: - for _, mv := range valMap { - tmv, ok := mv.(int16) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case int32: - for _, mv := range valMap { - tmv, ok := mv.(int32) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case int64: - for _, mv := range valMap { - tmv, ok := mv.(int64) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case uint: - for _, mv := range valMap { - tmv, ok := mv.(uint) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case uint8: - for _, mv := range valMap { - tmv, ok := mv.(uint8) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case uint16: - for _, mv := range valMap { - tmv, ok := mv.(uint16) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case uint32: - for _, mv := range valMap { - tmv, ok := mv.(uint32) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case uint64: - for _, mv := range valMap { - tmv, ok := mv.(uint64) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case float32: - for _, mv := range valMap { - tmv, ok := mv.(float32) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case float64: - for _, mv := range valMap { - tmv, ok := mv.(float64) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - case bool: - for _, mv := range valMap { - tmv, ok := mv.(bool) - if !ok { - continue - } - if tmv == vt { - didFind = true - break - } - } - - } // value type switch - - if !didFind { - valMap = append(valMap, v) - } - - } // For f.Tags - } // for features - return keyMap, valMap, nil -} - -// keyvalTagsMap will return the tags map as expected by the mapbox tile spec. It takes -// a keyMap and a valueMap that list the order of the expected keys and values. It will -// return a vector map that refers to these two maps. -func keyvalTagsMap(keyMap []string, valueMap []any, f *Feature) (tags []uint32, err error) { //nolint: cyclop,funlen - if f == nil { - return nil, ErrNilFeature - } - - var kidx, vidx int64 - - for key, val := range f.Tags { - - kidx, vidx = -1, -1 // Set to known not found value. - - for i, k := range keyMap { - if k != key { - continue // move to the next key - } - kidx = int64(i) - break // we found a match - } - - if kidx == -1 { - log.Printf("did not find key (%v) in keymap.", key) - return tags, fmt.Errorf("did not find key (%v) in keymap", key) - } - - // if val is nil we skip it for now - // https://github.com/mapbox/vector-tile-spec/issues/62 - if val == nil { - continue - } - - for i, v := range valueMap { - switch tv := val.(type) { - default: - return tags, fmt.Errorf("value (%[1]v) of type (%[1]T) for key (%[2]v) is not supported", tv, key) - case string: - vmt, ok := v.(string) // Make sure the type of the Value map matches the type of the Tag's value - if !ok || vmt != tv { // and that the values match - continue // if they don't match move to the next value. - } - case fmt.Stringer: - vmt, ok := v.(fmt.Stringer) - if !ok || vmt.String() != tv.String() { - continue - } - case int: - vmt, ok := v.(int) - if !ok || vmt != tv { - continue - } - case int8: - vmt, ok := v.(int8) - if !ok || vmt != tv { - continue - } - case int16: - vmt, ok := v.(int16) - if !ok || vmt != tv { - continue - } - case int32: - vmt, ok := v.(int32) - if !ok || vmt != tv { - continue - } - case int64: - vmt, ok := v.(int64) - if !ok || vmt != tv { - continue - } - case uint: - vmt, ok := v.(uint) - if !ok || vmt != tv { - continue - } - case uint8: - vmt, ok := v.(uint8) - if !ok || vmt != tv { - continue - } - case uint16: - vmt, ok := v.(uint16) - if !ok || vmt != tv { - continue - } - case uint32: - vmt, ok := v.(uint32) - if !ok || vmt != tv { - continue - } - case uint64: - vmt, ok := v.(uint64) - if !ok || vmt != tv { - continue - } - - case float32: - vmt, ok := v.(float32) - if !ok || vmt != tv { - continue - } - case float64: - vmt, ok := v.(float64) - if !ok || vmt != tv { - continue - } - case bool: - vmt, ok := v.(bool) - if !ok || vmt != tv { - continue - } - } // Values Switch Statement - // if the values match let's record the index. - vidx = int64(i) - break // we found our value no need to continue on. - } // range on value - - if vidx == -1 { // None of the values matched. - return tags, fmt.Errorf("did not find a value: %v in valuemap", val) - } - tags = append(tags, uint32(kidx), uint32(vidx)) //nolint: gosec // G115 casting sems unavoidable - } // Move to the next tag key and value. - - return tags, nil -} From 4d06364f30e7135efba5b47049b46b4a802dffd0 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 21 Jul 2026 11:13:32 +0200 Subject: [PATCH 15/25] refactor: rename file for clarity --- tile/{go_spatial_feature.go => go_spatial_encoding.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tile/{go_spatial_feature.go => go_spatial_encoding.go} (100%) diff --git a/tile/go_spatial_feature.go b/tile/go_spatial_encoding.go similarity index 100% rename from tile/go_spatial_feature.go rename to tile/go_spatial_encoding.go From 0a4e32556f0a6520957f3c6cb353f68ac5d70e91 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 21 Jul 2026 11:42:43 +0200 Subject: [PATCH 16/25] refactor: move code into central file --- tile/assemble.go | 221 ++++++++++++++++++ .../{build_layer_test.go => assemble_test.go} | 0 tile/build_feature.go | 46 ---- tile/build_layer.go | 90 ------- tile/keysvalues.go | 42 ---- tile/marshal.go | 9 - tile/tile.go | 40 ---- 7 files changed, 221 insertions(+), 227 deletions(-) create mode 100644 tile/assemble.go rename tile/{build_layer_test.go => assemble_test.go} (100%) delete mode 100644 tile/build_feature.go delete mode 100644 tile/build_layer.go delete mode 100644 tile/keysvalues.go delete mode 100644 tile/marshal.go delete mode 100644 tile/tile.go diff --git a/tile/assemble.go b/tile/assemble.go new file mode 100644 index 0000000..5260fca --- /dev/null +++ b/tile/assemble.go @@ -0,0 +1,221 @@ +package tile + +import ( + "fmt" + + "github.com/go-spatial/geom" + "github.com/go-spatial/geom/encoding/mvt" + oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on + "github.com/pdok/texel/pointindex" +) + +const ( + precision = 4096 + mvtVersion = 2 +) + +type EncodedGeometry struct { + Encoding []uint32 + GeometryType int32 + XTile uint + YTile uint +} + +// Transform geometry to tile extent, then encode. We assume the geometry is +// snapped to the proposed grid, in which case makevalid operations should not +// be necessary. +func MvtEncodeGeometry(q pointindex.Quadrant, g geom.Geometry) EncodedGeometry { + ext := q.Extent() + preparedGeo := mvt.PrepareGeo(g, &ext, float64(precision)) + + // This should not be necessary. + // sg, err := convert.ToTegola(preparedGeo) + // tegolaGeo, err := validate.CleanGeometry(context.TODO(), sg, &ext) + // validatedGeo := convert.ToGeom(tegolaGeo) + + encgeom, geomtype, err := EncodeGeometry(preparedGeo) + if err != nil { + panic(err) + } + + xTile, yTile := q.Coords() + + return EncodedGeometry{ + Encoding: encgeom, + GeometryType: int32(geomtype), + XTile: xTile, + YTile: yTile, + } +} + +// Serialze tile into protobuf format +func MarshalTile(t *GSTile) ([]byte, error) { + return oldproto.Marshal(t) +} + +// Internal representation of attributes: read table[fid][columnName]=value. +type InternalAttributeTable map[int64]map[string]any + +// Build indices from column names +func BuildKeyDictionary(columnNames []string) map[string]uint32 { + index := make(map[string]uint32, len(columnNames)) + for i, name := range columnNames { + if i > int(^uint32(0)) { + panic("Number of keys does not fit in uint32") + } + index[name] = uint32(i) //nolint:gosec // G115 + } + return index +} + +// Build index table for values. This function guarantees that all indices from +// 0 to len(index)-1 all appear exactly once. Absent attributes are not recorded. +func BuildValueDictionary(attributesPerFeature InternalAttributeTable) map[any]uint32 { + index := make(map[any]uint32) + for _, attributes := range attributesPerFeature { + for _, v := range attributes { + if v == nil { + continue + } + if _, ok := index[v]; ok { + continue + } + l := len(index) + if l > int(^uint32(0)) { + panic("Number of values does not fit in uint32") + } + index[v] = uint32(l) + } + } + return index +} + +// Convert index map to slice. Assumes all indices 0, 1, ..., len(index)-1 are +// used exactly once. Necessary since iteration over maps is in arbitrary order. +// This is a generic function since we need it with K = string and K = any. +func orderedByIndex[K comparable](index map[K]uint32) []K { + ordered := make([]K, len(index)) + for k, i := range index { + ordered[i] = k + } + return ordered +} + +// Construct tags for a single feature. `keyIndex` and `valueIndex` are built +// by above functions by considering all features. `attributes` are the +// attributes of the current feature. Absent values are skipped. Panics if a +// key of value is not present. +func buildTags(attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) ([]uint32, error) { + tags := make([]uint32, 0, 2*len(attributes)) + for key, val := range attributes { + if val == nil { + continue + } + + kidx, ok := keyIndex[key] + if !ok { + return nil, fmt.Errorf("did not find key (%v) in key dictionary", key) + } + + vidx, ok := valueIndex[val] + if !ok { + return nil, fmt.Errorf("did not find value (%v) in value dictionary", val) + } + + tags = append(tags, kidx, vidx) + } + return tags, nil +} + +// Construct feature according to protobuf format. +func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*GSTileFeature, error) { + tags, err := buildTags(attributes, keyIndex, valueIndex) + if err != nil { + return nil, fmt.Errorf("feature %d: %w", featureID, err) + } + + //nolint:gosec // G115 feature ids are expected to be non-negative + id := uint64(featureID) + gtype := GSTileGeomType(geom.GeometryType) + + return &GSTileFeature{ + Id: &id, + Tags: tags, + Type: >ype, + Geometry: geom.Encoding, + }, nil +} + +// Construct layer according to protobuf format +func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint32, valueIndex map[any]uint32) *GSTileLayer { + keys := orderedByIndex(keyIndex) + + rawValues := orderedByIndex(valueIndex) + values := make([]*GSTileValue, len(rawValues)) + for i, v := range rawValues { + values[i] = toGSTileValue(v) + } + + version := uint32(mvtVersion) + extent := uint32(precision) + layerName := name + + return &GSTileLayer{ + Version: &version, + Name: &layerName, + Features: features, + Keys: keys, + Values: values, + Extent: &extent, + } +} + +// Construct tile according to protobuf format +func BuildTile(layers ...*GSTileLayer) *GSTile { + return &GSTile{Layers: layers} +} + +// Convert value to GSTileValue. We do not support XXX_unrecognized and panic +// if an unknown type is encountered. This should never happen. +func toGSTileValue(v any) *GSTileValue { + tv := new(GSTileValue) + switch t := v.(type) { + case string: + tv.StringValue = &t + case fmt.Stringer: + str := t.String() + tv.StringValue = &str + case bool: + tv.BoolValue = &t + case int8: + iv := int64(t) + tv.SintValue = &iv + case int16: + iv := int64(t) + tv.SintValue = &iv + case int32: + iv := int64(t) + tv.SintValue = &iv + case int64: + tv.IntValue = &t + case uint8: + iv := int64(t) + tv.SintValue = &iv + case uint16: + iv := int64(t) + tv.SintValue = &iv + case uint32: + iv := int64(t) + tv.SintValue = &iv + case uint64: + tv.UintValue = &t + case float32: + tv.FloatValue = &t + case float64: + tv.DoubleValue = &t + default: + // We should never encounter this. + panic("Unknown type for GSTypeValue.") + } + return tv +} diff --git a/tile/build_layer_test.go b/tile/assemble_test.go similarity index 100% rename from tile/build_layer_test.go rename to tile/assemble_test.go diff --git a/tile/build_feature.go b/tile/build_feature.go deleted file mode 100644 index c14ab23..0000000 --- a/tile/build_feature.go +++ /dev/null @@ -1,46 +0,0 @@ -package tile - -import "fmt" - -// Given attribute map and indices for keys and values, build tag list. -// Panic if key or value is not present. -func buildTags(attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) ([]uint32, error) { - tags := make([]uint32, 0, 2*len(attributes)) - for key, val := range attributes { - if val == nil { - continue - } - - kidx, ok := keyIndex[key] - if !ok { - return nil, fmt.Errorf("did not find key (%v) in key dictionary", key) - } - - vidx, ok := valueIndex[val] - if !ok { - return nil, fmt.Errorf("did not find value (%v) in value dictionary", val) - } - - tags = append(tags, kidx, vidx) - } - return tags, nil -} - -// Build go-spatial compatible feature that can be marshalled. -func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*GSTileFeature, error) { - tags, err := buildTags(attributes, keyIndex, valueIndex) - if err != nil { - return nil, fmt.Errorf("feature %d: %w", featureID, err) - } - - //nolint:gosec // G115 feature ids are expected to be non-negative - id := uint64(featureID) - gtype := GSTileGeomType(geom.GeometryType) - - return &GSTileFeature{ - Id: &id, - Tags: tags, - Type: >ype, - Geometry: geom.Encoding, - }, nil -} diff --git a/tile/build_layer.go b/tile/build_layer.go deleted file mode 100644 index ae38552..0000000 --- a/tile/build_layer.go +++ /dev/null @@ -1,90 +0,0 @@ -package tile - -import "fmt" - -const mvtVersion = 2 - -// Convert tag map to slice. Assumes all indices 0, 1, ..., len(index)-1 are -// exactly used once. Necessary since iteration over maps is in arbitrary order. -// This is a generic function since we need it with K = string and K = any. -func orderedByIndex[K comparable](index map[K]uint32) []K { - ordered := make([]K, len(index)) - for k, i := range index { - ordered[i] = k - } - return ordered -} - -// Convert value to GSTileValue. We do not support XXX_unrecognized and panic -// if an unknown type is encountered. This should never happen. -func toGSTileValue(v any) *GSTileValue { - tv := new(GSTileValue) - switch t := v.(type) { - case string: - tv.StringValue = &t - case fmt.Stringer: - str := t.String() - tv.StringValue = &str - case bool: - tv.BoolValue = &t - case int8: - iv := int64(t) - tv.SintValue = &iv - case int16: - iv := int64(t) - tv.SintValue = &iv - case int32: - iv := int64(t) - tv.SintValue = &iv - case int64: - tv.IntValue = &t - case uint8: - iv := int64(t) - tv.SintValue = &iv - case uint16: - iv := int64(t) - tv.SintValue = &iv - case uint32: - iv := int64(t) - tv.SintValue = &iv - case uint64: - tv.UintValue = &t - case float32: - tv.FloatValue = &t - case float64: - tv.DoubleValue = &t - default: - // We should never encounter this. - panic("Unknown type for GSTypeValue.") - } - return tv -} - -// Assemble layer struct -func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint32, valueIndex map[any]uint32) *GSTileLayer { - keys := orderedByIndex(keyIndex) - - rawValues := orderedByIndex(valueIndex) - values := make([]*GSTileValue, len(rawValues)) - for i, v := range rawValues { - values[i] = toGSTileValue(v) - } - - version := uint32(mvtVersion) - extent := uint32(precision) - layerName := name - - return &GSTileLayer{ - Version: &version, - Name: &layerName, - Features: features, - Keys: keys, - Values: values, - Extent: &extent, - } -} - -// Assemble layers into GSTile. This struct can be marshalled by protobuf. -func BuildTile(layers ...*GSTileLayer) *GSTile { - return &GSTile{Layers: layers} -} diff --git a/tile/keysvalues.go b/tile/keysvalues.go deleted file mode 100644 index 7449ae6..0000000 --- a/tile/keysvalues.go +++ /dev/null @@ -1,42 +0,0 @@ -package tile - -// InternalAttributeTable maps feature ID to a map of attribute name to value, as -// produced by reading a source table's attribute columns for a set of features. -type InternalAttributeTable map[int64]map[string]any - -// Convert slice to map for easy access of indices -func BuildKeyDictionary(attributeNames []string) map[string]uint32 { - index := make(map[string]uint32, len(attributeNames)) - for i, name := range attributeNames { - if i > int(^uint32(0)) { - panic("Number of keys does not fit in uint32") - } - index[name] = uint32(i) //nolint:gosec // G115 this is guarded - } - return index -} - -// BuildValueDictionary scans a tile's feature attributes and returns an index from -// value to a stable dictionary position. Values are compared by (dynamic type, value) - -// matching the MVT tag value semantics used by keyvalTagsMap - which comparable Go types -// (string, the int/uint/float variants, bool) support directly as map keys. nil values -// are skipped, matching the MVT spec's handling of absent tag values (see keyvalTagsMap). -func BuildValueDictionary(attributes InternalAttributeTable) map[any]uint32 { - index := make(map[any]uint32) - for _, attrs := range attributes { - for _, v := range attrs { - if v == nil { - continue - } - if _, ok := index[v]; ok { - continue - } - l := len(index) - if l > int(^uint32(0)) { - panic("Number of values does not fit in uint32") - } - index[v] = uint32(l) - } - } - return index -} diff --git a/tile/marshal.go b/tile/marshal.go deleted file mode 100644 index 3d4a0ee..0000000 --- a/tile/marshal.go +++ /dev/null @@ -1,9 +0,0 @@ -package tile - -import oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on - -// MarshalTile serializes a GSTile into MVT protobuf bytes. Kept in this package so -// callers never need to depend on the protobuf library directly. -func MarshalTile(t *GSTile) ([]byte, error) { - return oldproto.Marshal(t) -} diff --git a/tile/tile.go b/tile/tile.go deleted file mode 100644 index 2310eb8..0000000 --- a/tile/tile.go +++ /dev/null @@ -1,40 +0,0 @@ -package tile - -import ( - "github.com/go-spatial/geom" - "github.com/go-spatial/geom/encoding/mvt" - "github.com/pdok/texel/pointindex" -) - -const precision = 4096 - -type EncodedGeometry struct { - Encoding []uint32 - GeometryType int32 - XTile uint - YTile uint -} - -func MvtEncodeGeometry(q pointindex.Quadrant, g geom.Geometry) EncodedGeometry { - ext := q.Extent() - preparedGeo := mvt.PrepareGeo(g, &ext, float64(precision)) - - // This should not be necessary. - // sg, err := convert.ToTegola(preparedGeo) - // tegolaGeo, err := validate.CleanGeometry(context.TODO(), sg, &ext) - // validatedGeo := convert.ToGeom(tegolaGeo) - - encgeom, geomtype, err := EncodeGeometry(preparedGeo) - if err != nil { - panic(err) - } - - xTile, yTile := q.Coords() - - return EncodedGeometry{ - Encoding: encgeom, - GeometryType: int32(geomtype), - XTile: xTile, - YTile: yTile, - } -} From 5293633387511d74c9b6e06a6e804f30b0df6ae6 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Fri, 24 Jul 2026 07:40:57 +0200 Subject: [PATCH 17/25] refactor: cleaner separation processing and encoding --- processing/gpkg/attributes.go | 8 ------- processing/gpkg/encoded.go | 12 +++------- processing/gpkg/mvttile.go | 43 +++++++++++------------------------ tile/assemble.go | 41 +++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 47 deletions(-) diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go index 8c562c6..25fafef 100644 --- a/processing/gpkg/attributes.go +++ b/processing/gpkg/attributes.go @@ -23,14 +23,6 @@ func (t Table) selectByFIDsSQL(batchSize int) string { return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` } -// Extract features from the map construct. -func AttributesForFeature(attributes tile.InternalAttributeTable, featureID int64) map[string]any { - if attrs, ok := attributes[featureID]; ok { - return attrs - } - return map[string]any{} -} - // GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) // for the given feature IDs, batching the query to respect SQLite's parameter limits. // The fid is assumed to be the table's first column. diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go index c0215db..4e5e381 100644 --- a/processing/gpkg/encoded.go +++ b/processing/gpkg/encoded.go @@ -58,23 +58,17 @@ func deserializeFromBytes(data []byte) []uint32 { return intslice } -// EncodedFeatureRow pairs a feature's identifier with its encoded, tile-relative geometry -// as read back from the "
_encoded" table. -type EncodedFeatureRow struct { - FeatureID int64 - Geom tile.EncodedGeometry -} // GetFeaturesForTile returns every encoded feature stored for the given tile, // i.e. the equivalent of the tile-features table's rows for (tileX, tileY). -func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]EncodedFeatureRow, error) { +func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) if err != nil { return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) } defer rows.Close() - var features []EncodedFeatureRow + var features []tile.EncodedFeatureRow for rows.Next() { var featureID int64 var geometryType int32 @@ -82,7 +76,7 @@ func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]EncodedF if err := rows.Scan(&featureID, &geometryType, &data); err != nil { return nil, fmt.Errorf("scanning encoded feature row for tile (%d,%d): %w", tileX, tileY, err) } - features = append(features, EncodedFeatureRow{ + features = append(features, tile.EncodedFeatureRow{ FeatureID: featureID, Geom: tile.EncodedGeometry{ Encoding: deserializeFromBytes(data), diff --git a/processing/gpkg/mvttile.go b/processing/gpkg/mvttile.go index 7c8b7cf..8f0d709 100644 --- a/processing/gpkg/mvttile.go +++ b/processing/gpkg/mvttile.go @@ -42,43 +42,26 @@ func (source SourceGeopackage) ListTiles() ([]TileCoord, error) { return tiles, nil } -// BuildMVTTile builds and serializes the MVT tile for (tileX, tileY): it fetches the -// tile's encoded features and their attributes, builds the per-tile value dictionary, -// assembles a single layer named layerName using the given (already built once, -// fixed-schema) key dictionary, and marshals the result to MVT protobuf bytes. A tile -// with no encoded features still yields a valid, empty layer rather than an error. +// Build tile for specific coordinates. Fetches all info from database except +// columnnames, which are passed via keyIndex. Returns marshalled vectortile. func (source SourceGeopackage) BuildMVTTile(layerName string, keyIndex map[string]uint32, tileX, tileY uint) ([]byte, error) { - rows, err := source.GetFeaturesForTile(tileX, tileY) + encFeatRows, err := source.GetFeaturesForTile(tileX, tileY) if err != nil { return nil, fmt.Errorf("getting features for tile (%d,%d): %w", tileX, tileY, err) } - - featureIDs := make([]int64, len(rows)) - for i, row := range rows { - featureIDs[i] = row.FeatureID + featIDs := make([]int64, len(encFeatRows)) + for i, encFeatRow := range encFeatRows { + featIDs[i] = encFeatRow.FeatureID } - attributes, err := source.GetAttributesForFeatures(featureIDs) + attributes, err := source.GetAttributesForFeatures(featIDs) if err != nil { return nil, fmt.Errorf("getting attributes for tile (%d,%d): %w", tileX, tileY, err) } - valueIndex := tile.BuildValueDictionary(attributes) - - features := make([]*tile.GSTileFeature, 0, len(rows)) - for _, row := range rows { - attrs := AttributesForFeature(attributes, row.FeatureID) - feature, err := tile.BuildFeature(row.FeatureID, row.Geom, attrs, keyIndex, valueIndex) - if err != nil { - return nil, fmt.Errorf("building tile (%d,%d): %w", tileX, tileY, err) - } - features = append(features, feature) - } - - layer := tile.BuildLayer(layerName, features, keyIndex, valueIndex) - data, err := tile.MarshalTile(tile.BuildTile(layer)) + data, err := tile.BuildMVTTile(layerName, keyIndex, encFeatRows, attributes) if err != nil { - return nil, fmt.Errorf("marshaling tile (%d,%d): %w", tileX, tileY, err) + return nil, fmt.Errorf("building tile (%d,%d): %w", tileX, tileY, err) } return data, nil } @@ -98,11 +81,11 @@ func WriteTileFile(baseDir string, tileX, tileY uint, data []byte) error { return nil } -// ProcessTilesToMVT enumerates every tile with encoded features for source.Table, -// builds each tile's MVT bytes (see BuildMVTTile) and writes it to outDir via -// WriteTileFile. The key dictionary is built once, up front, since the attribute -// schema is fixed for the whole table (see tile.BuildKeyDictionary). +// For all tiles, create the corresponding vectortile and write it to file. In +// this context `all tiles` means all tiles for which encoded geometries exist +// in the source geopackage. func (source SourceGeopackage) ProcessTilesToMVT(outDir string) error { + // Build key index (columns) just once. keyIndex := tile.BuildKeyDictionary(source.Table.AttributeColumnNames()) tiles, err := source.ListTiles() diff --git a/tile/assemble.go b/tile/assemble.go index 5260fca..9d85a7f 100644 --- a/tile/assemble.go +++ b/tile/assemble.go @@ -21,6 +21,13 @@ type EncodedGeometry struct { YTile uint } +// EncodedFeatureRow pairs a feature's identifier with its encoded, tile-relative geometry +// as read back from the "
_encoded" table. +type EncodedFeatureRow struct { + FeatureID int64 + Geom EncodedGeometry +} + // Transform geometry to tile extent, then encode. We assume the geometry is // snapped to the proposed grid, in which case makevalid operations should not // be necessary. @@ -175,6 +182,40 @@ func BuildTile(layers ...*GSTileLayer) *GSTile { return &GSTile{Layers: layers} } +// Main exported function that encodes a tile of a single layer. Assumes +// geometry and keyIndex have already been encoded. Returns marshalled byte +// string. +func BuildMVTTile(layerName string, keyIndex map[string]uint32, encFeatRows []EncodedFeatureRow, attributes InternalAttributeTable) ([]byte, error) { + valueIndex := BuildValueDictionary(attributes) + + features := make([]*GSTileFeature, 0, len(encFeatRows)) + for _, encodedFeature := range encFeatRows { + featureID := encodedFeature.FeatureID + attrs := attributesForFeature(attributes, featureID) + feature, err := BuildFeature(featureID, encodedFeature.Geom, attrs, keyIndex, valueIndex) + if err != nil { + return nil, err + } + features = append(features, feature) + } + + layer := BuildLayer(layerName, features, keyIndex, valueIndex) + + data, err := MarshalTile(BuildTile(layer)) + if err != nil { + return nil, fmt.Errorf("marshaling tile: %w", err) + } + return data, nil +} + +// attributesForFeature extracts a single feature's attributes from the map construct. +func attributesForFeature(attributes InternalAttributeTable, featureID int64) map[string]any { + if attrs, ok := attributes[featureID]; ok { + return attrs + } + return map[string]any{} +} + // Convert value to GSTileValue. We do not support XXX_unrecognized and panic // if an unknown type is encountered. This should never happen. func toGSTileValue(v any) *GSTileValue { From 7a533faf88a7f5ef4723d0aad1b006cbf83d269a Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Fri, 24 Jul 2026 08:14:14 +0200 Subject: [PATCH 18/25] refactor: align with protobuf specs --- processing/gpkg/encoded.go | 1 - tile/assemble.go | 84 +++++++++++++-------- tile/assemble_test.go | 143 +++++++++++++++++++++++++++++++++++- tile/go_spatial_encoding.go | 23 +++--- tile/go_spatial_types.go | 100 ------------------------- 5 files changed, 206 insertions(+), 145 deletions(-) delete mode 100644 tile/go_spatial_types.go diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go index 4e5e381..178c87e 100644 --- a/processing/gpkg/encoded.go +++ b/processing/gpkg/encoded.go @@ -58,7 +58,6 @@ func deserializeFromBytes(data []byte) []uint32 { return intslice } - // GetFeaturesForTile returns every encoded feature stored for the given tile, // i.e. the equivalent of the tile-features table's rows for (tileX, tileY). func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { diff --git a/tile/assemble.go b/tile/assemble.go index 9d85a7f..1ffc0fd 100644 --- a/tile/assemble.go +++ b/tile/assemble.go @@ -1,10 +1,13 @@ package tile import ( + "bytes" + "encoding/binary" "fmt" "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/mvt" + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on "github.com/pdok/texel/pointindex" ) @@ -56,7 +59,7 @@ func MvtEncodeGeometry(q pointindex.Quadrant, g geom.Geometry) EncodedGeometry { } // Serialze tile into protobuf format -func MarshalTile(t *GSTile) ([]byte, error) { +func MarshalTile(t *vectorTile.Tile) ([]byte, error) { return oldproto.Marshal(t) } @@ -135,7 +138,7 @@ func buildTags(attributes map[string]any, keyIndex map[string]uint32, valueIndex } // Construct feature according to protobuf format. -func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*GSTileFeature, error) { +func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*vectorTile.Tile_Feature, error) { tags, err := buildTags(attributes, keyIndex, valueIndex) if err != nil { return nil, fmt.Errorf("feature %d: %w", featureID, err) @@ -143,9 +146,9 @@ func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]a //nolint:gosec // G115 feature ids are expected to be non-negative id := uint64(featureID) - gtype := GSTileGeomType(geom.GeometryType) + gtype := vectorTile.Tile_GeomType(geom.GeometryType) - return &GSTileFeature{ + return &vectorTile.Tile_Feature{ Id: &id, Tags: tags, Type: >ype, @@ -154,20 +157,20 @@ func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]a } // Construct layer according to protobuf format -func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint32, valueIndex map[any]uint32) *GSTileLayer { +func BuildLayer(name string, features []*vectorTile.Tile_Feature, keyIndex map[string]uint32, valueIndex map[any]uint32) *vectorTile.Tile_Layer { keys := orderedByIndex(keyIndex) rawValues := orderedByIndex(valueIndex) - values := make([]*GSTileValue, len(rawValues)) + values := make([]*vectorTile.Tile_Value, len(rawValues)) for i, v := range rawValues { - values[i] = toGSTileValue(v) + values[i] = vectorTileValue(v) } version := uint32(mvtVersion) extent := uint32(precision) layerName := name - return &GSTileLayer{ + return &vectorTile.Tile_Layer{ Version: &version, Name: &layerName, Features: features, @@ -178,8 +181,8 @@ func BuildLayer(name string, features []*GSTileFeature, keyIndex map[string]uint } // Construct tile according to protobuf format -func BuildTile(layers ...*GSTileLayer) *GSTile { - return &GSTile{Layers: layers} +func BuildTile(layers ...*vectorTile.Tile_Layer) *vectorTile.Tile { + return &vectorTile.Tile{Layers: layers} } // Main exported function that encodes a tile of a single layer. Assumes @@ -188,7 +191,7 @@ func BuildTile(layers ...*GSTileLayer) *GSTile { func BuildMVTTile(layerName string, keyIndex map[string]uint32, encFeatRows []EncodedFeatureRow, attributes InternalAttributeTable) ([]byte, error) { valueIndex := BuildValueDictionary(attributes) - features := make([]*GSTileFeature, 0, len(encFeatRows)) + features := make([]*vectorTile.Tile_Feature, 0, len(encFeatRows)) for _, encodedFeature := range encFeatRows { featureID := encodedFeature.FeatureID attrs := attributesForFeature(attributes, featureID) @@ -216,47 +219,64 @@ func attributesForFeature(attributes InternalAttributeTable, featureID int64) ma return map[string]any{} } -// Convert value to GSTileValue. We do not support XXX_unrecognized and panic -// if an unknown type is encountered. This should never happen. -func toGSTileValue(v any) *GSTileValue { - tv := new(GSTileValue) - switch t := v.(type) { +// Convert value to Tile_Value. Copied from go-spatial/encoding/mvt/layer.go +func vectorTileValue(i any) *vectorTile.Tile_Value { //nolint:cyclop, funlen + tv := new(vectorTile.Tile_Value) + switch t := i.(type) { + default: + buff := new(bytes.Buffer) + err := binary.Write(buff, binary.BigEndian, t) + // We are going to ignore the value and return an empty TileValue + if err == nil { + tv.XXX_unrecognized = buff.Bytes() + } + case string: tv.StringValue = &t + case fmt.Stringer: str := t.String() tv.StringValue = &str + case bool: tv.BoolValue = &t + case int8: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case int16: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case int32: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case int64: tv.IntValue = &t + case uint8: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case uint16: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case uint32: - iv := int64(t) - tv.SintValue = &iv + intv := int64(t) + tv.SintValue = &intv + case uint64: tv.UintValue = &t + case float32: tv.FloatValue = &t + case float64: tv.DoubleValue = &t - default: - // We should never encounter this. - panic("Unknown type for GSTypeValue.") - } + + } // switch return tv } diff --git a/tile/assemble_test.go b/tile/assemble_test.go index 0c8e8bc..674eee2 100644 --- a/tile/assemble_test.go +++ b/tile/assemble_test.go @@ -1,6 +1,7 @@ package tile_test import ( + "reflect" "testing" vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" @@ -37,7 +38,7 @@ func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { t.Fatalf("BuildFeature(2): %v", err) } - layer := tile.BuildLayer("mytable", []*tile.GSTileFeature{feature1, feature2}, keyIndex, valueIndex) + layer := tile.BuildLayer("mytable", []*vectorTile.Tile_Feature{feature1, feature2}, keyIndex, valueIndex) gsTile := tile.BuildTile(layer) b, err := oldproto.Marshal(gsTile) @@ -88,6 +89,146 @@ func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { } } +// TestBuildMVTTile verifies that BuildMVTTile's per-tile assembly (value dictionary +// construction, feature/attribute lookup, layer/tile building and marshaling) +// produces a tile whose features/attributes match expectations, by decoding the +// marshaled bytes back and resolving each feature's tags through the layer's +// Keys/Values dictionaries. This is compared structurally rather than byte-for-byte, +// since BuildValueDictionary/buildTags iterate Go maps, whose order (and thus the +// resulting dictionary/tag order) is not guaranteed across runs. +func TestBuildMVTTile(t *testing.T) { + keys := []string{"ownerID", "function"} + keyIndex := tile.BuildKeyDictionary(keys) + fids := []uint64{2, 100, 101, 404} + encGeometries := []tile.EncodedGeometry{ + {Encoding: []uint32{9, 2, 2}, GeometryType: int32(vectorTile.Tile_POINT)}, + {Encoding: []uint32{1, 2, 3}, GeometryType: int32(vectorTile.Tile_POLYGON)}, + {Encoding: []uint32{9, 4, 4}, GeometryType: int32(vectorTile.Tile_POINT)}, + {Encoding: []uint32{9, 6, 6}, GeometryType: int32(vectorTile.Tile_POINT)}, + } + attributeTable := tile.InternalAttributeTable{ + 2: {"ownerID": int64(3), "function": "production"}, + 100: {"ownerID": int64(4)}, + 101: {"ownerID": int64(3), "function": "living"}, + } + encRows := make([]tile.EncodedFeatureRow, len(fids)) + for i, fid := range fids { + encRows[i] = tile.EncodedFeatureRow{FeatureID: int64(fid), Geom: encGeometries[i]} //nolint:gosec // G115 test fids fit within int64 + } + + layerName := "mylayer" + + got, err := tile.BuildMVTTile(layerName, keyIndex, encRows, attributeTable) + if err != nil { + t.Fatalf("BuildMVTTile: %v", err) + } + + var gotTile vectorTile.Tile + if err := oldproto.Unmarshal(got, &gotTile); err != nil { + t.Fatalf("unmarshaling got tile: %v", err) + } + + wantGeoms := map[uint64]tile.EncodedGeometry{ + fids[0]: encGeometries[0], + fids[1]: encGeometries[1], + fids[2]: encGeometries[2], + fids[3]: encGeometries[3], + } + wantAttrs := map[uint64]map[string]any{ + fids[0]: {"ownerID": int64(3), "function": "production"}, + fids[1]: {"ownerID": int64(4)}, + fids[2]: {"ownerID": int64(3), "function": "living"}, + fids[3]: {}, + } + + assertMVTLayer(t, gotTile, layerName, keys, fids, wantGeoms, wantAttrs) +} + +// assertMVTLayer checks that tl contains a single layer named layerName, with the +// given keys, and one feature per fid (in that order) whose geometry matches +// wantGeoms[fid] and whose tags, resolved through the layer's Keys/Values +// dictionaries, match wantAttrs[fid]. Resolving through the dictionaries makes the +// comparison independent of the (unspecified) order BuildValueDictionary assigns. +func assertMVTLayer(t *testing.T, tl vectorTile.Tile, layerName string, keys []string, fids []uint64, wantGeoms map[uint64]tile.EncodedGeometry, wantAttrs map[uint64]map[string]any) { + t.Helper() + if len(tl.GetLayers()) != 1 { + t.Fatalf("expected 1 layer, got %d", len(tl.GetLayers())) + } + l := tl.GetLayers()[0] + if l.Name == nil || *l.Name != layerName { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + t.Errorf("layer name = %v, want %q", l.Name, layerName) + } + if l.Version == nil || *l.Version != 2 { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + t.Errorf("layer version = %v, want 2", l.Version) + } + if l.Extent == nil || *l.Extent != 4096 { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + t.Errorf("layer extent = %v, want 4096", l.Extent) + } + if !reflect.DeepEqual(l.GetKeys(), keys) { + t.Errorf("layer keys = %v, want %v", l.GetKeys(), keys) + } + if len(l.GetFeatures()) != len(fids) { + t.Fatalf("expected %d features, got %d", len(fids), len(l.GetFeatures())) + } + + for i, fid := range fids { + feat := l.GetFeatures()[i] + if feat.Id == nil || *feat.Id != fid { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + t.Errorf("feature %d id = %v, want %d", i, feat.Id, fid) + continue + } + wantGeom := wantGeoms[fid] + if feat.Type == nil || *feat.Type != vectorTile.Tile_GeomType(wantGeom.GeometryType) { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + t.Errorf("feature %d (fid %d) type = %v, want %v", i, fid, feat.Type, wantGeom.GeometryType) + } + if !reflect.DeepEqual(feat.GetGeometry(), wantGeom.Encoding) { + t.Errorf("feature %d (fid %d) geometry = %v, want %v", i, fid, feat.GetGeometry(), wantGeom.Encoding) + } + + gotAttrs := resolveTags(t, feat.GetTags(), l.GetKeys(), l.GetValues()) + if !reflect.DeepEqual(gotAttrs, wantAttrs[fid]) { + t.Errorf("feature %d (fid %d) attrs = %v, want %v", i, fid, gotAttrs, wantAttrs[fid]) + } + } +} + +// resolveTags decodes a feature's (key index, value index) tag pairs back into a +// plain map[string]any, using the layer's key/value dictionaries. +func resolveTags(t *testing.T, tags []uint32, keys []string, values []*vectorTile.Tile_Value) map[string]any { + t.Helper() + attrs := make(map[string]any, len(tags)/2) + for i := 0; i+1 < len(tags); i += 2 { + kidx, vidx := tags[i], tags[i+1] + if int(kidx) >= len(keys) || int(vidx) >= len(values) { + t.Fatalf("tag pair (%d, %d) out of range (keys=%d, values=%d)", kidx, vidx, len(keys), len(values)) + } + attrs[keys[kidx]] = decodeGSValue(values[vidx]) + } + return attrs +} + +// decodeGSValue extracts the single set field of a GSTileValue as a plain Go value. +func decodeGSValue(v *vectorTile.Tile_Value) any { + switch { + case v.StringValue != nil: + return *v.StringValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.IntValue != nil: + return *v.IntValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.SintValue != nil: + return *v.SintValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.UintValue != nil: + return *v.UintValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.FloatValue != nil: + return *v.FloatValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.DoubleValue != nil: + return *v.DoubleValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + case v.BoolValue != nil: + return *v.BoolValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters + default: + return nil + } +} + func equalStrings(a, b []string) bool { if len(a) != len(b) { return false diff --git a/tile/go_spatial_encoding.go b/tile/go_spatial_encoding.go index f19324c..5835943 100644 --- a/tile/go_spatial_encoding.go +++ b/tile/go_spatial_encoding.go @@ -13,6 +13,7 @@ import ( "log" "github.com/go-spatial/geom" + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" "github.com/go-spatial/geom/encoding/wkt" "github.com/go-spatial/geom/winding" ) @@ -241,9 +242,9 @@ func (c *cursor) encodePolygon(geo geom.Polygon) []uint32 { // EncodeGeometry will take a geom.Geometry and encode it according to the // mapbox vector_tile spec. -func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, err error) { +func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp vectorTile.Tile_GeomType, err error) { if geometry == nil { - return nil, TileUNKNOWN, ErrNilGeometryType + return nil, vectorTile.Tile_UNKNOWN, ErrNilGeometryType } c := NewCursor() @@ -251,17 +252,17 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er switch t := geometry.(type) { case geom.Point: g = append(g, c.MoveTo(t)...) - return g, TilePOINT, nil + return g, vectorTile.Tile_POINT, nil case geom.MultiPoint: g = append(g, c.MoveTo(t.Points()...)...) - return g, TilePOINT, nil + return g, vectorTile.Tile_POINT, nil case geom.LineString: points := t.Vertices() g = append(g, c.MoveTo(points[0])...) g = append(g, c.LineTo(points[1:]...)...) - return g, TileLINESTRING, nil + return g, vectorTile.Tile_LINESTRING, nil case geom.MultiLineString: lines := t.LineStrings() @@ -270,31 +271,31 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er g = append(g, c.MoveTo(points[0])...) g = append(g, c.LineTo(points[1:]...)...) } - return g, TileLINESTRING, nil + return g, vectorTile.Tile_LINESTRING, nil case geom.Polygon: g = append(g, c.encodePolygon(t)...) - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil case geom.MultiPolygon: polygons := t.Polygons() for _, p := range polygons { g = append(g, c.encodePolygon(p)...) } - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil case *geom.MultiPolygon: if t == nil { - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil } polygons := t.Polygons() for _, p := range polygons { g = append(g, c.encodePolygon(p)...) } - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil default: - return nil, TileUNKNOWN, ErrUnknownGeometryType + return nil, vectorTile.Tile_UNKNOWN, ErrUnknownGeometryType } } diff --git a/tile/go_spatial_types.go b/tile/go_spatial_types.go deleted file mode 100644 index ed5b483..0000000 --- a/tile/go_spatial_types.go +++ /dev/null @@ -1,100 +0,0 @@ -package tile - -// These are the types obtained from go-spatial/encoding/mvt/vectortile needed -// for encoding. Modifications made: -// - Renamed types for linter purposes. -// - Dropped some protobuf functionality not used. -// - Use fmt for debug printing rather than proto. - -import ( - "fmt" -) - -// Geometry types - -type GSTileGeomType int32 - -const ( - TileUNKNOWN GSTileGeomType = 0 - TilePOINT GSTileGeomType = 1 - TileLINESTRING GSTileGeomType = 2 - TilePOLYGON GSTileGeomType = 3 -) - -var TileGeomTypeName = map[int32]string{ - 0: "UNKNOWN", - 1: "POINT", - 2: "LINESTRING", - 3: "POLYGON", -} - -var TileGeomTypeValue = map[string]int32{ - "UNKNOWN": 0, - "POINT": 1, - "LINESTRING": 2, - "POLYGON": 3, -} - -// GSTileFeature mirrors vector_tile.pb.go's Tile_Feature. - -type GSTileFeature struct { - Id *uint64 `protobuf:"varint,1,opt,name=id,def=0" json:"id,omitempty"` - // Tags of this feature are encoded as repeated pairs of - // integers. - // A detailed description of tags is located in sections - // 4.2 and 4.4 of the specification - Tags []uint32 `protobuf:"varint,2,rep,packed,name=tags" json:"tags,omitempty"` - // The type of geometry stored in this feature. - Type *GSTileGeomType `protobuf:"varint,3,opt,name=type,enum=vector_tile.Tile_GeomType,def=0" json:"type,omitempty"` - // Contains a stream of commands and parameters (vertices). - // A detailed description on geometry encoding is located in - // section 4.3 of the specification. - Geometry []uint32 `protobuf:"varint,4,rep,packed,name=geometry" json:"geometry,omitempty"` - XXXunrecognized []byte `json:"-"` -} - -func (m *GSTileFeature) Reset() { *m = GSTileFeature{} } -func (m *GSTileFeature) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTileFeature) ProtoMessage() {} - -// GSTileValue mirrors vector_tile.pb.go's Tile_Value. -// Exactly one of these fields should be set for a valid value. - -type GSTileValue struct { - StringValue *string `protobuf:"bytes,1,opt,name=string_value" json:"stringValue,omitempty"` - FloatValue *float32 `protobuf:"fixed32,2,opt,name=float_value" json:"floatValue,omitempty"` - DoubleValue *float64 `protobuf:"fixed64,3,opt,name=double_value" json:"doubleValue,omitempty"` - IntValue *int64 `protobuf:"varint,4,opt,name=int_value" json:"intValue,omitempty"` - UintValue *uint64 `protobuf:"varint,5,opt,name=uint_value" json:"uintValue,omitempty"` - SintValue *int64 `protobuf:"zigzag64,6,opt,name=sint_value" json:"sintValue,omitempty"` - BoolValue *bool `protobuf:"varint,7,opt,name=bool_value" json:"boolValue,omitempty"` -} - -func (m *GSTileValue) Reset() { *m = GSTileValue{} } -func (m *GSTileValue) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTileValue) ProtoMessage() {} - -// GSTileLayer mirrors vector_tile.pb.go's Tile_Layer. - -type GSTileLayer struct { - Version *uint32 `protobuf:"varint,15,req,name=version,def=1" json:"version,omitempty"` - Name *string `protobuf:"bytes,1,req,name=name" json:"name,omitempty"` - Features []*GSTileFeature `protobuf:"bytes,2,rep,name=features" json:"features,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=keys" json:"keys,omitempty"` - Values []*GSTileValue `protobuf:"bytes,4,rep,name=values" json:"values,omitempty"` - Extent *uint32 `protobuf:"varint,5,opt,name=extent,def=4096" json:"extent,omitempty"` -} - -func (m *GSTileLayer) Reset() { *m = GSTileLayer{} } -func (m *GSTileLayer) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTileLayer) ProtoMessage() {} - -// GSTile mirrors vector_tile.pb.go's Tile. - -type GSTile struct { - Layers []*GSTileLayer `protobuf:"bytes,3,rep,name=layers" json:"layers,omitempty"` -} - -func (m *GSTile) Reset() { *m = GSTile{} } -func (m *GSTile) String() string { return fmt.Sprintf("%+v", *m) } -func (*GSTile) ProtoMessage() {} From 337597db6ed1457ae8680eb0807a46be3b3f01a3 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Fri, 24 Jul 2026 09:51:57 +0200 Subject: [PATCH 19/25] test: tile encoding --- tile/assemble_test.go | 428 +++++++++++++++++++++--------------------- 1 file changed, 211 insertions(+), 217 deletions(-) diff --git a/tile/assemble_test.go b/tile/assemble_test.go index 674eee2..79aecfb 100644 --- a/tile/assemble_test.go +++ b/tile/assemble_test.go @@ -1,242 +1,236 @@ -package tile_test +package tile import ( - "reflect" + "sort" "testing" vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" - oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on - "github.com/pdok/texel/tile" + oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // matches MarshalTile's use of the reflection-based marshaler + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -// TestBuildLayerIsWireCompatibleWithVectorTile verifies that our hand-copied, -// GS-prefixed protobuf types (GSTile/GSTileLayer/GSTileValue/GSTileFeature in -// go_spatial_layer.go and go_spatial_feature.go) produce bytes on the wire that are -// indistinguishable from the real, code-generated go-spatial/geom vector_tile types. -// -// This matters because our types are not registered/generated protobuf messages: they -// only implement the minimal proto.Message interface (Reset/String/ProtoMessage) and -// rely on the same `protobuf:"..."` struct tags as the original vector_tile.pb.go for -// the classic reflection-based (github.com/golang/protobuf/proto) marshaler to encode -// them correctly. A typo in a tag (wrong field number/wire type) would silently produce -// an invalid or misinterpreted MVT tile, so this test marshals with our types and -// unmarshals with the real, generated ones to prove the two are wire-compatible. -func TestBuildLayerIsWireCompatibleWithVectorTile(t *testing.T) { - keyIndex := tile.BuildKeyDictionary([]string{"name", "count"}) - attributes := tile.InternalAttributeTable{ - 1: {"name": "foo", "count": int64(3)}, - 2: {"name": "bar", "count": int64(3)}, // shares the "count" value with feature 1 - } - valueIndex := tile.BuildValueDictionary(attributes) - - feature1, err := tile.BuildFeature(1, tile.EncodedGeometry{Encoding: []uint32{9, 2, 2}, GeometryType: 1}, attributes[1], keyIndex, valueIndex) - if err != nil { - t.Fatalf("BuildFeature(1): %v", err) - } - feature2, err := tile.BuildFeature(2, tile.EncodedGeometry{Encoding: []uint32{9, 4, 4}, GeometryType: 1}, attributes[2], keyIndex, valueIndex) - if err != nil { - t.Fatalf("BuildFeature(2): %v", err) - } - - layer := tile.BuildLayer("mytable", []*vectorTile.Tile_Feature{feature1, feature2}, keyIndex, valueIndex) - gsTile := tile.BuildTile(layer) - - b, err := oldproto.Marshal(gsTile) - if err != nil { - t.Fatalf("marshaling our GSTile: %v", err) - } - - var vt vectorTile.Tile - if err := oldproto.Unmarshal(b, &vt); err != nil { - t.Fatalf("unmarshaling with the real vectorTile.Tile: %v", err) +// Quick test BuildKeyDictionary +func TestBuildKeyDictionary(t *testing.T) { + tests := []struct { + name string + columnNames []string + want map[string]uint32 + }{ + { + name: "empty slice yields empty map", + columnNames: []string{}, + want: map[string]uint32{}, + }, + { + name: "single column", + columnNames: []string{"a"}, + want: map[string]uint32{"a": 0}, + }, + { + name: "multiple columns keep their position as index", + columnNames: []string{"a", "b", "c"}, + want: map[string]uint32{"a": 0, "b": 1, "c": 2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildKeyDictionary(tt.columnNames) + assert.Equal(t, tt.want, got) + }) } +} - if len(vt.GetLayers()) != 1 { - t.Fatalf("expected 1 layer, got %d", len(vt.GetLayers())) +// indexedKeys returns the keys of index sorted by their assigned uint32, +// i.e. it undoes the (arbitrary, since built from a map) assignment of +// indices so tests can assert on order-independent content. +func indexedKeys(index map[any]uint32) []any { + keys := make([]any, len(index)) + for k, i := range index { + keys[i] = k } - l := vt.GetLayers()[0] + return keys +} - if got := l.GetName(); got != "mytable" { - t.Errorf("layer name = %q, want %q", got, "mytable") - } - if got := l.GetVersion(); got != 2 { - t.Errorf("layer version = %d, want 2", got) - } - if got := l.GetExtent(); got != 4096 { - t.Errorf("layer extent = %d, want 4096", got) - } - if want := []string{"name", "count"}; !equalStrings(l.GetKeys(), want) { - t.Errorf("layer keys = %v, want %v", l.GetKeys(), want) - } - // "foo", "bar" and the shared int64(3) => 3 distinct values. - if len(l.GetValues()) != 3 { - t.Fatalf("expected 3 distinct values, got %d: %+v", len(l.GetValues()), l.GetValues()) - } - if len(l.GetFeatures()) != 2 { - t.Fatalf("expected 2 features, got %d", len(l.GetFeatures())) +func TestBuildValueDictionary(t *testing.T) { + tests := []struct { + name string + attributesPerFeature InternalAttributeTable + wantValues []any // expected distinct, non-nil values (order-independent) + }{ + { + name: "empty table yields empty dictionary", + attributesPerFeature: InternalAttributeTable{}, + wantValues: []any{}, + }, + { + name: "single feature, single attribute", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo"}, + }, + wantValues: []any{"foo"}, + }, + { + name: "shared values across features are only counted once", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo"}, + 2: {"name": "foo"}, + 3: {"name": "bar"}, + }, + wantValues: []any{"foo", "bar"}, + }, + { + name: "nil values are skipped", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo", "optional": nil}, + }, + wantValues: []any{"foo"}, + }, + { + name: "mixed value types are all indexed", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo", "count": int64(3), "active": true}, + }, + wantValues: []any{"foo", int64(3), true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildValueDictionary(tt.attributesPerFeature) + + // Every index from 0..len(got)-1 must be used exactly once. + assert.Len(t, got, len(tt.wantValues)) + gotValues := indexedKeys(got) + assert.ElementsMatch(t, tt.wantValues, gotValues) + }) } +} - for i, feat := range l.GetFeatures() { - if feat.GetType() != vectorTile.Tile_POINT { - t.Errorf("feature %d type = %v, want POINT", i, feat.GetType()) +func TestBuildTags(t *testing.T) { + keyIndex := map[string]uint32{"name": 0, "count": 1} + valueIndex := map[any]uint32{"foo": 0, int64(3): 1} + + t.Run("empty attributes yield empty tags", func(t *testing.T) { + tags, err := buildTags(map[string]any{}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Empty(t, tags) + }) + + t.Run("single attribute produces a key/value index pair", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo"}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Equal(t, []uint32{0, 0}, tags) + }) + + t.Run("nil-valued attribute is skipped", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo", "count": nil}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Equal(t, []uint32{0, 0}, tags) + }) + + t.Run("multiple attributes produce all pairs", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo", "count": int64(3)}, keyIndex, valueIndex) + require.NoError(t, err) + + // Tags is built from map iteration, so pair order is not + // guaranteed; compare as a set of (key, value) pairs instead. + require.Len(t, tags, 4) + pairs := map[[2]uint32]bool{} + for i := 0; i < len(tags); i += 2 { + pairs[[2]uint32{tags[i], tags[i+1]}] = true } - if len(feat.GetTags())%2 != 0 { - t.Errorf("feature %d tags = %v, expected an even number of (key,value) indices", i, feat.GetTags()) - } - } - if feat0Geo, feat1Geo := l.GetFeatures()[0].GetGeometry(), l.GetFeatures()[1].GetGeometry(); len(feat0Geo) == 0 || len(feat1Geo) == 0 { - t.Errorf("expected both features to keep their encoded geometry, got %v and %v", feat0Geo, feat1Geo) - } + assert.True(t, pairs[[2]uint32{0, 0}]) + assert.True(t, pairs[[2]uint32{1, 1}]) + }) + + t.Run("missing key returns an error", func(t *testing.T) { + _, err := buildTags(map[string]any{"unknown": "foo"}, keyIndex, valueIndex) + require.Error(t, err) + }) + + t.Run("missing value returns an error", func(t *testing.T) { + _, err := buildTags(map[string]any{"name": "unindexed value"}, keyIndex, valueIndex) + require.Error(t, err) + }) } -// TestBuildMVTTile verifies that BuildMVTTile's per-tile assembly (value dictionary -// construction, feature/attribute lookup, layer/tile building and marshaling) -// produces a tile whose features/attributes match expectations, by decoding the -// marshaled bytes back and resolving each feature's tags through the layer's -// Keys/Values dictionaries. This is compared structurally rather than byte-for-byte, -// since BuildValueDictionary/buildTags iterate Go maps, whose order (and thus the -// resulting dictionary/tag order) is not guaranteed across runs. -func TestBuildMVTTile(t *testing.T) { - keys := []string{"ownerID", "function"} - keyIndex := tile.BuildKeyDictionary(keys) - fids := []uint64{2, 100, 101, 404} - encGeometries := []tile.EncodedGeometry{ - {Encoding: []uint32{9, 2, 2}, GeometryType: int32(vectorTile.Tile_POINT)}, - {Encoding: []uint32{1, 2, 3}, GeometryType: int32(vectorTile.Tile_POLYGON)}, - {Encoding: []uint32{9, 4, 4}, GeometryType: int32(vectorTile.Tile_POINT)}, - {Encoding: []uint32{9, 6, 6}, GeometryType: int32(vectorTile.Tile_POINT)}, - } - attributeTable := tile.InternalAttributeTable{ - 2: {"ownerID": int64(3), "function": "production"}, - 100: {"ownerID": int64(4)}, - 101: {"ownerID": int64(3), "function": "living"}, - } - encRows := make([]tile.EncodedFeatureRow, len(fids)) - for i, fid := range fids { - encRows[i] = tile.EncodedFeatureRow{FeatureID: int64(fid), Geom: encGeometries[i]} //nolint:gosec // G115 test fids fit within int64 - } - - layerName := "mylayer" - - got, err := tile.BuildMVTTile(layerName, keyIndex, encRows, attributeTable) - if err != nil { - t.Fatalf("BuildMVTTile: %v", err) - } - - var gotTile vectorTile.Tile - if err := oldproto.Unmarshal(got, &gotTile); err != nil { - t.Fatalf("unmarshaling got tile: %v", err) - } - - wantGeoms := map[uint64]tile.EncodedGeometry{ - fids[0]: encGeometries[0], - fids[1]: encGeometries[1], - fids[2]: encGeometries[2], - fids[3]: encGeometries[3], - } - wantAttrs := map[uint64]map[string]any{ - fids[0]: {"ownerID": int64(3), "function": "production"}, - fids[1]: {"ownerID": int64(4)}, - fids[2]: {"ownerID": int64(3), "function": "living"}, - fids[3]: {}, - } - - assertMVTLayer(t, gotTile, layerName, keys, fids, wantGeoms, wantAttrs) +func TestBuildLayer(t *testing.T) { + t.Run("happy path building layer", func(t *testing.T) { + keyIndex := map[string]uint32{"b": 1, "a": 0} + valueIndex := map[any]uint32{"second": 1, "first": 0} + features := []*vectorTile.Tile_Feature{{}, {}} + + layer := BuildLayer("mylayer", features, keyIndex, valueIndex) + + assert.Equal(t, "mylayer", layer.GetName()) + assert.Equal(t, uint32(mvtVersion), layer.GetVersion()) + assert.Equal(t, uint32(precision), layer.GetExtent()) + assert.Same(t, features[0], layer.GetFeatures()[0]) + assert.Same(t, features[1], layer.GetFeatures()[1]) + + // Keys/values must be ordered according to their assigned index, + // regardless of the (arbitrary) map iteration order used to build them. + assert.Equal(t, []string{"a", "b"}, layer.GetKeys()) + require.Len(t, layer.GetValues(), 2) + assert.Equal(t, "first", layer.GetValues()[0].GetStringValue()) + assert.Equal(t, "second", layer.GetValues()[1].GetStringValue()) + }) + t.Run("empty layer", func(t *testing.T) { + layer := BuildLayer("empty", nil, map[string]uint32{}, map[any]uint32{}) + + assert.Equal(t, "empty", layer.GetName()) + assert.Empty(t, layer.GetKeys()) + assert.Empty(t, layer.GetValues()) + assert.Empty(t, layer.GetFeatures()) + }) } -// assertMVTLayer checks that tl contains a single layer named layerName, with the -// given keys, and one feature per fid (in that order) whose geometry matches -// wantGeoms[fid] and whose tags, resolved through the layer's Keys/Values -// dictionaries, match wantAttrs[fid]. Resolving through the dictionaries makes the -// comparison independent of the (unspecified) order BuildValueDictionary assigns. -func assertMVTLayer(t *testing.T, tl vectorTile.Tile, layerName string, keys []string, fids []uint64, wantGeoms map[uint64]tile.EncodedGeometry, wantAttrs map[uint64]map[string]any) { - t.Helper() - if len(tl.GetLayers()) != 1 { - t.Fatalf("expected 1 layer, got %d", len(tl.GetLayers())) - } - l := tl.GetLayers()[0] - if l.Name == nil || *l.Name != layerName { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - t.Errorf("layer name = %v, want %q", l.Name, layerName) - } - if l.Version == nil || *l.Version != 2 { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - t.Errorf("layer version = %v, want 2", l.Version) - } - if l.Extent == nil || *l.Extent != 4096 { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - t.Errorf("layer extent = %v, want 4096", l.Extent) - } - if !reflect.DeepEqual(l.GetKeys(), keys) { - t.Errorf("layer keys = %v, want %v", l.GetKeys(), keys) - } - if len(l.GetFeatures()) != len(fids) { - t.Fatalf("expected %d features, got %d", len(fids), len(l.GetFeatures())) - } - - for i, fid := range fids { - feat := l.GetFeatures()[i] - if feat.Id == nil || *feat.Id != fid { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - t.Errorf("feature %d id = %v, want %d", i, feat.Id, fid) - continue - } - wantGeom := wantGeoms[fid] - if feat.Type == nil || *feat.Type != vectorTile.Tile_GeomType(wantGeom.GeometryType) { //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - t.Errorf("feature %d (fid %d) type = %v, want %v", i, fid, feat.Type, wantGeom.GeometryType) +func TestBuildMVTTile(t *testing.T) { + t.Run("happy path roundtrips features, tags and attribute values", func(t *testing.T) { + keyIndex := BuildKeyDictionary([]string{"name"}) + attributes := InternalAttributeTable{ + 1: {"name": "foo"}, + 2: {"name": "bar"}, } - if !reflect.DeepEqual(feat.GetGeometry(), wantGeom.Encoding) { - t.Errorf("feature %d (fid %d) geometry = %v, want %v", i, fid, feat.GetGeometry(), wantGeom.Encoding) + encFeatRows := []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + {FeatureID: 2, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, } - gotAttrs := resolveTags(t, feat.GetTags(), l.GetKeys(), l.GetValues()) - if !reflect.DeepEqual(gotAttrs, wantAttrs[fid]) { - t.Errorf("feature %d (fid %d) attrs = %v, want %v", i, fid, gotAttrs, wantAttrs[fid]) + data, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) + require.NoError(t, err) + require.NotEmpty(t, data) + + got := &vectorTile.Tile{} + require.NoError(t, oldproto.Unmarshal(data, got)) + require.Len(t, got.GetLayers(), 1) + + layer := got.GetLayers()[0] + assert.Equal(t, "mylayer", layer.GetName()) + require.Len(t, layer.GetFeatures(), 2) + + // Collect the string values assigned to each feature via its tags, + // independent of feature/value ordering. + gotNames := make([]string, 0, 2) + for _, feature := range layer.GetFeatures() { + require.Len(t, feature.GetTags(), 2) + valueIdx := feature.GetTags()[1] + gotNames = append(gotNames, layer.GetValues()[valueIdx].GetStringValue()) } - } -} - -// resolveTags decodes a feature's (key index, value index) tag pairs back into a -// plain map[string]any, using the layer's key/value dictionaries. -func resolveTags(t *testing.T, tags []uint32, keys []string, values []*vectorTile.Tile_Value) map[string]any { - t.Helper() - attrs := make(map[string]any, len(tags)/2) - for i := 0; i+1 < len(tags); i += 2 { - kidx, vidx := tags[i], tags[i+1] - if int(kidx) >= len(keys) || int(vidx) >= len(values) { - t.Fatalf("tag pair (%d, %d) out of range (keys=%d, values=%d)", kidx, vidx, len(keys), len(values)) + sort.Strings(gotNames) + assert.Equal(t, []string{"bar", "foo"}, gotNames) + }) + + t.Run("error building a feature's tags propagates", func(t *testing.T) { + keyIndex := map[string]uint32{"name": 0} + attributes := InternalAttributeTable{ + 1: {"unknown-key": "foo"}, } - attrs[keys[kidx]] = decodeGSValue(values[vidx]) - } - return attrs -} - -// decodeGSValue extracts the single set field of a GSTileValue as a plain Go value. -func decodeGSValue(v *vectorTile.Tile_Value) any { - switch { - case v.StringValue != nil: - return *v.StringValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.IntValue != nil: - return *v.IntValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.SintValue != nil: - return *v.SintValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.UintValue != nil: - return *v.UintValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.FloatValue != nil: - return *v.FloatValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.DoubleValue != nil: - return *v.DoubleValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - case v.BoolValue != nil: - return *v.BoolValue //nolint:protogetter // GSTile* are hand-copied plain structs with no generated getters - default: - return nil - } -} - -func equalStrings(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false + encFeatRows := []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{}}, } - } - return true + + _, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) + require.Error(t, err) + }) } From 2c8336c742bd7f49d22a0cdee426a4a7bdafbfc3 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 27 Jul 2026 10:30:58 +0200 Subject: [PATCH 20/25] fix: encoded table creation with flag --- main.go | 8 ++++++-- processing/gpkg/gpkg.go | 8 ++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 7ec8bfe..3829395 100644 --- a/main.go +++ b/main.go @@ -169,7 +169,7 @@ func main() { EncodeTiles: c.Bool(ENCODETILES), } for _, tmID := range tileMatrixIDs { - gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize) + gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize, c.Bool(ENCODETILES)) defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func } @@ -200,6 +200,9 @@ func main() { } log.Println("=== done snapping ===") + if(c.Bool(ENCODETILES)) { + log.Println("Generated encoded tables.") + } return nil }, }, @@ -247,7 +250,7 @@ func validateTileMatrixSet(tms tms20.TileMatrixSet, tileMatrixIDs []tms20.TMID) return pointindex.IsQuadTree(tms) } -func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int) *gpkg.TargetGeopackage { +func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int, encodeTiles bool) *gpkg.TargetGeopackage { targetPath := fmt.Sprintf(targetPathFmt, tmID) if overwrite { err := os.Remove(targetPath) @@ -259,6 +262,7 @@ func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int } target := gpkg.TargetGeopackage{} target.Init(targetPath, pagesize) + target.EncodeTiles = encodeTiles return &target } diff --git a/processing/gpkg/gpkg.go b/processing/gpkg/gpkg.go index fd1cfa1..ce5f93d 100644 --- a/processing/gpkg/gpkg.go +++ b/processing/gpkg/gpkg.go @@ -176,7 +176,7 @@ type TargetGeopackage struct { Table Table pagesize int handle *gpkg.Handle - encodeTiles bool + EncodeTiles bool } func (target *TargetGeopackage) Init(file string, pagesize int) { @@ -200,7 +200,7 @@ func (target *TargetGeopackage) CreateTables(tables []Table) error { return err } - if target.encodeTiles { + if target.EncodeTiles { err = buildEncodedTable(target.handle, table) if err != nil { return err @@ -217,7 +217,7 @@ func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.Featu feature, hasMore := <-inFeatures if !hasMore { target.writeFeatures(features) - if target.encodeTiles { + if target.EncodeTiles { target.writeEncodedFeatures(features) } break @@ -226,7 +226,7 @@ func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.Featu if len(features)%target.pagesize == 0 { target.writeFeatures(features) - if target.encodeTiles { + if target.EncodeTiles { target.writeEncodedFeatures(features) } features = nil From f1476429e6f8ba67ed83a7448c49b1c55ec4064f Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 27 Jul 2026 11:17:52 +0200 Subject: [PATCH 21/25] fix: incorrect index in tile selection --- main.go | 3 --- pointindex/pointindex.go | 8 ++++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 3829395..c874ac9 100644 --- a/main.go +++ b/main.go @@ -200,9 +200,6 @@ func main() { } log.Println("=== done snapping ===") - if(c.Bool(ENCODETILES)) { - log.Println("Generated encoded tables.") - } return nil }, }, diff --git a/pointindex/pointindex.go b/pointindex/pointindex.go index 97e67d1..4760c95 100644 --- a/pointindex/pointindex.go +++ b/pointindex/pointindex.go @@ -135,16 +135,16 @@ func (ix *PointIndex) GetPrimitiveQBBox(l Level) []Quadrant { maxY = max(y, maxY) } - quadrantSlice := make([]Quadrant, (maxY-minY)*(maxX-minX)) - for i := range maxX - minX { - for j := range maxY - minY { + quadrantSlice := make([]Quadrant, (maxY-minY+1)*(maxX-minX+1)) + for i := range maxX - minX + 1 { + for j := range maxY - minY + 1 { extent, centroid := ix.getQuadrantExtentAndCentroid(l, minX+i, minY+j, ix.intExtent) newQuadrant := Quadrant{ z: morton.MustToZ(minX+i, minY+j), intExtent: extent, intCentroid: centroid, } - quadrantSlice[i*(maxY-minY)+j] = newQuadrant + quadrantSlice[i*(maxY-minY+1)+j] = newQuadrant } } return quadrantSlice From 67dd1f090e05f8e4585bc1172a74ed16790bb3da Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Mon, 27 Jul 2026 15:28:44 +0200 Subject: [PATCH 22/25] refactor: separate snap and mvt commands --- main.go | 10 +- processing/gpkg/attributes.go | 132 ------------ processing/gpkg/encoded.go | 144 ------------- processing/gpkg/gpkg.go | 313 +++-------------------------- processing/gpkg/gpkgmvt.go | 261 ++++++++++++++++++++++++ processing/gpkg/gpkgsnap.go | 366 ++++++++++++++++++++++++++++++++++ processing/gpkg/mvttile.go | 106 ---------- processing/interface.go | 22 +- processing/processing.go | 11 +- processing/processingmvt.go | 44 ++++ snap/snap.go | 2 - 11 files changed, 726 insertions(+), 685 deletions(-) delete mode 100644 processing/gpkg/attributes.go delete mode 100644 processing/gpkg/encoded.go create mode 100644 processing/gpkg/gpkgmvt.go create mode 100644 processing/gpkg/gpkgsnap.go delete mode 100644 processing/gpkg/mvttile.go create mode 100644 processing/processingmvt.go diff --git a/main.go b/main.go index c874ac9..5cfa647 100644 --- a/main.go +++ b/main.go @@ -273,26 +273,28 @@ func injectSuffixIntoPath(p string) string { func processBySnapping(source processing.Source, targets map[tms20.TMID]processing.Target, tileMatrixSet tms20.TileMatrixSet, snapConfig snap.Config) { processing.ProcessFeatures(source, targets, func(p geom.Polygon, tmIDs []tms20.TMID) map[tms20.TMID]processing.SnapResult { return snap.SnapPolygon(p, tileMatrixSet, tmIDs, snapConfig) - }) + }, snapConfig.EncodeTiles) } // runBuildMVTTiles builds and writes MVT tiles for every table in the given source // GeoPackage (a GeoPackage previously produced with --encodetiles), using -// gpkg.SourceGeopackage.ProcessTilesToMVT for each table. +// processing.BuildAndWriteMVTTiles for each table. func runBuildMVTTiles(sourcePath, outDir string) error { if _, err := os.Stat(sourcePath); os.IsNotExist(err) { return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath) } - source := gpkg.SourceGeopackage{} + source := gpkg.MVTSourceGeopackage{} source.Init(sourcePath) defer source.Close() + mvtTarget := gpkg.MVTFileTarget{OutDir: outDir} + tables := source.GetTableInfo() for _, table := range tables { source.Table = table log.Printf(" building MVT tiles for %s", table.Name) - if err := source.ProcessTilesToMVT(outDir); err != nil { + if err := processing.BuildAndWriteMVTTiles(&source, &mvtTarget, table.Name); err != nil { return fmt.Errorf("building MVT tiles for %s: %w", table.Name, err) } } diff --git a/processing/gpkg/attributes.go b/processing/gpkg/attributes.go deleted file mode 100644 index 25fafef..0000000 --- a/processing/gpkg/attributes.go +++ /dev/null @@ -1,132 +0,0 @@ -package gpkg - -import ( - "fmt" - "strings" - "time" - - "github.com/pdok/texel/tile" -) - -// Batch value for reading features from the geopackage table -const maxSQLiteBatchParams = 500 - -// selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. -// The fid is assumed to be the table's first column. -func (t Table) selectByFIDsSQL(batchSize int) string { - var csql []string //nolint:prealloc - for _, c := range t.columns { - csql = append(csql, c.name) - } - placeholders := strings.TrimSuffix(strings.Repeat("?,", batchSize), ",") - fidColumn := t.columns[0].name - return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` -} - -// GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) -// for the given feature IDs, batching the query to respect SQLite's parameter limits. -// The fid is assumed to be the table's first column. -func (source SourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) { - attributes := make(tile.InternalAttributeTable, len(featureIDs)) - fidColumn := source.Table.columns[0].name - - for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { - batch := featureIDs[start:min(start+maxSQLiteBatchParams, len(featureIDs))] - - batchAsAny := make([]any, len(batch)) - for i, fid := range batch { - batchAsAny[i] = fid - } - - if err := source.queryAttributesBatch(fidColumn, batchAsAny, attributes); err != nil { - return nil, err - } - } - - return attributes, nil -} - -func (source SourceGeopackage) getColNames() []string { - cols := source.Table.columns - colNames := make([]string, len(cols)) - for i, col := range cols { - colNames[i] = col.name - } - return colNames -} - -// Read from source.Table, store rows as attributes[fid][columnName] = value for all fids -func (source SourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes tile.InternalAttributeTable) error { - // Read relevant rows - rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) - if err != nil { - return fmt.Errorf("querying attributes for %d feature(s): %w", len(fids), err) - } - defer rows.Close() - - cols := source.getColNames() - - // Process rows - for rows.Next() { - // Read data with helper vars - vals := make([]any, len(cols)) - valPtrs := make([]any, len(cols)) - for i := range cols { - valPtrs[i] = &vals[i] - } - if err := rows.Scan(valPtrs...); err != nil { - return fmt.Errorf("scanning attribute row: %w", err) - } - - // Interpret data - row := make(map[string]any, len(cols)) - var fid int64 - var fidFound bool - for i, colName := range cols { - switch colName { - case fidColumn: - // Read fid - id, ok := vals[i].(int64) - if !ok { - return fmt.Errorf("expected int64 fid for column %s, got %T", colName, vals[i]) - } - fid, fidFound = id, true - case source.Table.gcolumn: - // Skip geometry column - default: - // Interpret data - v, ok := cleanData(vals[i]) - if !ok { - return fmt.Errorf("unexpected type for sqlite column data: %v: %T", colName, v) - } - row[colName] = v - } - } - if !fidFound { - return fmt.Errorf("row did not contain fid column %s", fidColumn) - } - attributes[fid] = row - } - return rows.Err() -} - -func cleanData(v any) (any, bool) { - switch v := v.(type) { - case []uint8: - asBytes := make([]byte, len(v)) - copy(asBytes, v) - return v, true - case int64: - return v, true - case float64: - return v, true - case time.Time: - return v.Format(time.RFC3339), true - case string: - return v, true - case nil: - return nil, true - default: - return nil, false - } -} diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go deleted file mode 100644 index 178c87e..0000000 --- a/processing/gpkg/encoded.go +++ /dev/null @@ -1,144 +0,0 @@ -package gpkg - -import ( - "encoding/binary" - "fmt" - "log" - - "github.com/go-spatial/geom/encoding/gpkg" - "github.com/pdok/texel/processing" - "github.com/pdok/texel/tile" -) - -func (t Table) EncodedName() string { - return t.Name + "_encoded" -} - -// List all columns except fid and geometry column -// Assumptions: fid column has index 0 -func (t Table) AttributeColumnNames() []string { - names := make([]string, 0, len(t.columns)) - for i, c := range t.columns { - if i == 0 || c.name == t.gcolumn { - continue // skip the fid and geometry columns, they are not attributes - } - names = append(names, c.name) - } - return names -} - -func (t Table) insertSQLEncoded() string { - return `INSERT INTO "` + t.EncodedName() + `"` + - ` (tile_x, tile_y, feature_id, geometry_type, data)` + - ` VALUES (?, ?, ?, ?, ?)` -} - -// selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile -func (t Table) selectEncodedSQL() string { - return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?` -} - -func serializeToBytes(intslice []uint32) []byte { - bytes := make([]byte, len(intslice)*4) - - for i, x := range intslice { - binary.LittleEndian.PutUint32(bytes[i*4:], x) - } - - return bytes -} - -// deserializeFromBytes is the inverse of serializeToBytes: it turns a BLOB of -// little-endian uint32s back into the encoded geometry command/coordinate stream. -func deserializeFromBytes(data []byte) []uint32 { - intslice := make([]uint32, len(data)/4) - for i := range intslice { - intslice[i] = binary.LittleEndian.Uint32(data[i*4:]) - } - return intslice -} - -// GetFeaturesForTile returns every encoded feature stored for the given tile, -// i.e. the equivalent of the tile-features table's rows for (tileX, tileY). -func (source SourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { - rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) - if err != nil { - return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) - } - defer rows.Close() - - var features []tile.EncodedFeatureRow - for rows.Next() { - var featureID int64 - var geometryType int32 - var data []byte - if err := rows.Scan(&featureID, &geometryType, &data); err != nil { - return nil, fmt.Errorf("scanning encoded feature row for tile (%d,%d): %w", tileX, tileY, err) - } - features = append(features, tile.EncodedFeatureRow{ - FeatureID: featureID, - Geom: tile.EncodedGeometry{ - Encoding: deserializeFromBytes(data), - GeometryType: geometryType, - XTile: tileX, - YTile: tileY, - }, - }) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating encoded feature rows for tile (%d,%d): %w", tileX, tileY, err) - } - return features, nil -} - -func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.FeatureForTileMatrix) { - tx, err := target.handle.Begin() - if err != nil { - log.Fatalf("Could not start a transaction: %s", err) - } - - stmt, err := tx.Prepare(target.Table.insertSQLEncoded()) - if err != nil { - log.Fatalf("Could not prepare a statement: %s", err) - } - - for _, ef := range encFeature { - cols := ef.Columns() - if len(cols) == 0 { - log.Fatalf("Processing feature without data") - } - fid := cols[0] - for _, eg := range ef.EncodedGeoms() { - bytes := serializeToBytes(eg.Encoding) - _, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes) - if err != nil { - log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) - } - } - - } - - stmt.Close() - _ = tx.Commit() -} - -func buildEncodedTable(h *gpkg.Handle, t Table) error { - db := h.DB - - query := fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s ( - id INTEGER PRIMARY KEY, - tile_x INTEGER NOT NULL, - tile_y INTEGER NOT NULL, - feature_id INTEGER NOT NULL, - geometry_type INTEGER NOT NULL, - data BLOB - ) - `, t.EncodedName()) - _, err := db.Exec(query) - if err != nil { - log.Println("error adding encoded table in target GeoPackage:", err) - return err - } - return nil -} diff --git a/processing/gpkg/gpkg.go b/processing/gpkg/gpkg.go index ce5f93d..b6af9c0 100644 --- a/processing/gpkg/gpkg.go +++ b/processing/gpkg/gpkg.go @@ -1,29 +1,16 @@ package gpkg +// This file defines general geopackage-related infrastructure shared by both +// the `snap` and `mvt` flows. + import ( "fmt" "log" "strings" - "time" - "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/gpkg" - "github.com/pdok/texel/processing" ) -type featureGPKG struct { - columns []any - geometry geom.Geometry -} - -func (f featureGPKG) Columns() []any { - return f.columns -} - -func (f featureGPKG) Geometry() geom.Geometry { - return f.geometry -} - type column struct { cid int name string @@ -33,6 +20,7 @@ type column struct { pk int } +// Abstraction of geometry table in a gpkg type Table struct { Name string columns []column @@ -41,7 +29,11 @@ type Table struct { srs gpkg.SpatialReferenceSystem } -// geometryTypeFromString returns the numeric value of a gometry string +// Name of associated table of a geometry table. +func (t Table) EncodedName() string { + return t.Name + "_encoded" +} + func geometryTypeFromString(geometrytype string) gpkg.GeometryType { switch strings.ToUpper(geometrytype) { case "GEOMETRY": @@ -65,89 +57,21 @@ func geometryTypeFromString(geometrytype string) gpkg.GeometryType { } } -//nolint:recvcheck // TODO double check this -type SourceGeopackage struct { - Table Table - handle *gpkg.Handle -} +// Custom wrapper for gpkg.Handle. To be embedded in other structs +type geopackageHandle struct{ handle *gpkg.Handle } -func (source *SourceGeopackage) Init(file string) { - source.handle = openGeopackage(file) +func (gH *geopackageHandle) Init(file string) { + gH.handle = openGeopackage(file) } -func (source SourceGeopackage) Close() { - source.handle.Close() +func (gH *geopackageHandle) Close() { + gH.handle.Close() } -//nolint:funlen,cyclop -func (source SourceGeopackage) ReadFeatures(features chan<- processing.Feature) { - rows, err := source.handle.Query(source.Table.selectSQL()) - if err != nil { - log.Fatalf("err during closing rows: %s", err) - } - - cols, err := rows.Columns() - if err != nil { - log.Fatalf("error reading the columns: %s", err) - } - - for rows.Next() { - vals := make([]any, len(cols)) - valPtrs := make([]any, len(cols)) - for i := range cols { - valPtrs[i] = &vals[i] - } - - if err = rows.Scan(valPtrs...); err != nil { - log.Fatalf("err reading row values: %v", err) - } - var f featureGPKG - var c []any - - for i, colName := range cols { - switch colName { - case source.Table.gcolumn: - wkbgeom, err := gpkg.DecodeGeometry(vals[i].([]byte)) - if err != nil { - log.Fatalf("error decoding the geometry: %s", err) - } - f.geometry = wkbgeom.Geometry - default: - switch v := vals[i].(type) { - case []uint8: - asBytes := make([]byte, len(v)) - copy(asBytes, v) - c = append(c, string(asBytes)) - case int64: - c = append(c, v) - case float64: - c = append(c, v) - case time.Time: - c = append(c, v.Format(time.RFC3339)) - case string: - c = append(c, v) - case nil: - c = append(c, v) - default: - log.Fatalf("unexpected type for sqlite column data: %v: %T", cols[i], v) - } - } - f.columns = c - } - ff := &f - features <- ff - } - err = rows.Err() - if err != nil { - log.Fatal(err) - } - close(features) - defer rows.Close() -} - -func (source SourceGeopackage) GetTableInfo() []Table { +// Generate []Table slice for looping over. +func (gH *geopackageHandle) GetTableInfo() []Table { query := `SELECT table_name, column_name, geometry_type_name, srs_id FROM gpkg_geometry_columns;` - rows, err := source.handle.Query(query) + rows, err := gH.handle.Query(query) if err != nil { log.Fatalf("error during closing rows: %v - %v", query, err) } @@ -162,9 +86,9 @@ func (source SourceGeopackage) GetTableInfo() []Table { log.Fatalf("error retrieving the source table information: %s", err) } - t.columns = getTableColumns(source.handle, t.Name) + t.columns = gH.getTableColumns(t.Name) t.gtype = geometryTypeFromString(gtype) - t.srs = getSpatialReferenceSystem(source.handle, srsID) + t.srs = gH.getSpatialReferenceSystem(srsID) tables = append(tables, t) } @@ -172,120 +96,6 @@ func (source SourceGeopackage) GetTableInfo() []Table { return tables } -type TargetGeopackage struct { - Table Table - pagesize int - handle *gpkg.Handle - EncodeTiles bool -} - -func (target *TargetGeopackage) Init(file string, pagesize int) { - target.pagesize = pagesize - target.handle = openGeopackage(file) -} - -func (target *TargetGeopackage) Close() { - target.handle.Close() -} - -func (target *TargetGeopackage) CreateTables(tables []Table) error { - for _, table := range tables { - err := target.handle.UpdateSRS(table.srs) - if err != nil { - return err - } - - err = buildTable(target.handle, table) - if err != nil { - return err - } - - if target.EncodeTiles { - err = buildEncodedTable(target.handle, table) - if err != nil { - return err - } - } - } - return nil -} - -func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.FeatureForTileMatrix) { - var features []processing.FeatureForTileMatrix - - for { - feature, hasMore := <-inFeatures - if !hasMore { - target.writeFeatures(features) - if target.EncodeTiles { - target.writeEncodedFeatures(features) - } - break - } - features = append(features, feature) - - if len(features)%target.pagesize == 0 { - target.writeFeatures(features) - if target.EncodeTiles { - target.writeEncodedFeatures(features) - } - features = nil - } - } -} - -func (target *TargetGeopackage) writeFeatures(features []processing.FeatureForTileMatrix) { - tx, err := target.handle.Begin() - if err != nil { - log.Fatalf("Could not start a transaction: %s", err) - } - - stmt, err := tx.Prepare(target.Table.insertSQL()) - if err != nil { - log.Fatalf("Could not prepare a statement: %s", err) - } - - var ext *geom.Extent - - for _, f := range features { - //nolint:gosec // G115 - sb, err := gpkg.NewBinary(int32(target.Table.srs.ID), f.Geometry()) - if err != nil { - log.Fatalf("Could not create a binary geometry: %s", err) - } - - data := f.Columns() - data = append(data, sb) - - _, err = stmt.Exec(data...) - if err != nil { - var fid any = "unknown" - if len(data) > 0 { - fid = data[0] - } - log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) - } - - if ext == nil { - ext, err = geom.NewExtentFromGeometry(f.Geometry()) - if err != nil { - ext = nil - log.Println("Failed to create new extent:", err) - continue - } - } else { - _ = ext.AddGeometry(f.Geometry()) - } - } - stmt.Close() - _ = tx.Commit() - - err = target.handle.UpdateGeometryExtent(target.Table.Name, ext) - if err != nil { - log.Fatalln("Failed to update new extent:", err) - } -} - func openGeopackage(file string) *gpkg.Handle { handle, err := gpkg.Open(file) if err != nil { @@ -294,60 +104,12 @@ func openGeopackage(file string) *gpkg.Handle { return handle } -// createSQL creates a CREATE statement on the given table and column information -// used for creating feature tables in the target Geopackage -func (t Table) createSQL() string { - create := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%v"`, t.Name) - var columnparts []string //nolint:prealloc - for _, column := range t.columns { - columnpart := column.name + ` ` + column.ctype - if column.notnull == 1 { - columnpart += ` NOT NULL` - } - if column.pk == 1 { - columnpart += ` PRIMARY KEY` - } - - columnparts = append(columnparts, columnpart) - } - - query := create + `(` + strings.Join(columnparts, `, `) + `);` - return query -} - -// selectSQL build a SELECT statement based on the table and columns -// used for reading the source features -func (t Table) selectSQL() string { - var csql []string //nolint:prealloc - for _, c := range t.columns { - csql = append(csql, c.name) - } - query := `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `";` - return query -} - -// insertSQL used for writing the features -// build the INSERT statement based on the table and columns -func (t Table) insertSQL() string { - var csql, vsql []string - for _, c := range t.columns { - if c.name != t.gcolumn { - csql = append(csql, c.name) - vsql = append(vsql, `?`) - } - } - csql = append(csql, t.gcolumn) - vsql = append(vsql, `?`) - query := `INSERT INTO "` + t.Name + `"(` + strings.Join(csql, `,`) + `) VALUES(` + strings.Join(vsql, `,`) + `)` - return query -} - // getSpatialReferenceSystem extracts this based on the given SRS id -func getSpatialReferenceSystem(h *gpkg.Handle, id int) gpkg.SpatialReferenceSystem { +func (gH *geopackageHandle) getSpatialReferenceSystem(id int) gpkg.SpatialReferenceSystem { var srs gpkg.SpatialReferenceSystem query := `SELECT srs_name, srs_id, organization, organization_coordsys_id, definition, description FROM gpkg_spatial_ref_sys WHERE srs_id = %v;` - row := h.QueryRow(fmt.Sprintf(query, id)) + row := gH.handle.QueryRow(fmt.Sprintf(query, id)) var description *string _ = row.Scan(&srs.Name, &srs.ID, &srs.Organization, &srs.OrganizationCoordsysID, &srs.Definition, &description) if description != nil { @@ -358,10 +120,10 @@ func getSpatialReferenceSystem(h *gpkg.Handle, id int) gpkg.SpatialReferenceSyst } // getTableColumns collects the column information of a given table -func getTableColumns(h *gpkg.Handle, table string) []column { +func (gH *geopackageHandle) getTableColumns(table string) []column { var columns []column query := `PRAGMA table_info('%v');` - rows, err := h.Query(fmt.Sprintf(query, table)) + rows, err := gH.handle.Query(fmt.Sprintf(query, table)) if err != nil { log.Fatalf("err during closing rows: %v - %v", query, err) } @@ -377,30 +139,3 @@ func getTableColumns(h *gpkg.Handle, table string) []column { defer rows.Close() return columns } - -// buildTable creates a given destination table with the necessary gpkg_ information -func buildTable(h *gpkg.Handle, t Table) error { - query := t.createSQL() - _, err := h.Exec(query) - if err != nil { - log.Fatalf("error building table in target GeoPackage: %s", err) - } - - err = h.AddGeometryTable(gpkg.TableDescription{ - Name: t.Name, - ShortName: t.Name, - Description: t.Name, - GeometryField: t.gcolumn, - GeometryType: t.gtype, - //nolint:gosec // G115 - SRS: int32(t.srs.ID), - // - Z: gpkg.Prohibited, - M: gpkg.Prohibited, - }) - if err != nil { - log.Println("error adding geometry table in target GeoPackage:", err) - return err - } - return nil -} diff --git a/processing/gpkg/gpkgmvt.go b/processing/gpkg/gpkgmvt.go new file mode 100644 index 0000000..ad5b662 --- /dev/null +++ b/processing/gpkg/gpkgmvt.go @@ -0,0 +1,261 @@ +package gpkg + +// This file contains the geopackage functionality used for `texel mvt`. +// The exported functionality is precisely given by MVTSource and MVTTarget. +// We reuse SourceGeopackage as MVTSource. + +import ( + "encoding/binary" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/pdok/texel/processing" + "github.com/pdok/texel/tile" +) + +// Batch value for reading features from the geopackage table +const maxSQLiteBatchParams = 500 + +type MVTSourceGeopackage struct { + Table Table + geopackageHandle +} + +// GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) +// for the given feature IDs, batching the query to respect SQLite's parameter limits. +// The fid is assumed to be the table's first column. +func (source MVTSourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) { + attributes := make(tile.InternalAttributeTable, len(featureIDs)) + fidColumn := source.Table.columns[0].name + + for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { + batch := featureIDs[start:min(start+maxSQLiteBatchParams, len(featureIDs))] + + batchAsAny := make([]any, len(batch)) + for i, fid := range batch { + batchAsAny[i] = fid + } + + if err := source.queryAttributesBatch(fidColumn, batchAsAny, attributes); err != nil { + return nil, err + } + } + + return attributes, nil +} + +// selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. +// The fid is assumed to be the table's first column. +func (t Table) selectByFIDsSQL(batchSize int) string { + var csql []string //nolint:prealloc + for _, c := range t.columns { + csql = append(csql, c.name) + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", batchSize), ",") + fidColumn := t.columns[0].name + return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` +} + +func cleanData(v any) (any, bool) { + switch v := v.(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + return v, true + case int64: + return v, true + case float64: + return v, true + case time.Time: + return v.Format(time.RFC3339), true + case string: + return v, true + case nil: + return nil, true + default: + return nil, false + } +} + +// attributeColumnNames lists all columns except fid and geometry column. +// Assumptions: fid column has index 0 +func (t Table) attributeColumnNames() []string { + names := make([]string, 0, len(t.columns)) + for i, c := range t.columns { + if i == 0 || c.name == t.gcolumn { + continue // skip the fid and geometry columns, they are not attributes + } + names = append(names, c.name) + } + return names +} + +// AttributeColumnNames satisfies processing.MVTSource by delegating to the +// current Table. +func (source MVTSourceGeopackage) AttributeColumnNames() []string { + return source.Table.attributeColumnNames() +} + +// selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile +func (t Table) selectEncodedSQL() string { + return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?` +} + +// selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at +// least one encoded feature. +func (t Table) selectDistinctTilesSQL() string { + return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y` +} + +// GetFeaturesForTile returns every encoded feature stored for the given tile, +// i.e. the equivalent of the tile-features table's rows for (tileX, tileY). +func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { + rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) + if err != nil { + return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) + } + defer rows.Close() + + var features []tile.EncodedFeatureRow + for rows.Next() { + var featureID int64 + var geometryType int32 + var data []byte + if err := rows.Scan(&featureID, &geometryType, &data); err != nil { + return nil, fmt.Errorf("scanning encoded feature row for tile (%d,%d): %w", tileX, tileY, err) + } + features = append(features, tile.EncodedFeatureRow{ + FeatureID: featureID, + Geom: tile.EncodedGeometry{ + Encoding: deserializeFromBytes(data), + GeometryType: geometryType, + XTile: tileX, + YTile: tileY, + }, + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating encoded feature rows for tile (%d,%d): %w", tileX, tileY, err) + } + return features, nil +} + +// deserializeFromBytes is the inverse of serializeToBytes: it turns a BLOB of +// little-endian uint32s back into the encoded geometry command/coordinate stream. +func deserializeFromBytes(data []byte) []uint32 { + intslice := make([]uint32, len(data)/4) + for i := range intslice { + intslice[i] = binary.LittleEndian.Uint32(data[i*4:]) + } + return intslice +} + +// ListTiles returns every distinct tile present in the "
_encoded" table. +func (source MVTSourceGeopackage) ListTiles() ([]processing.TileCoord, error) { + rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL()) + if err != nil { + return nil, fmt.Errorf("listing tiles: %w", err) + } + defer rows.Close() + + var tiles []processing.TileCoord + for rows.Next() { + var x, y int64 + if err := rows.Scan(&x, &y); err != nil { + return nil, fmt.Errorf("scanning tile row: %w", err) + } + tiles = append(tiles, processing.TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating tile rows: %w", err) + } + return tiles, nil +} + +// Read from source.Table, store rows as attributes[fid][columnName] = value for all fids +func (source MVTSourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes tile.InternalAttributeTable) error { + // Read relevant rows + rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) + if err != nil { + return fmt.Errorf("querying attributes for %d feature(s): %w", len(fids), err) + } + defer rows.Close() + + cols := source.getColNames() + + // Process rows + for rows.Next() { + // Read data with helper vars + vals := make([]any, len(cols)) + valPtrs := make([]any, len(cols)) + for i := range cols { + valPtrs[i] = &vals[i] + } + if err := rows.Scan(valPtrs...); err != nil { + return fmt.Errorf("scanning attribute row: %w", err) + } + + // Interpret data + row := make(map[string]any, len(cols)) + var fid int64 + var fidFound bool + for i, colName := range cols { + switch colName { + case fidColumn: + // Read fid + id, ok := vals[i].(int64) + if !ok { + return fmt.Errorf("expected int64 fid for column %s, got %T", colName, vals[i]) + } + fid, fidFound = id, true + case source.Table.gcolumn: + // Skip geometry column + default: + // Interpret data + v, ok := cleanData(vals[i]) + if !ok { + return fmt.Errorf("unexpected type for sqlite column data: %v: %T", colName, v) + } + row[colName] = v + } + } + if !fidFound { + return fmt.Errorf("row did not contain fid column %s", fidColumn) + } + attributes[fid] = row + } + return rows.Err() +} + +func (source MVTSourceGeopackage) getColNames() []string { + cols := source.Table.columns + colNames := make([]string, len(cols)) + for i, col := range cols { + colNames[i] = col.name + } + return colNames +} + +// MVTFileTarget writes built MVT tiles to //.mvt. +type MVTFileTarget struct { + OutDir string +} + +// WriteTile writes one tile's serialized bytes to //.mvt, +// creating directories as needed. +func (t *MVTFileTarget) WriteTile(x, y uint, data []byte) error { + dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(x), 10)) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating tile directory %s: %w", dir, err) + } + + path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".mvt") + if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions + return fmt.Errorf("writing tile file %s: %w", path, err) + } + return nil +} diff --git a/processing/gpkg/gpkgsnap.go b/processing/gpkg/gpkgsnap.go new file mode 100644 index 0000000..8fe6181 --- /dev/null +++ b/processing/gpkg/gpkgsnap.go @@ -0,0 +1,366 @@ +package gpkg + +// Functionality regarding geopackages for `texel snap`. Exported are: +// SourceGeopackage.ReadFeatures +// TargetGeopackage.CreateTables +// TargetGeopackage.WriteFeatures + +import ( + "encoding/binary" + "fmt" + "log" + "strings" + "time" + + "github.com/go-spatial/geom" + "github.com/go-spatial/geom/encoding/gpkg" + "github.com/pdok/texel/processing" +) + +// Abstraction of a source. Table is a field that represents the table that is +// currently being processed. +type SourceGeopackage struct { + Table Table + geopackageHandle +} + +// Abstraction of target geopackage. Table represents the table that is +// currently being processed. +type TargetGeopackage struct { + Table Table + pagesize int + geopackageHandle + EncodeTiles bool +} + +// Overrides geopackageHandle.Init +func (target *TargetGeopackage) Init(file string, pagesize int) { + target.pagesize = pagesize + target.geopackageHandle.Init(file) +} + +type featureGPKG struct { + columns []any + geometry geom.Geometry +} + +func (f featureGPKG) Columns() []any { + return f.columns +} + +func (f featureGPKG) Geometry() geom.Geometry { + return f.geometry +} + +//nolint:funlen,cyclop +func (source SourceGeopackage) ReadFeatures(features chan<- processing.Feature) { + rows, err := source.handle.Query(source.Table.selectSQL()) + if err != nil { + log.Fatalf("err during closing rows: %s", err) + } + + cols, err := rows.Columns() + if err != nil { + log.Fatalf("error reading the columns: %s", err) + } + + for rows.Next() { + vals := make([]any, len(cols)) + valPtrs := make([]any, len(cols)) + for i := range cols { + valPtrs[i] = &vals[i] + } + + if err = rows.Scan(valPtrs...); err != nil { + log.Fatalf("err reading row values: %v", err) + } + var f featureGPKG + var c []any + + for i, colName := range cols { + switch colName { + case source.Table.gcolumn: + wkbgeom, err := gpkg.DecodeGeometry(vals[i].([]byte)) + if err != nil { + log.Fatalf("error decoding the geometry: %s", err) + } + f.geometry = wkbgeom.Geometry + default: + switch v := vals[i].(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + c = append(c, string(asBytes)) + case int64: + c = append(c, v) + case float64: + c = append(c, v) + case time.Time: + c = append(c, v.Format(time.RFC3339)) + case string: + c = append(c, v) + case nil: + c = append(c, v) + default: + log.Fatalf("unexpected type for sqlite column data: %v: %T", cols[i], v) + } + } + f.columns = c + } + ff := &f + features <- ff + } + err = rows.Err() + if err != nil { + log.Fatal(err) + } + close(features) + defer rows.Close() +} + +// selectSQL build a SELECT statement based on the table and columns +// used for reading the source features +func (t Table) selectSQL() string { + var csql []string //nolint:prealloc + for _, c := range t.columns { + csql = append(csql, c.name) + } + query := `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `";` + return query +} + +// insertSQL used for writing the features +// build the INSERT statement based on the table and columns +func (t Table) insertSQL() string { + var csql, vsql []string + for _, c := range t.columns { + if c.name != t.gcolumn { + csql = append(csql, c.name) + vsql = append(vsql, `?`) + } + } + csql = append(csql, t.gcolumn) + vsql = append(vsql, `?`) + query := `INSERT INTO "` + t.Name + `"(` + strings.Join(csql, `,`) + `) VALUES(` + strings.Join(vsql, `,`) + `)` + return query +} + +// insertSQLEncoded used for writing the encoded geometries when -enc is on +func (t Table) insertSQLEncoded() string { + return `INSERT INTO "` + t.EncodedName() + `"` + + ` (tile_x, tile_y, feature_id, geometry_type, data)` + + ` VALUES (?, ?, ?, ?, ?)` +} + +func serializeToBytes(intslice []uint32) []byte { + bytes := make([]byte, len(intslice)*4) + + for i, x := range intslice { + binary.LittleEndian.PutUint32(bytes[i*4:], x) + } + + return bytes +} + +// Create tables in target. Defers to buildTable and buildEncodedTable +func (target *TargetGeopackage) CreateTables(tables []Table) error { + for _, table := range tables { + err := target.handle.UpdateSRS(table.srs) + if err != nil { + return err + } + + err = buildTable(target.handle, table) + if err != nil { + return err + } + + if target.EncodeTiles { + err = buildEncodedTable(target.handle, table) + if err != nil { + return err + } + } + } + return nil +} + +// Write the features to geopackage. Defers to writeFeatures and writeEncodedFeatures +func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.FeatureForTileMatrix) { + var features []processing.FeatureForTileMatrix + + for { + feature, hasMore := <-inFeatures + if !hasMore { + target.writeFeatures(features) + if target.EncodeTiles { + target.writeEncodedFeatures(features) + } + break + } + features = append(features, feature) + + if len(features)%target.pagesize == 0 { + target.writeFeatures(features) + if target.EncodeTiles { + target.writeEncodedFeatures(features) + } + features = nil + } + } +} + +// Write snapped features to table +func (target *TargetGeopackage) writeFeatures(features []processing.FeatureForTileMatrix) { + tx, err := target.handle.Begin() + if err != nil { + log.Fatalf("Could not start a transaction: %s", err) + } + + stmt, err := tx.Prepare(target.Table.insertSQL()) + if err != nil { + log.Fatalf("Could not prepare a statement: %s", err) + } + + var ext *geom.Extent + + for _, f := range features { + //nolint:gosec // G115 + sb, err := gpkg.NewBinary(int32(target.Table.srs.ID), f.Geometry()) + if err != nil { + log.Fatalf("Could not create a binary geometry: %s", err) + } + + data := f.Columns() + data = append(data, sb) + + _, err = stmt.Exec(data...) + if err != nil { + var fid any = "unknown" + if len(data) > 0 { + fid = data[0] + } + log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) + } + + if ext == nil { + ext, err = geom.NewExtentFromGeometry(f.Geometry()) + if err != nil { + ext = nil + log.Println("Failed to create new extent:", err) + continue + } + } else { + _ = ext.AddGeometry(f.Geometry()) + } + } + stmt.Close() + _ = tx.Commit() + + err = target.handle.UpdateGeometryExtent(target.Table.Name, ext) + if err != nil { + log.Fatalln("Failed to update new extent:", err) + } +} + +// Write encoded geometries to corresponding tables +func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.FeatureForTileMatrix) { + tx, err := target.handle.Begin() + if err != nil { + log.Fatalf("Could not start a transaction: %s", err) + } + + stmt, err := tx.Prepare(target.Table.insertSQLEncoded()) + if err != nil { + log.Fatalf("Could not prepare a statement: %s", err) + } + + for _, ef := range encFeature { + cols := ef.Columns() + if len(cols) == 0 { + log.Fatalf("Processing feature without data") + } + fid := cols[0] + for _, eg := range ef.EncodedGeoms() { + bytes := serializeToBytes(eg.Encoding) + _, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes) + if err != nil { + log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) + } + } + + } + + stmt.Close() + _ = tx.Commit() +} + +// createSQL creates a CREATE statement on the given table and column information +// used for creating feature tables in the target Geopackage +func (t Table) createSQL() string { + create := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%v"`, t.Name) + var columnparts []string //nolint:prealloc + for _, column := range t.columns { + columnpart := column.name + ` ` + column.ctype + if column.notnull == 1 { + columnpart += ` NOT NULL` + } + if column.pk == 1 { + columnpart += ` PRIMARY KEY` + } + + columnparts = append(columnparts, columnpart) + } + + query := create + `(` + strings.Join(columnparts, `, `) + `);` + return query +} + +// buildTable creates a given destination table with the necessary gpkg_ information +func buildTable(h *gpkg.Handle, t Table) error { + query := t.createSQL() + _, err := h.Exec(query) + if err != nil { + log.Fatalf("error building table in target GeoPackage: %s", err) + } + + err = h.AddGeometryTable(gpkg.TableDescription{ + Name: t.Name, + ShortName: t.Name, + Description: t.Name, + GeometryField: t.gcolumn, + GeometryType: t.gtype, + //nolint:gosec // G115 + SRS: int32(t.srs.ID), + // + Z: gpkg.Prohibited, + M: gpkg.Prohibited, + }) + if err != nil { + log.Println("error adding geometry table in target GeoPackage:", err) + return err + } + return nil +} + +// create table for storing encoded geometries +func buildEncodedTable(h *gpkg.Handle, t Table) error { + db := h.DB + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id INTEGER PRIMARY KEY, + tile_x INTEGER NOT NULL, + tile_y INTEGER NOT NULL, + feature_id INTEGER NOT NULL, + geometry_type INTEGER NOT NULL, + data BLOB + ) + `, t.EncodedName()) + _, err := db.Exec(query) + if err != nil { + log.Println("error adding encoded table in target GeoPackage:", err) + return err + } + return nil +} diff --git a/processing/gpkg/mvttile.go b/processing/gpkg/mvttile.go deleted file mode 100644 index 8f0d709..0000000 --- a/processing/gpkg/mvttile.go +++ /dev/null @@ -1,106 +0,0 @@ -package gpkg - -import ( - "fmt" - "os" - "path/filepath" - "strconv" - - "github.com/pdok/texel/tile" -) - -// TileCoord identifies a tile by its column/row in the tile matrix. -type TileCoord struct { - X, Y uint -} - -// selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at -// least one encoded feature. -func (t Table) selectDistinctTilesSQL() string { - return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y` -} - -// ListTiles returns every distinct tile present in the "
_encoded" table. -func (source SourceGeopackage) ListTiles() ([]TileCoord, error) { - rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL()) - if err != nil { - return nil, fmt.Errorf("listing tiles: %w", err) - } - defer rows.Close() - - var tiles []TileCoord - for rows.Next() { - var x, y int64 - if err := rows.Scan(&x, &y); err != nil { - return nil, fmt.Errorf("scanning tile row: %w", err) - } - tiles = append(tiles, TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating tile rows: %w", err) - } - return tiles, nil -} - -// Build tile for specific coordinates. Fetches all info from database except -// columnnames, which are passed via keyIndex. Returns marshalled vectortile. -func (source SourceGeopackage) BuildMVTTile(layerName string, keyIndex map[string]uint32, tileX, tileY uint) ([]byte, error) { - encFeatRows, err := source.GetFeaturesForTile(tileX, tileY) - if err != nil { - return nil, fmt.Errorf("getting features for tile (%d,%d): %w", tileX, tileY, err) - } - featIDs := make([]int64, len(encFeatRows)) - for i, encFeatRow := range encFeatRows { - featIDs[i] = encFeatRow.FeatureID - } - - attributes, err := source.GetAttributesForFeatures(featIDs) - if err != nil { - return nil, fmt.Errorf("getting attributes for tile (%d,%d): %w", tileX, tileY, err) - } - - data, err := tile.BuildMVTTile(layerName, keyIndex, encFeatRows, attributes) - if err != nil { - return nil, fmt.Errorf("building tile (%d,%d): %w", tileX, tileY, err) - } - return data, nil -} - -// WriteTileFile writes one tile's serialized bytes to //.mvt, -// creating directories as needed. -func WriteTileFile(baseDir string, tileX, tileY uint, data []byte) error { - dir := filepath.Join(baseDir, strconv.FormatUint(uint64(tileX), 10)) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("creating tile directory %s: %w", dir, err) - } - - path := filepath.Join(dir, strconv.FormatUint(uint64(tileY), 10)+".mvt") - if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions - return fmt.Errorf("writing tile file %s: %w", path, err) - } - return nil -} - -// For all tiles, create the corresponding vectortile and write it to file. In -// this context `all tiles` means all tiles for which encoded geometries exist -// in the source geopackage. -func (source SourceGeopackage) ProcessTilesToMVT(outDir string) error { - // Build key index (columns) just once. - keyIndex := tile.BuildKeyDictionary(source.Table.AttributeColumnNames()) - - tiles, err := source.ListTiles() - if err != nil { - return err - } - - for _, tc := range tiles { - data, err := source.BuildMVTTile(source.Table.Name, keyIndex, tc.X, tc.Y) - if err != nil { - return fmt.Errorf("processing tile (%d,%d): %w", tc.X, tc.Y, err) - } - if err := WriteTileFile(outDir, tc.X, tc.Y, data); err != nil { - return fmt.Errorf("processing tile (%d,%d): %w", tc.X, tc.Y, err) - } - } - return nil -} diff --git a/processing/interface.go b/processing/interface.go index 04d9a5c..7db78a7 100644 --- a/processing/interface.go +++ b/processing/interface.go @@ -14,19 +14,21 @@ type Feature interface { type FeatureForTileMatrix interface { Feature TileMatrixID() int + // EncodedGeoms is nil if encoding is not desired EncodedGeoms() []tile.EncodedGeometry } type SnapResult struct { Geometry geom.Geometry - Tiles []pointindex.Quadrant + // Tiles is nil if encoding is not desired + Tiles []pointindex.Quadrant } -type EncodedFeature struct { - Feature Feature - EncodedGeoms []tile.EncodedGeometry +type TileCoord struct { + X, Y uint } +// Source and target for snapping type Source interface { ReadFeatures(ch chan<- Feature) } @@ -34,3 +36,15 @@ type Source interface { type Target interface { WriteFeatures(ch <-chan FeatureForTileMatrix) } + +// Source and target for tile generation +type MVTSource interface { + ListTiles() ([]TileCoord, error) + GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) + GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) + AttributeColumnNames() []string +} + +type MVTTarget interface { + WriteTile(x, y uint, data []byte) error +} diff --git a/processing/processing.go b/processing/processing.go index 1cd29ce..6398d66 100644 --- a/processing/processing.go +++ b/processing/processing.go @@ -128,7 +128,7 @@ func encodeGeometry(s SnapResult) []tile.EncodedGeometry { } // processFeatures processes the geometries in the features with the given function -func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTileMatrix, tmIDs []tms20.TMID, f processPolygonFunc) { +func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTileMatrix, tmIDs []tms20.TMID, f processPolygonFunc, encodeTiles bool) { stats := initStats() for { feature, hasMore := <-featuresIn @@ -144,7 +144,10 @@ func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTil stats.postCount++ } for tmID, snapResult := range newGeometriesPerTileMatrix { - encGeoms := encodeGeometry(snapResult) + var encGeoms []tile.EncodedGeometry + if encodeTiles { + encGeoms = encodeGeometry(snapResult) + } featuresOut <- wrapFeatureForTileMatrix(feature, tmID, snapResult.Geometry, encGeoms) } @@ -202,7 +205,7 @@ func writeFeaturesToTargets(featuresForTileMatrices <-chan FeatureForTileMatrix, type processPolygonFunc func(p geom.Polygon, tileMatrixIDs []tms20.TMID) map[tms20.TMID]SnapResult // ProcessFeatures applies the processing function/operation to each Target. -func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPolygonFunc) { +func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPolygonFunc, encodeTiles bool) { featuresBefore := make(chan Feature) featuresAfter := make(chan FeatureForTileMatrix) tileMatrixIDs := make([]tms20.TMID, 0, len(targets)) @@ -216,7 +219,7 @@ func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPoly defer wg.Done() writeFeaturesToTargets(featuresAfter, targets) }() - go processFeatures(featuresBefore, featuresAfter, tileMatrixIDs, f) + go processFeatures(featuresBefore, featuresAfter, tileMatrixIDs, f, encodeTiles) go readFeaturesFromSource(source, featuresBefore) wg.Wait() diff --git a/processing/processingmvt.go b/processing/processingmvt.go new file mode 100644 index 0000000..3bc00fe --- /dev/null +++ b/processing/processingmvt.go @@ -0,0 +1,44 @@ +package processing + +import ( + "fmt" + + "github.com/pdok/texel/tile" +) + +// Orchestrator for the tile creation pipeline +func BuildAndWriteMVTTiles(source MVTSource, target MVTTarget, tableName string) error { + // Build key index (columns) just once. + keyIndex := tile.BuildKeyDictionary(source.AttributeColumnNames()) + + tiles, err := source.ListTiles() + if err != nil { + return err + } + + for _, tc := range tiles { + encFeatRows, err := source.GetFeaturesForTile(tc.X, tc.Y) + if err != nil { + return fmt.Errorf("getting features for tile (%d,%d): %w", tc.X, tc.Y, err) + } + featIDs := make([]int64, len(encFeatRows)) + for i, encFeatRow := range encFeatRows { + featIDs[i] = encFeatRow.FeatureID + } + + attributes, err := source.GetAttributesForFeatures(featIDs) + if err != nil { + return fmt.Errorf("getting attributes for tile (%d,%d): %w", tc.X, tc.Y, err) + } + + data, err := tile.BuildMVTTile(tableName, keyIndex, encFeatRows, attributes) + if err != nil { + return fmt.Errorf("building tile (%d,%d): %w", tc.X, tc.Y, err) + } + + if err := target.WriteTile(tc.X, tc.Y, data); err != nil { + return fmt.Errorf("writing tile (%d,%d): %w", tc.X, tc.Y, err) + } + } + return nil +} diff --git a/snap/snap.go b/snap/snap.go index 226d130..ae6fd0e 100644 --- a/snap/snap.go +++ b/snap/snap.go @@ -71,8 +71,6 @@ func SnapPolygon(polygon geom.Polygon, tileMatrixSet tms20.TileMatrixSet, tmIDs var tilesbbox []pointindex.Quadrant if config.EncodeTiles { tilesbbox = ix.GetPrimitiveQBBox(pointindex.Level(tmIDsByLevels[level])) //nolint:gosec // G115 These are numbers < 40 - } else { - tilesbbox = []pointindex.Quadrant{} } newGeometry := geomhelp.PolygonSliceToGeom(newPolygons) newPolygonsPerTileMatrixID[tmIDsByLevels[level]] = processing.SnapResult{Geometry: newGeometry, Tiles: tilesbbox} From bc6e7b80ec7f4e189b81af059fb3541601d5f242 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 28 Jul 2026 09:50:27 +0200 Subject: [PATCH 23/25] test: new processGeometry test --- processing/processing_test.go | 114 ++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 processing/processing_test.go diff --git a/processing/processing_test.go b/processing/processing_test.go new file mode 100644 index 0000000..9bf04a4 --- /dev/null +++ b/processing/processing_test.go @@ -0,0 +1,114 @@ +package processing + +import ( + "reflect" + "testing" + + "github.com/go-spatial/geom" + "github.com/pdok/texel/pointindex" + "github.com/pdok/texel/tms20" +) + +// TestProcessGeometry covers processGeometry and, transitively, +// processMultiPolygon (which it delegates to for geom.MultiPolygon input). +func TestProcessGeometry(t *testing.T) { + polyA := geom.Polygon{{{0, 0}, {1, 0}, {1, 1}, {0, 0}}} + polyB := geom.Polygon{{{2, 2}, {3, 2}, {3, 3}, {2, 2}}} + tileA := pointindex.Quadrant{} + tileB := pointindex.Quadrant{} + + tests := []struct { + name string + // geometry passed to processGeometry + geometry geom.Geometry + tmIDs []tms20.TMID + // callResults is the sequence of results the stub processPolygonFunc + // returns, one entry per expected invocation, in call order. + callResults []map[tms20.TMID]SnapResult + want map[tms20.TMID]SnapResult + }{ + { + name: "single Polygon, single tmID: passthrough of f's result", + geometry: polyA, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}, + }, + }, + { + name: "unsupported geometry type (Point): default branch, f not called", + geometry: geom.Point{9, 9}, + tmIDs: []tms20.TMID{1, 2}, + callResults: nil, + want: map[tms20.TMID]SnapResult{ + 1: {}, + 2: {}, + }, + }, + { + name: "nil geometry value: default branch, f not called", + geometry: nil, + tmIDs: []tms20.TMID{1}, + callResults: nil, + want: map[tms20.TMID]SnapResult{ + 1: {}, + }, + }, + { + name: "empty MultiPolygon (len 0, non-nil): f never called, empty result map", + geometry: geom.MultiPolygon{}, + tmIDs: []tms20.TMID{1, 2}, + callResults: nil, + want: map[tms20.TMID]SnapResult{}, + }, + { + name: "MultiPolygon with 1 polygon, single tmID: passthrough", + geometry: geom.MultiPolygon{polyA}, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}, + }, + }, + { + name: "MultiPolygon with 2 polygons, single tmID: merged geometry and concatenated tiles", + geometry: geom.MultiPolygon{polyA, polyB}, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + {1: {Geometry: polyB, Tiles: []pointindex.Quadrant{tileB}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: geom.MultiPolygon{polyA, polyB}, Tiles: []pointindex.Quadrant{tileA, tileB}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callCount := 0 + f := func(_ geom.Polygon, _ []tms20.TMID) map[tms20.TMID]SnapResult { + if callCount >= len(tt.callResults) { + t.Fatalf("unexpected call %d to f", callCount+1) + } + result := tt.callResults[callCount] + callCount++ + return result + } + + got := processGeometry(tt.geometry, tt.tmIDs, f) + + if callCount != len(tt.callResults) { + t.Errorf("f called %d times, want %d", callCount, len(tt.callResults)) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("processGeometry() = %#v, want %#v", got, tt.want) + } + }) + } +} From a5b8f26e3a927ff84f82f0a84dad2170e4c0eb47 Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 28 Jul 2026 10:08:00 +0200 Subject: [PATCH 24/25] docs: update comments in source --- main.go | 8 +++----- processing/processing.go | 4 ++-- processing/processing_test.go | 8 ++++---- processing/processingmvt.go | 2 ++ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index 5cfa647..fd9f54d 100644 --- a/main.go +++ b/main.go @@ -48,10 +48,10 @@ func main() { app.Usage = "A Golang Polygon Snapping application" app.Version = versioninfo.Short() - // bare invocation (no subcommand) is routed to "snap", so its Required flags - // are still enforced as before. + // `texel` without command defaults to `snap`. app.DefaultCommand = "snap" + // Define two commands: `texel snap` and `texel mvt`. app.Commands = []*cli.Command{ { Name: "snap", @@ -276,9 +276,7 @@ func processBySnapping(source processing.Source, targets map[tms20.TMID]processi }, snapConfig.EncodeTiles) } -// runBuildMVTTiles builds and writes MVT tiles for every table in the given source -// GeoPackage (a GeoPackage previously produced with --encodetiles), using -// processing.BuildAndWriteMVTTiles for each table. +// Initialize resources for creating vecotrtiles and delegate to processing. func runBuildMVTTiles(sourcePath, outDir string) error { if _, err := os.Stat(sourcePath); os.IsNotExist(err) { return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath) diff --git a/processing/processing.go b/processing/processing.go index 6398d66..b107200 100644 --- a/processing/processing.go +++ b/processing/processing.go @@ -1,7 +1,7 @@ -// Package processing takes care of the logistics around reading and writing to a Target. -// Not the processing operation(s) itself. package processing +// Orchestrating logic around processing the snap command. + import ( "fmt" "log" diff --git a/processing/processing_test.go b/processing/processing_test.go index 9bf04a4..80fdee5 100644 --- a/processing/processing_test.go +++ b/processing/processing_test.go @@ -9,8 +9,8 @@ import ( "github.com/pdok/texel/tms20" ) -// TestProcessGeometry covers processGeometry and, transitively, -// processMultiPolygon (which it delegates to for geom.MultiPolygon input). +// TestProcessGeometry covers processGeometry and processMultiPolygon +// Uses a stub processPolygonFunc func TestProcessGeometry(t *testing.T) { polyA := geom.Polygon{{{0, 0}, {1, 0}, {1, 1}, {0, 0}}} polyB := geom.Polygon{{{2, 2}, {3, 2}, {3, 3}, {2, 2}}} @@ -22,8 +22,8 @@ func TestProcessGeometry(t *testing.T) { // geometry passed to processGeometry geometry geom.Geometry tmIDs []tms20.TMID - // callResults is the sequence of results the stub processPolygonFunc - // returns, one entry per expected invocation, in call order. + // callResults is a list of return values of the stub function in + // call order. callResults []map[tms20.TMID]SnapResult want map[tms20.TMID]SnapResult }{ diff --git a/processing/processingmvt.go b/processing/processingmvt.go index 3bc00fe..f90cb8f 100644 --- a/processing/processingmvt.go +++ b/processing/processingmvt.go @@ -1,5 +1,7 @@ package processing +// Orchestrating functionality for the mvt command + import ( "fmt" From c346fb6c702189ccdcd6d525281dd651ac9f260e Mon Sep 17 00:00:00 2001 From: Dirk van Bree Date: Tue, 28 Jul 2026 10:23:40 +0200 Subject: [PATCH 25/25] chore: remove trailing whitespace --- processing/processing_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/processing_test.go b/processing/processing_test.go index 80fdee5..f866ca9 100644 --- a/processing/processing_test.go +++ b/processing/processing_test.go @@ -9,7 +9,7 @@ import ( "github.com/pdok/texel/tms20" ) -// TestProcessGeometry covers processGeometry and processMultiPolygon +// TestProcessGeometry covers processGeometry and processMultiPolygon // Uses a stub processPolygonFunc func TestProcessGeometry(t *testing.T) { polyA := geom.Polygon{{{0, 0}, {1, 0}, {1, 1}, {0, 0}}}