diff --git a/go.mod b/go.mod index 49ff554..24ab543 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/creasty/defaults v1.8.0 github.com/go-playground/validator/v10 v10.30.3 github.com/go-spatial/geom v0.1.0 + github.com/golang/protobuf v1.5.3 github.com/iancoleman/strcase v0.3.0 github.com/muesli/reflow v0.3.0 github.com/perimeterx/marshmallow v1.1.5 @@ -29,7 +30,6 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-spatial/proj v0.3.0 // indirect github.com/go-test/deep v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/main.go b/main.go index a7dd19e..fd9f54d 100644 --- a/main.go +++ b/main.go @@ -36,6 +36,9 @@ const ( IGNOREOUTSIDEGRID string = `ignoreoutsidegrid` REVERSEWINDINGORDER string = `reversewindingorder` ENCODETILES string = `encodetiles` + + MVTSOURCE string = `mvtSourceGpkg` + MVTOUTDIR string = `mvtOutDir` ) //nolint:funlen @@ -45,152 +48,184 @@ func main() { app.Usage = "A Golang Polygon Snapping application" app.Version = versioninfo.Short() - app.Flags = []cli.Flag{ - &cli.StringFlag{ - Name: SOURCE, - Aliases: []string{"s"}, - Usage: "Source GPKG", - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(SOURCE)}, - }, - &cli.StringFlag{ - Name: TARGET, - Aliases: []string{"t"}, - Usage: "Target GPKG (prefix). One GPKG per tile matrix cq zoom level will be created and the filename will be suffixed. E.g. target_6.gpkg", - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TARGET)}, - }, - &cli.BoolFlag{ - Name: OVERWRITE, - Aliases: []string{"o"}, - Usage: "Overwrite a target GPKG if it exists", - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(OVERWRITE)}, - }, - &cli.StringFlag{ - Name: TILEMATRIXSET, - Aliases: []string{"tms"}, - Usage: `ID of a (built-in) tile matrix set. E.g.: NetherlandsRDNewQuad`, - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TILEMATRIXSET)}, - }, - &cli.StringFlag{ - Name: TILEMATRICES, - Aliases: []string{"z"}, - Usage: `IDs (usually the same as the zoom levels) of the tile matrices in the tile matrix set that should be processed for. JSON array of integers. E.g.: [4,5,6,7,8]`, - Required: true, - EnvVars: []string{strcase.ToScreamingSnake(TILEMATRICES)}, - }, - &cli.IntFlag{ - Name: PAGESIZE, - Aliases: []string{"p"}, - Usage: "Page Size, how many features are written per transaction to a target GPKG", - Value: 1000, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(PAGESIZE)}, - }, - &cli.BoolFlag{ - Name: KEEPPOINTSANDLINES, - Aliases: []string{"pl"}, - Usage: "Parts of polygons are reduced to points and lines after texel, keep these details or not.", - // TODO: This results in broken tiles, we should change this to a collapse option (optionally collapse rings from polygons to lines and points; grow lines as long as possible and add an option for line-length-to-keep) - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(KEEPPOINTSANDLINES)}, - }, - &cli.BoolFlag{ - Name: IGNOREOUTSIDEGRID, - Aliases: []string{"iog"}, - Usage: "Ignore polygons that fall (partly) outside the grid, instead of panicking", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(IGNOREOUTSIDEGRID)}, - }, - &cli.BoolFlag{ - Name: REVERSEWINDINGORDER, - Aliases: []string{"rwo"}, - Usage: "Reverse the winding order of rings in polygons, contrary to the OGC simple features spec", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(REVERSEWINDINGORDER)}, - }, - &cli.BoolFlag{ - Name: ENCODETILES, - Aliases: []string{"enc"}, - Usage: "Add tables with mapbox-encoded geometries.", - Value: false, - Required: false, - EnvVars: []string{strcase.ToScreamingSnake(ENCODETILES)}, - }, - } + // `texel` without command defaults to `snap`. + app.DefaultCommand = "snap" - app.Action = func(c *cli.Context) error { - tileMatrixSet, err := tms20.LoadEmbeddedTileMatrixSet(c.String(TILEMATRIXSET)) - if err != nil { - return err - } - var tileMatrixIDs []int - err = json.Unmarshal([]byte(c.String(TILEMATRICES)), &tileMatrixIDs) - if err != nil { - return err - } - if err = validateTileMatrixSet(tileMatrixSet, tileMatrixIDs); err != nil { - return err - } + // Define two commands: `texel snap` and `texel mvt`. + app.Commands = []*cli.Command{ + { + Name: "snap", + Usage: "Snap polygons in a source GeoPackage to a tile grid and write the result to a target GeoPackage", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: SOURCE, + Aliases: []string{"s"}, + Usage: "Source GPKG", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(SOURCE)}, + }, + &cli.StringFlag{ + Name: TARGET, + Aliases: []string{"t"}, + Usage: "Target GPKG (prefix). One GPKG per tile matrix cq zoom level will be created and the filename will be suffixed. E.g. target_6.gpkg", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TARGET)}, + }, + &cli.BoolFlag{ + Name: OVERWRITE, + Aliases: []string{"o"}, + Usage: "Overwrite a target GPKG if it exists", + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(OVERWRITE)}, + }, + &cli.StringFlag{ + Name: TILEMATRIXSET, + Aliases: []string{"tms"}, + Usage: `ID of a (built-in) tile matrix set. E.g.: NetherlandsRDNewQuad`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TILEMATRIXSET)}, + }, + &cli.StringFlag{ + Name: TILEMATRICES, + Aliases: []string{"z"}, + Usage: `IDs (usually the same as the zoom levels) of the tile matrices in the tile matrix set that should be processed for. JSON array of integers. E.g.: [4,5,6,7,8]`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(TILEMATRICES)}, + }, + &cli.IntFlag{ + Name: PAGESIZE, + Aliases: []string{"p"}, + Usage: "Page Size, how many features are written per transaction to a target GPKG", + Value: 1000, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(PAGESIZE)}, + }, + &cli.BoolFlag{ + Name: KEEPPOINTSANDLINES, + Aliases: []string{"pl"}, + Usage: "Parts of polygons are reduced to points and lines after texel, keep these details or not.", + // TODO: This results in broken tiles, we should change this to a collapse option (optionally collapse rings from polygons to lines and points; grow lines as long as possible and add an option for line-length-to-keep) + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(KEEPPOINTSANDLINES)}, + }, + &cli.BoolFlag{ + Name: IGNOREOUTSIDEGRID, + Aliases: []string{"iog"}, + Usage: "Ignore polygons that fall (partly) outside the grid, instead of panicking", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(IGNOREOUTSIDEGRID)}, + }, + &cli.BoolFlag{ + Name: REVERSEWINDINGORDER, + Aliases: []string{"rwo"}, + Usage: "Reverse the winding order of rings in polygons, contrary to the OGC simple features spec", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(REVERSEWINDINGORDER)}, + }, + &cli.BoolFlag{ + Name: ENCODETILES, + Aliases: []string{"enc"}, + Usage: "Add tables with mapbox-encoded geometries.", + Value: false, + Required: false, + EnvVars: []string{strcase.ToScreamingSnake(ENCODETILES)}, + }, + }, + Action: func(c *cli.Context) error { + tileMatrixSet, err := tms20.LoadEmbeddedTileMatrixSet(c.String(TILEMATRIXSET)) + if err != nil { + return err + } + var tileMatrixIDs []int + err = json.Unmarshal([]byte(c.String(TILEMATRICES)), &tileMatrixIDs) + if err != nil { + return err + } + if err = validateTileMatrixSet(tileMatrixSet, tileMatrixIDs); err != nil { + return err + } - _, err = os.Stat(c.String(SOURCE)) - if os.IsNotExist(err) { - log.Fatalf("error opening source GeoPackage: %s", err) - } + _, err = os.Stat(c.String(SOURCE)) + if os.IsNotExist(err) { + log.Fatalf("error opening source GeoPackage: %s", err) + } - source := gpkg.SourceGeopackage{} - source.Init(c.String(SOURCE)) - defer source.Close() + source := gpkg.SourceGeopackage{} + source.Init(c.String(SOURCE)) + defer source.Close() - targetPathFmt := injectSuffixIntoPath(c.String(TARGET)) + targetPathFmt := injectSuffixIntoPath(c.String(TARGET)) - gpkgTargets := make(map[int]*gpkg.TargetGeopackage, len(tileMatrixIDs)) - overwrite := c.Bool(OVERWRITE) - pagesize := c.Int(PAGESIZE) // TODO divide by tile matrices count - snapConfig := snap.Config{ - KeepPointsAndLines: c.Bool(KEEPPOINTSANDLINES), - IgnoreOutsideGrid: c.Bool(IGNOREOUTSIDEGRID), - ReverseWindingOrder: c.Bool(REVERSEWINDINGORDER), - EncodeTiles: c.Bool(ENCODETILES), - } - for _, tmID := range tileMatrixIDs { - gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize) - defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func - } + gpkgTargets := make(map[int]*gpkg.TargetGeopackage, len(tileMatrixIDs)) + overwrite := c.Bool(OVERWRITE) + pagesize := c.Int(PAGESIZE) // TODO divide by tile matrices count + snapConfig := snap.Config{ + KeepPointsAndLines: c.Bool(KEEPPOINTSANDLINES), + IgnoreOutsideGrid: c.Bool(IGNOREOUTSIDEGRID), + ReverseWindingOrder: c.Bool(REVERSEWINDINGORDER), + EncodeTiles: c.Bool(ENCODETILES), + } + for _, tmID := range tileMatrixIDs { + gpkgTargets[tmID] = initGPKGTarget(targetPathFmt, tmID, overwrite, pagesize, c.Bool(ENCODETILES)) + defer gpkgTargets[tmID].Close() // yes, supposed to go here, want to close all at end of func + } - tables := source.GetTableInfo() - for _, target := range gpkgTargets { - err = target.CreateTables(tables) - if err != nil { - log.Fatalf("error initialization the target GeoPackage: %s", err) - } - } + tables := source.GetTableInfo() + for _, target := range gpkgTargets { + err = target.CreateTables(tables) + if err != nil { + log.Fatalf("error initialization the target GeoPackage: %s", err) + } + } - log.Println("=== start snapping ===") + log.Println("=== start snapping ===") - // need a copied map because of type difference processing.Target vs gpkg.TargetGeopackage - targets := make(map[int]processing.Target, len(gpkgTargets)) - for tmID, target := range gpkgTargets { - targets[tmID] = target - } - // Process the tables sequentially - for _, table := range tables { - log.Printf(" snapping %s", table.Name) - for _, target := range gpkgTargets { - source.Table = table - target.Table = table - } - processBySnapping(source, targets, tileMatrixSet, snapConfig) - log.Printf(" finished %s", table.Name) - } + // need a copied map because of type difference processing.Target vs gpkg.TargetGeopackage + targets := make(map[int]processing.Target, len(gpkgTargets)) + for tmID, target := range gpkgTargets { + targets[tmID] = target + } + // Process the tables sequentially + for _, table := range tables { + log.Printf(" snapping %s", table.Name) + for _, target := range gpkgTargets { + source.Table = table + target.Table = table + } + processBySnapping(source, targets, tileMatrixSet, snapConfig) + log.Printf(" finished %s", table.Name) + } - log.Println("=== done snapping ===") - return nil + log.Println("=== done snapping ===") + return nil + }, + }, + { + Name: "mvt", + Usage: "Build MVT tiles from an already-encoded target GeoPackage (see --" + ENCODETILES + ")", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: MVTSOURCE, + Aliases: []string{"s"}, + Usage: `GeoPackage containing the encoded ("_encoded") and attribute tables`, + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(MVTSOURCE)}, + }, + &cli.StringFlag{ + Name: MVTOUTDIR, + Aliases: []string{"o"}, + Usage: "Directory to write the generated /.mvt files to", + Required: true, + EnvVars: []string{strcase.ToScreamingSnake(MVTOUTDIR)}, + }, + }, + Action: func(c *cli.Context) error { + return runBuildMVTTiles(c.String(MVTSOURCE), c.String(MVTOUTDIR)) + }, + }, } err := app.Run(os.Args) @@ -212,7 +247,7 @@ func validateTileMatrixSet(tms tms20.TileMatrixSet, tileMatrixIDs []tms20.TMID) return pointindex.IsQuadTree(tms) } -func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int) *gpkg.TargetGeopackage { +func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int, encodeTiles bool) *gpkg.TargetGeopackage { targetPath := fmt.Sprintf(targetPathFmt, tmID) if overwrite { err := os.Remove(targetPath) @@ -224,6 +259,7 @@ func initGPKGTarget(targetPathFmt string, tmID int, overwrite bool, pagesize int } target := gpkg.TargetGeopackage{} target.Init(targetPath, pagesize) + target.EncodeTiles = encodeTiles return &target } @@ -237,5 +273,28 @@ func injectSuffixIntoPath(p string) string { func processBySnapping(source processing.Source, targets map[tms20.TMID]processing.Target, tileMatrixSet tms20.TileMatrixSet, snapConfig snap.Config) { processing.ProcessFeatures(source, targets, func(p geom.Polygon, tmIDs []tms20.TMID) map[tms20.TMID]processing.SnapResult { return snap.SnapPolygon(p, tileMatrixSet, tmIDs, snapConfig) - }) + }, snapConfig.EncodeTiles) +} + +// Initialize resources for creating vecotrtiles and delegate to processing. +func runBuildMVTTiles(sourcePath, outDir string) error { + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + return fmt.Errorf("source GeoPackage does not exist: %s", sourcePath) + } + + source := gpkg.MVTSourceGeopackage{} + source.Init(sourcePath) + defer source.Close() + + mvtTarget := gpkg.MVTFileTarget{OutDir: outDir} + + tables := source.GetTableInfo() + for _, table := range tables { + source.Table = table + log.Printf(" building MVT tiles for %s", table.Name) + if err := processing.BuildAndWriteMVTTiles(&source, &mvtTarget, table.Name); err != nil { + return fmt.Errorf("building MVT tiles for %s: %w", table.Name, err) + } + } + return nil } diff --git a/pointindex/pointindex.go b/pointindex/pointindex.go index 97e67d1..4760c95 100644 --- a/pointindex/pointindex.go +++ b/pointindex/pointindex.go @@ -135,16 +135,16 @@ func (ix *PointIndex) GetPrimitiveQBBox(l Level) []Quadrant { maxY = max(y, maxY) } - quadrantSlice := make([]Quadrant, (maxY-minY)*(maxX-minX)) - for i := range maxX - minX { - for j := range maxY - minY { + quadrantSlice := make([]Quadrant, (maxY-minY+1)*(maxX-minX+1)) + for i := range maxX - minX + 1 { + for j := range maxY - minY + 1 { extent, centroid := ix.getQuadrantExtentAndCentroid(l, minX+i, minY+j, ix.intExtent) newQuadrant := Quadrant{ z: morton.MustToZ(minX+i, minY+j), intExtent: extent, intCentroid: centroid, } - quadrantSlice[i*(maxY-minY)+j] = newQuadrant + quadrantSlice[i*(maxY-minY+1)+j] = newQuadrant } } return quadrantSlice diff --git a/processing/gpkg/encoded.go b/processing/gpkg/encoded.go deleted file mode 100644 index a6b410b..0000000 --- a/processing/gpkg/encoded.go +++ /dev/null @@ -1,82 +0,0 @@ -package gpkg - -import ( - "encoding/binary" - "fmt" - "log" - - "github.com/go-spatial/geom/encoding/gpkg" - "github.com/pdok/texel/processing" -) - -func (t Table) EncodedName() string { - return t.Name + "_encoded" -} - -func (t Table) insertSQLEncoded() string { - return `INSERT INTO "` + t.EncodedName() + `"` + - ` (tile_x, tile_y, feature_id, geometry_type, data)` + - ` VALUES (?, ?, ?, ?, ?)` -} - -func serializeToBytes(intslice []uint32) []byte { - bytes := make([]byte, len(intslice)*4) - - for i, x := range intslice { - binary.LittleEndian.PutUint32(bytes[i*4:], x) - } - - return bytes -} - -func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.FeatureForTileMatrix) { - tx, err := target.handle.Begin() - if err != nil { - log.Fatalf("Could not start a transaction: %s", err) - } - - stmt, err := tx.Prepare(target.Table.insertSQLEncoded()) - if err != nil { - log.Fatalf("Could not prepare a statement: %s", err) - } - - for _, ef := range encFeature { - cols := ef.Columns() - if len(cols) == 0 { - log.Fatalf("Processing feature without data") - } - fid := cols[0] - for _, eg := range ef.EncodedGeoms() { - bytes := serializeToBytes(eg.Encoding) - _, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes) - if err != nil { - log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) - } - } - - } - - stmt.Close() - _ = tx.Commit() -} - -func buildEncodedTable(h *gpkg.Handle, t Table) error { - db := h.DB - - query := fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s ( - id INTEGER PRIMARY KEY, - tile_x INTEGER NOT NULL, - tile_y INTEGER NOT NULL, - feature_id INTEGER NOT NULL, - geometry_type INTEGER NOT NULL, - data BLOB - ) - `, t.EncodedName()) - _, err := db.Exec(query) - if err != nil { - log.Println("error adding encoded table in target GeoPackage:", err) - return err - } - return nil -} diff --git a/processing/gpkg/gpkg.go b/processing/gpkg/gpkg.go index fd1cfa1..b6af9c0 100644 --- a/processing/gpkg/gpkg.go +++ b/processing/gpkg/gpkg.go @@ -1,29 +1,16 @@ package gpkg +// This file defines general geopackage-related infrastructure shared by both +// the `snap` and `mvt` flows. + import ( "fmt" "log" "strings" - "time" - "github.com/go-spatial/geom" "github.com/go-spatial/geom/encoding/gpkg" - "github.com/pdok/texel/processing" ) -type featureGPKG struct { - columns []any - geometry geom.Geometry -} - -func (f featureGPKG) Columns() []any { - return f.columns -} - -func (f featureGPKG) Geometry() geom.Geometry { - return f.geometry -} - type column struct { cid int name string @@ -33,6 +20,7 @@ type column struct { pk int } +// Abstraction of geometry table in a gpkg type Table struct { Name string columns []column @@ -41,7 +29,11 @@ type Table struct { srs gpkg.SpatialReferenceSystem } -// geometryTypeFromString returns the numeric value of a gometry string +// Name of associated table of a geometry table. +func (t Table) EncodedName() string { + return t.Name + "_encoded" +} + func geometryTypeFromString(geometrytype string) gpkg.GeometryType { switch strings.ToUpper(geometrytype) { case "GEOMETRY": @@ -65,89 +57,21 @@ func geometryTypeFromString(geometrytype string) gpkg.GeometryType { } } -//nolint:recvcheck // TODO double check this -type SourceGeopackage struct { - Table Table - handle *gpkg.Handle -} +// Custom wrapper for gpkg.Handle. To be embedded in other structs +type geopackageHandle struct{ handle *gpkg.Handle } -func (source *SourceGeopackage) Init(file string) { - source.handle = openGeopackage(file) +func (gH *geopackageHandle) Init(file string) { + gH.handle = openGeopackage(file) } -func (source SourceGeopackage) Close() { - source.handle.Close() +func (gH *geopackageHandle) Close() { + gH.handle.Close() } -//nolint:funlen,cyclop -func (source SourceGeopackage) ReadFeatures(features chan<- processing.Feature) { - rows, err := source.handle.Query(source.Table.selectSQL()) - if err != nil { - log.Fatalf("err during closing rows: %s", err) - } - - cols, err := rows.Columns() - if err != nil { - log.Fatalf("error reading the columns: %s", err) - } - - for rows.Next() { - vals := make([]any, len(cols)) - valPtrs := make([]any, len(cols)) - for i := range cols { - valPtrs[i] = &vals[i] - } - - if err = rows.Scan(valPtrs...); err != nil { - log.Fatalf("err reading row values: %v", err) - } - var f featureGPKG - var c []any - - for i, colName := range cols { - switch colName { - case source.Table.gcolumn: - wkbgeom, err := gpkg.DecodeGeometry(vals[i].([]byte)) - if err != nil { - log.Fatalf("error decoding the geometry: %s", err) - } - f.geometry = wkbgeom.Geometry - default: - switch v := vals[i].(type) { - case []uint8: - asBytes := make([]byte, len(v)) - copy(asBytes, v) - c = append(c, string(asBytes)) - case int64: - c = append(c, v) - case float64: - c = append(c, v) - case time.Time: - c = append(c, v.Format(time.RFC3339)) - case string: - c = append(c, v) - case nil: - c = append(c, v) - default: - log.Fatalf("unexpected type for sqlite column data: %v: %T", cols[i], v) - } - } - f.columns = c - } - ff := &f - features <- ff - } - err = rows.Err() - if err != nil { - log.Fatal(err) - } - close(features) - defer rows.Close() -} - -func (source SourceGeopackage) GetTableInfo() []Table { +// Generate []Table slice for looping over. +func (gH *geopackageHandle) GetTableInfo() []Table { query := `SELECT table_name, column_name, geometry_type_name, srs_id FROM gpkg_geometry_columns;` - rows, err := source.handle.Query(query) + rows, err := gH.handle.Query(query) if err != nil { log.Fatalf("error during closing rows: %v - %v", query, err) } @@ -162,9 +86,9 @@ func (source SourceGeopackage) GetTableInfo() []Table { log.Fatalf("error retrieving the source table information: %s", err) } - t.columns = getTableColumns(source.handle, t.Name) + t.columns = gH.getTableColumns(t.Name) t.gtype = geometryTypeFromString(gtype) - t.srs = getSpatialReferenceSystem(source.handle, srsID) + t.srs = gH.getSpatialReferenceSystem(srsID) tables = append(tables, t) } @@ -172,120 +96,6 @@ func (source SourceGeopackage) GetTableInfo() []Table { return tables } -type TargetGeopackage struct { - Table Table - pagesize int - handle *gpkg.Handle - encodeTiles bool -} - -func (target *TargetGeopackage) Init(file string, pagesize int) { - target.pagesize = pagesize - target.handle = openGeopackage(file) -} - -func (target *TargetGeopackage) Close() { - target.handle.Close() -} - -func (target *TargetGeopackage) CreateTables(tables []Table) error { - for _, table := range tables { - err := target.handle.UpdateSRS(table.srs) - if err != nil { - return err - } - - err = buildTable(target.handle, table) - if err != nil { - return err - } - - if target.encodeTiles { - err = buildEncodedTable(target.handle, table) - if err != nil { - return err - } - } - } - return nil -} - -func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.FeatureForTileMatrix) { - var features []processing.FeatureForTileMatrix - - for { - feature, hasMore := <-inFeatures - if !hasMore { - target.writeFeatures(features) - if target.encodeTiles { - target.writeEncodedFeatures(features) - } - break - } - features = append(features, feature) - - if len(features)%target.pagesize == 0 { - target.writeFeatures(features) - if target.encodeTiles { - target.writeEncodedFeatures(features) - } - features = nil - } - } -} - -func (target *TargetGeopackage) writeFeatures(features []processing.FeatureForTileMatrix) { - tx, err := target.handle.Begin() - if err != nil { - log.Fatalf("Could not start a transaction: %s", err) - } - - stmt, err := tx.Prepare(target.Table.insertSQL()) - if err != nil { - log.Fatalf("Could not prepare a statement: %s", err) - } - - var ext *geom.Extent - - for _, f := range features { - //nolint:gosec // G115 - sb, err := gpkg.NewBinary(int32(target.Table.srs.ID), f.Geometry()) - if err != nil { - log.Fatalf("Could not create a binary geometry: %s", err) - } - - data := f.Columns() - data = append(data, sb) - - _, err = stmt.Exec(data...) - if err != nil { - var fid any = "unknown" - if len(data) > 0 { - fid = data[0] - } - log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) - } - - if ext == nil { - ext, err = geom.NewExtentFromGeometry(f.Geometry()) - if err != nil { - ext = nil - log.Println("Failed to create new extent:", err) - continue - } - } else { - _ = ext.AddGeometry(f.Geometry()) - } - } - stmt.Close() - _ = tx.Commit() - - err = target.handle.UpdateGeometryExtent(target.Table.Name, ext) - if err != nil { - log.Fatalln("Failed to update new extent:", err) - } -} - func openGeopackage(file string) *gpkg.Handle { handle, err := gpkg.Open(file) if err != nil { @@ -294,60 +104,12 @@ func openGeopackage(file string) *gpkg.Handle { return handle } -// createSQL creates a CREATE statement on the given table and column information -// used for creating feature tables in the target Geopackage -func (t Table) createSQL() string { - create := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%v"`, t.Name) - var columnparts []string //nolint:prealloc - for _, column := range t.columns { - columnpart := column.name + ` ` + column.ctype - if column.notnull == 1 { - columnpart += ` NOT NULL` - } - if column.pk == 1 { - columnpart += ` PRIMARY KEY` - } - - columnparts = append(columnparts, columnpart) - } - - query := create + `(` + strings.Join(columnparts, `, `) + `);` - return query -} - -// selectSQL build a SELECT statement based on the table and columns -// used for reading the source features -func (t Table) selectSQL() string { - var csql []string //nolint:prealloc - for _, c := range t.columns { - csql = append(csql, c.name) - } - query := `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `";` - return query -} - -// insertSQL used for writing the features -// build the INSERT statement based on the table and columns -func (t Table) insertSQL() string { - var csql, vsql []string - for _, c := range t.columns { - if c.name != t.gcolumn { - csql = append(csql, c.name) - vsql = append(vsql, `?`) - } - } - csql = append(csql, t.gcolumn) - vsql = append(vsql, `?`) - query := `INSERT INTO "` + t.Name + `"(` + strings.Join(csql, `,`) + `) VALUES(` + strings.Join(vsql, `,`) + `)` - return query -} - // getSpatialReferenceSystem extracts this based on the given SRS id -func getSpatialReferenceSystem(h *gpkg.Handle, id int) gpkg.SpatialReferenceSystem { +func (gH *geopackageHandle) getSpatialReferenceSystem(id int) gpkg.SpatialReferenceSystem { var srs gpkg.SpatialReferenceSystem query := `SELECT srs_name, srs_id, organization, organization_coordsys_id, definition, description FROM gpkg_spatial_ref_sys WHERE srs_id = %v;` - row := h.QueryRow(fmt.Sprintf(query, id)) + row := gH.handle.QueryRow(fmt.Sprintf(query, id)) var description *string _ = row.Scan(&srs.Name, &srs.ID, &srs.Organization, &srs.OrganizationCoordsysID, &srs.Definition, &description) if description != nil { @@ -358,10 +120,10 @@ func getSpatialReferenceSystem(h *gpkg.Handle, id int) gpkg.SpatialReferenceSyst } // getTableColumns collects the column information of a given table -func getTableColumns(h *gpkg.Handle, table string) []column { +func (gH *geopackageHandle) getTableColumns(table string) []column { var columns []column query := `PRAGMA table_info('%v');` - rows, err := h.Query(fmt.Sprintf(query, table)) + rows, err := gH.handle.Query(fmt.Sprintf(query, table)) if err != nil { log.Fatalf("err during closing rows: %v - %v", query, err) } @@ -377,30 +139,3 @@ func getTableColumns(h *gpkg.Handle, table string) []column { defer rows.Close() return columns } - -// buildTable creates a given destination table with the necessary gpkg_ information -func buildTable(h *gpkg.Handle, t Table) error { - query := t.createSQL() - _, err := h.Exec(query) - if err != nil { - log.Fatalf("error building table in target GeoPackage: %s", err) - } - - err = h.AddGeometryTable(gpkg.TableDescription{ - Name: t.Name, - ShortName: t.Name, - Description: t.Name, - GeometryField: t.gcolumn, - GeometryType: t.gtype, - //nolint:gosec // G115 - SRS: int32(t.srs.ID), - // - Z: gpkg.Prohibited, - M: gpkg.Prohibited, - }) - if err != nil { - log.Println("error adding geometry table in target GeoPackage:", err) - return err - } - return nil -} diff --git a/processing/gpkg/gpkgmvt.go b/processing/gpkg/gpkgmvt.go new file mode 100644 index 0000000..ad5b662 --- /dev/null +++ b/processing/gpkg/gpkgmvt.go @@ -0,0 +1,261 @@ +package gpkg + +// This file contains the geopackage functionality used for `texel mvt`. +// The exported functionality is precisely given by MVTSource and MVTTarget. +// We reuse SourceGeopackage as MVTSource. + +import ( + "encoding/binary" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/pdok/texel/processing" + "github.com/pdok/texel/tile" +) + +// Batch value for reading features from the geopackage table +const maxSQLiteBatchParams = 500 + +type MVTSourceGeopackage struct { + Table Table + geopackageHandle +} + +// GetAttributesForFeatures fetches the attributes (i.e. all non-geometry, non-fid columns) +// for the given feature IDs, batching the query to respect SQLite's parameter limits. +// The fid is assumed to be the table's first column. +func (source MVTSourceGeopackage) GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) { + attributes := make(tile.InternalAttributeTable, len(featureIDs)) + fidColumn := source.Table.columns[0].name + + for start := 0; start < len(featureIDs); start += maxSQLiteBatchParams { + batch := featureIDs[start:min(start+maxSQLiteBatchParams, len(featureIDs))] + + batchAsAny := make([]any, len(batch)) + for i, fid := range batch { + batchAsAny[i] = fid + } + + if err := source.queryAttributesBatch(fidColumn, batchAsAny, attributes); err != nil { + return nil, err + } + } + + return attributes, nil +} + +// selectByFIDsSQL builds a SELECT of all the table's columns for a batch of feature IDs. +// The fid is assumed to be the table's first column. +func (t Table) selectByFIDsSQL(batchSize int) string { + var csql []string //nolint:prealloc + for _, c := range t.columns { + csql = append(csql, c.name) + } + placeholders := strings.TrimSuffix(strings.Repeat("?,", batchSize), ",") + fidColumn := t.columns[0].name + return `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `" WHERE ` + fidColumn + ` IN (` + placeholders + `)` +} + +func cleanData(v any) (any, bool) { + switch v := v.(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + return v, true + case int64: + return v, true + case float64: + return v, true + case time.Time: + return v.Format(time.RFC3339), true + case string: + return v, true + case nil: + return nil, true + default: + return nil, false + } +} + +// attributeColumnNames lists all columns except fid and geometry column. +// Assumptions: fid column has index 0 +func (t Table) attributeColumnNames() []string { + names := make([]string, 0, len(t.columns)) + for i, c := range t.columns { + if i == 0 || c.name == t.gcolumn { + continue // skip the fid and geometry columns, they are not attributes + } + names = append(names, c.name) + } + return names +} + +// AttributeColumnNames satisfies processing.MVTSource by delegating to the +// current Table. +func (source MVTSourceGeopackage) AttributeColumnNames() []string { + return source.Table.attributeColumnNames() +} + +// selectEncodedSQL builds the SELECT for fetching the encoded features of a single tile +func (t Table) selectEncodedSQL() string { + return `SELECT feature_id, geometry_type, data FROM "` + t.EncodedName() + `" WHERE tile_x = ? AND tile_y = ?` +} + +// selectDistinctTilesSQL builds the SELECT for enumerating every tile that has at +// least one encoded feature. +func (t Table) selectDistinctTilesSQL() string { + return `SELECT DISTINCT tile_x, tile_y FROM "` + t.EncodedName() + `" ORDER BY tile_x, tile_y` +} + +// GetFeaturesForTile returns every encoded feature stored for the given tile, +// i.e. the equivalent of the tile-features table's rows for (tileX, tileY). +func (source MVTSourceGeopackage) GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) { + rows, err := source.handle.Query(source.Table.selectEncodedSQL(), tileX, tileY) + if err != nil { + return nil, fmt.Errorf("querying encoded features for tile (%d,%d): %w", tileX, tileY, err) + } + defer rows.Close() + + var features []tile.EncodedFeatureRow + for rows.Next() { + var featureID int64 + var geometryType int32 + var data []byte + if err := rows.Scan(&featureID, &geometryType, &data); err != nil { + return nil, fmt.Errorf("scanning encoded feature row for tile (%d,%d): %w", tileX, tileY, err) + } + features = append(features, tile.EncodedFeatureRow{ + FeatureID: featureID, + Geom: tile.EncodedGeometry{ + Encoding: deserializeFromBytes(data), + GeometryType: geometryType, + XTile: tileX, + YTile: tileY, + }, + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating encoded feature rows for tile (%d,%d): %w", tileX, tileY, err) + } + return features, nil +} + +// deserializeFromBytes is the inverse of serializeToBytes: it turns a BLOB of +// little-endian uint32s back into the encoded geometry command/coordinate stream. +func deserializeFromBytes(data []byte) []uint32 { + intslice := make([]uint32, len(data)/4) + for i := range intslice { + intslice[i] = binary.LittleEndian.Uint32(data[i*4:]) + } + return intslice +} + +// ListTiles returns every distinct tile present in the "
_encoded" table. +func (source MVTSourceGeopackage) ListTiles() ([]processing.TileCoord, error) { + rows, err := source.handle.Query(source.Table.selectDistinctTilesSQL()) + if err != nil { + return nil, fmt.Errorf("listing tiles: %w", err) + } + defer rows.Close() + + var tiles []processing.TileCoord + for rows.Next() { + var x, y int64 + if err := rows.Scan(&x, &y); err != nil { + return nil, fmt.Errorf("scanning tile row: %w", err) + } + tiles = append(tiles, processing.TileCoord{X: uint(x), Y: uint(y)}) //nolint:gosec // G115 Tile coords fit within uint + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating tile rows: %w", err) + } + return tiles, nil +} + +// Read from source.Table, store rows as attributes[fid][columnName] = value for all fids +func (source MVTSourceGeopackage) queryAttributesBatch(fidColumn string, fids []any, attributes tile.InternalAttributeTable) error { + // Read relevant rows + rows, err := source.handle.Query(source.Table.selectByFIDsSQL(len(fids)), fids...) + if err != nil { + return fmt.Errorf("querying attributes for %d feature(s): %w", len(fids), err) + } + defer rows.Close() + + cols := source.getColNames() + + // Process rows + for rows.Next() { + // Read data with helper vars + vals := make([]any, len(cols)) + valPtrs := make([]any, len(cols)) + for i := range cols { + valPtrs[i] = &vals[i] + } + if err := rows.Scan(valPtrs...); err != nil { + return fmt.Errorf("scanning attribute row: %w", err) + } + + // Interpret data + row := make(map[string]any, len(cols)) + var fid int64 + var fidFound bool + for i, colName := range cols { + switch colName { + case fidColumn: + // Read fid + id, ok := vals[i].(int64) + if !ok { + return fmt.Errorf("expected int64 fid for column %s, got %T", colName, vals[i]) + } + fid, fidFound = id, true + case source.Table.gcolumn: + // Skip geometry column + default: + // Interpret data + v, ok := cleanData(vals[i]) + if !ok { + return fmt.Errorf("unexpected type for sqlite column data: %v: %T", colName, v) + } + row[colName] = v + } + } + if !fidFound { + return fmt.Errorf("row did not contain fid column %s", fidColumn) + } + attributes[fid] = row + } + return rows.Err() +} + +func (source MVTSourceGeopackage) getColNames() []string { + cols := source.Table.columns + colNames := make([]string, len(cols)) + for i, col := range cols { + colNames[i] = col.name + } + return colNames +} + +// MVTFileTarget writes built MVT tiles to //.mvt. +type MVTFileTarget struct { + OutDir string +} + +// WriteTile writes one tile's serialized bytes to //.mvt, +// creating directories as needed. +func (t *MVTFileTarget) WriteTile(x, y uint, data []byte) error { + dir := filepath.Join(t.OutDir, strconv.FormatUint(uint64(x), 10)) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("creating tile directory %s: %w", dir, err) + } + + path := filepath.Join(dir, strconv.FormatUint(uint64(y), 10)+".mvt") + if err := os.WriteFile(path, data, 0o644); err != nil { //nolint:gosec // G306 tile output does not need restrictive permissions + return fmt.Errorf("writing tile file %s: %w", path, err) + } + return nil +} diff --git a/processing/gpkg/gpkgsnap.go b/processing/gpkg/gpkgsnap.go new file mode 100644 index 0000000..8fe6181 --- /dev/null +++ b/processing/gpkg/gpkgsnap.go @@ -0,0 +1,366 @@ +package gpkg + +// Functionality regarding geopackages for `texel snap`. Exported are: +// SourceGeopackage.ReadFeatures +// TargetGeopackage.CreateTables +// TargetGeopackage.WriteFeatures + +import ( + "encoding/binary" + "fmt" + "log" + "strings" + "time" + + "github.com/go-spatial/geom" + "github.com/go-spatial/geom/encoding/gpkg" + "github.com/pdok/texel/processing" +) + +// Abstraction of a source. Table is a field that represents the table that is +// currently being processed. +type SourceGeopackage struct { + Table Table + geopackageHandle +} + +// Abstraction of target geopackage. Table represents the table that is +// currently being processed. +type TargetGeopackage struct { + Table Table + pagesize int + geopackageHandle + EncodeTiles bool +} + +// Overrides geopackageHandle.Init +func (target *TargetGeopackage) Init(file string, pagesize int) { + target.pagesize = pagesize + target.geopackageHandle.Init(file) +} + +type featureGPKG struct { + columns []any + geometry geom.Geometry +} + +func (f featureGPKG) Columns() []any { + return f.columns +} + +func (f featureGPKG) Geometry() geom.Geometry { + return f.geometry +} + +//nolint:funlen,cyclop +func (source SourceGeopackage) ReadFeatures(features chan<- processing.Feature) { + rows, err := source.handle.Query(source.Table.selectSQL()) + if err != nil { + log.Fatalf("err during closing rows: %s", err) + } + + cols, err := rows.Columns() + if err != nil { + log.Fatalf("error reading the columns: %s", err) + } + + for rows.Next() { + vals := make([]any, len(cols)) + valPtrs := make([]any, len(cols)) + for i := range cols { + valPtrs[i] = &vals[i] + } + + if err = rows.Scan(valPtrs...); err != nil { + log.Fatalf("err reading row values: %v", err) + } + var f featureGPKG + var c []any + + for i, colName := range cols { + switch colName { + case source.Table.gcolumn: + wkbgeom, err := gpkg.DecodeGeometry(vals[i].([]byte)) + if err != nil { + log.Fatalf("error decoding the geometry: %s", err) + } + f.geometry = wkbgeom.Geometry + default: + switch v := vals[i].(type) { + case []uint8: + asBytes := make([]byte, len(v)) + copy(asBytes, v) + c = append(c, string(asBytes)) + case int64: + c = append(c, v) + case float64: + c = append(c, v) + case time.Time: + c = append(c, v.Format(time.RFC3339)) + case string: + c = append(c, v) + case nil: + c = append(c, v) + default: + log.Fatalf("unexpected type for sqlite column data: %v: %T", cols[i], v) + } + } + f.columns = c + } + ff := &f + features <- ff + } + err = rows.Err() + if err != nil { + log.Fatal(err) + } + close(features) + defer rows.Close() +} + +// selectSQL build a SELECT statement based on the table and columns +// used for reading the source features +func (t Table) selectSQL() string { + var csql []string //nolint:prealloc + for _, c := range t.columns { + csql = append(csql, c.name) + } + query := `SELECT ` + strings.Join(csql, `,`) + ` FROM "` + t.Name + `";` + return query +} + +// insertSQL used for writing the features +// build the INSERT statement based on the table and columns +func (t Table) insertSQL() string { + var csql, vsql []string + for _, c := range t.columns { + if c.name != t.gcolumn { + csql = append(csql, c.name) + vsql = append(vsql, `?`) + } + } + csql = append(csql, t.gcolumn) + vsql = append(vsql, `?`) + query := `INSERT INTO "` + t.Name + `"(` + strings.Join(csql, `,`) + `) VALUES(` + strings.Join(vsql, `,`) + `)` + return query +} + +// insertSQLEncoded used for writing the encoded geometries when -enc is on +func (t Table) insertSQLEncoded() string { + return `INSERT INTO "` + t.EncodedName() + `"` + + ` (tile_x, tile_y, feature_id, geometry_type, data)` + + ` VALUES (?, ?, ?, ?, ?)` +} + +func serializeToBytes(intslice []uint32) []byte { + bytes := make([]byte, len(intslice)*4) + + for i, x := range intslice { + binary.LittleEndian.PutUint32(bytes[i*4:], x) + } + + return bytes +} + +// Create tables in target. Defers to buildTable and buildEncodedTable +func (target *TargetGeopackage) CreateTables(tables []Table) error { + for _, table := range tables { + err := target.handle.UpdateSRS(table.srs) + if err != nil { + return err + } + + err = buildTable(target.handle, table) + if err != nil { + return err + } + + if target.EncodeTiles { + err = buildEncodedTable(target.handle, table) + if err != nil { + return err + } + } + } + return nil +} + +// Write the features to geopackage. Defers to writeFeatures and writeEncodedFeatures +func (target *TargetGeopackage) WriteFeatures(inFeatures <-chan processing.FeatureForTileMatrix) { + var features []processing.FeatureForTileMatrix + + for { + feature, hasMore := <-inFeatures + if !hasMore { + target.writeFeatures(features) + if target.EncodeTiles { + target.writeEncodedFeatures(features) + } + break + } + features = append(features, feature) + + if len(features)%target.pagesize == 0 { + target.writeFeatures(features) + if target.EncodeTiles { + target.writeEncodedFeatures(features) + } + features = nil + } + } +} + +// Write snapped features to table +func (target *TargetGeopackage) writeFeatures(features []processing.FeatureForTileMatrix) { + tx, err := target.handle.Begin() + if err != nil { + log.Fatalf("Could not start a transaction: %s", err) + } + + stmt, err := tx.Prepare(target.Table.insertSQL()) + if err != nil { + log.Fatalf("Could not prepare a statement: %s", err) + } + + var ext *geom.Extent + + for _, f := range features { + //nolint:gosec // G115 + sb, err := gpkg.NewBinary(int32(target.Table.srs.ID), f.Geometry()) + if err != nil { + log.Fatalf("Could not create a binary geometry: %s", err) + } + + data := f.Columns() + data = append(data, sb) + + _, err = stmt.Exec(data...) + if err != nil { + var fid any = "unknown" + if len(data) > 0 { + fid = data[0] + } + log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) + } + + if ext == nil { + ext, err = geom.NewExtentFromGeometry(f.Geometry()) + if err != nil { + ext = nil + log.Println("Failed to create new extent:", err) + continue + } + } else { + _ = ext.AddGeometry(f.Geometry()) + } + } + stmt.Close() + _ = tx.Commit() + + err = target.handle.UpdateGeometryExtent(target.Table.Name, ext) + if err != nil { + log.Fatalln("Failed to update new extent:", err) + } +} + +// Write encoded geometries to corresponding tables +func (target *TargetGeopackage) writeEncodedFeatures(encFeature []processing.FeatureForTileMatrix) { + tx, err := target.handle.Begin() + if err != nil { + log.Fatalf("Could not start a transaction: %s", err) + } + + stmt, err := tx.Prepare(target.Table.insertSQLEncoded()) + if err != nil { + log.Fatalf("Could not prepare a statement: %s", err) + } + + for _, ef := range encFeature { + cols := ef.Columns() + if len(cols) == 0 { + log.Fatalf("Processing feature without data") + } + fid := cols[0] + for _, eg := range ef.EncodedGeoms() { + bytes := serializeToBytes(eg.Encoding) + _, err := stmt.Exec(eg.XTile, eg.YTile, fid, eg.GeometryType, bytes) + if err != nil { + log.Fatalf("Could not get a result summary from the prepared statement for fid %s: %s", fid, err) + } + } + + } + + stmt.Close() + _ = tx.Commit() +} + +// createSQL creates a CREATE statement on the given table and column information +// used for creating feature tables in the target Geopackage +func (t Table) createSQL() string { + create := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS "%v"`, t.Name) + var columnparts []string //nolint:prealloc + for _, column := range t.columns { + columnpart := column.name + ` ` + column.ctype + if column.notnull == 1 { + columnpart += ` NOT NULL` + } + if column.pk == 1 { + columnpart += ` PRIMARY KEY` + } + + columnparts = append(columnparts, columnpart) + } + + query := create + `(` + strings.Join(columnparts, `, `) + `);` + return query +} + +// buildTable creates a given destination table with the necessary gpkg_ information +func buildTable(h *gpkg.Handle, t Table) error { + query := t.createSQL() + _, err := h.Exec(query) + if err != nil { + log.Fatalf("error building table in target GeoPackage: %s", err) + } + + err = h.AddGeometryTable(gpkg.TableDescription{ + Name: t.Name, + ShortName: t.Name, + Description: t.Name, + GeometryField: t.gcolumn, + GeometryType: t.gtype, + //nolint:gosec // G115 + SRS: int32(t.srs.ID), + // + Z: gpkg.Prohibited, + M: gpkg.Prohibited, + }) + if err != nil { + log.Println("error adding geometry table in target GeoPackage:", err) + return err + } + return nil +} + +// create table for storing encoded geometries +func buildEncodedTable(h *gpkg.Handle, t Table) error { + db := h.DB + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id INTEGER PRIMARY KEY, + tile_x INTEGER NOT NULL, + tile_y INTEGER NOT NULL, + feature_id INTEGER NOT NULL, + geometry_type INTEGER NOT NULL, + data BLOB + ) + `, t.EncodedName()) + _, err := db.Exec(query) + if err != nil { + log.Println("error adding encoded table in target GeoPackage:", err) + return err + } + return nil +} diff --git a/processing/interface.go b/processing/interface.go index 04d9a5c..7db78a7 100644 --- a/processing/interface.go +++ b/processing/interface.go @@ -14,19 +14,21 @@ type Feature interface { type FeatureForTileMatrix interface { Feature TileMatrixID() int + // EncodedGeoms is nil if encoding is not desired EncodedGeoms() []tile.EncodedGeometry } type SnapResult struct { Geometry geom.Geometry - Tiles []pointindex.Quadrant + // Tiles is nil if encoding is not desired + Tiles []pointindex.Quadrant } -type EncodedFeature struct { - Feature Feature - EncodedGeoms []tile.EncodedGeometry +type TileCoord struct { + X, Y uint } +// Source and target for snapping type Source interface { ReadFeatures(ch chan<- Feature) } @@ -34,3 +36,15 @@ type Source interface { type Target interface { WriteFeatures(ch <-chan FeatureForTileMatrix) } + +// Source and target for tile generation +type MVTSource interface { + ListTiles() ([]TileCoord, error) + GetFeaturesForTile(tileX, tileY uint) ([]tile.EncodedFeatureRow, error) + GetAttributesForFeatures(featureIDs []int64) (tile.InternalAttributeTable, error) + AttributeColumnNames() []string +} + +type MVTTarget interface { + WriteTile(x, y uint, data []byte) error +} diff --git a/processing/processing.go b/processing/processing.go index 1cd29ce..b107200 100644 --- a/processing/processing.go +++ b/processing/processing.go @@ -1,7 +1,7 @@ -// Package processing takes care of the logistics around reading and writing to a Target. -// Not the processing operation(s) itself. package processing +// Orchestrating logic around processing the snap command. + import ( "fmt" "log" @@ -128,7 +128,7 @@ func encodeGeometry(s SnapResult) []tile.EncodedGeometry { } // processFeatures processes the geometries in the features with the given function -func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTileMatrix, tmIDs []tms20.TMID, f processPolygonFunc) { +func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTileMatrix, tmIDs []tms20.TMID, f processPolygonFunc, encodeTiles bool) { stats := initStats() for { feature, hasMore := <-featuresIn @@ -144,7 +144,10 @@ func processFeatures(featuresIn <-chan Feature, featuresOut chan<- FeatureForTil stats.postCount++ } for tmID, snapResult := range newGeometriesPerTileMatrix { - encGeoms := encodeGeometry(snapResult) + var encGeoms []tile.EncodedGeometry + if encodeTiles { + encGeoms = encodeGeometry(snapResult) + } featuresOut <- wrapFeatureForTileMatrix(feature, tmID, snapResult.Geometry, encGeoms) } @@ -202,7 +205,7 @@ func writeFeaturesToTargets(featuresForTileMatrices <-chan FeatureForTileMatrix, type processPolygonFunc func(p geom.Polygon, tileMatrixIDs []tms20.TMID) map[tms20.TMID]SnapResult // ProcessFeatures applies the processing function/operation to each Target. -func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPolygonFunc) { +func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPolygonFunc, encodeTiles bool) { featuresBefore := make(chan Feature) featuresAfter := make(chan FeatureForTileMatrix) tileMatrixIDs := make([]tms20.TMID, 0, len(targets)) @@ -216,7 +219,7 @@ func ProcessFeatures(source Source, targets map[tms20.TMID]Target, f processPoly defer wg.Done() writeFeaturesToTargets(featuresAfter, targets) }() - go processFeatures(featuresBefore, featuresAfter, tileMatrixIDs, f) + go processFeatures(featuresBefore, featuresAfter, tileMatrixIDs, f, encodeTiles) go readFeaturesFromSource(source, featuresBefore) wg.Wait() diff --git a/processing/processing_test.go b/processing/processing_test.go new file mode 100644 index 0000000..f866ca9 --- /dev/null +++ b/processing/processing_test.go @@ -0,0 +1,114 @@ +package processing + +import ( + "reflect" + "testing" + + "github.com/go-spatial/geom" + "github.com/pdok/texel/pointindex" + "github.com/pdok/texel/tms20" +) + +// TestProcessGeometry covers processGeometry and processMultiPolygon +// Uses a stub processPolygonFunc +func TestProcessGeometry(t *testing.T) { + polyA := geom.Polygon{{{0, 0}, {1, 0}, {1, 1}, {0, 0}}} + polyB := geom.Polygon{{{2, 2}, {3, 2}, {3, 3}, {2, 2}}} + tileA := pointindex.Quadrant{} + tileB := pointindex.Quadrant{} + + tests := []struct { + name string + // geometry passed to processGeometry + geometry geom.Geometry + tmIDs []tms20.TMID + // callResults is a list of return values of the stub function in + // call order. + callResults []map[tms20.TMID]SnapResult + want map[tms20.TMID]SnapResult + }{ + { + name: "single Polygon, single tmID: passthrough of f's result", + geometry: polyA, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}, + }, + }, + { + name: "unsupported geometry type (Point): default branch, f not called", + geometry: geom.Point{9, 9}, + tmIDs: []tms20.TMID{1, 2}, + callResults: nil, + want: map[tms20.TMID]SnapResult{ + 1: {}, + 2: {}, + }, + }, + { + name: "nil geometry value: default branch, f not called", + geometry: nil, + tmIDs: []tms20.TMID{1}, + callResults: nil, + want: map[tms20.TMID]SnapResult{ + 1: {}, + }, + }, + { + name: "empty MultiPolygon (len 0, non-nil): f never called, empty result map", + geometry: geom.MultiPolygon{}, + tmIDs: []tms20.TMID{1, 2}, + callResults: nil, + want: map[tms20.TMID]SnapResult{}, + }, + { + name: "MultiPolygon with 1 polygon, single tmID: passthrough", + geometry: geom.MultiPolygon{polyA}, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}, + }, + }, + { + name: "MultiPolygon with 2 polygons, single tmID: merged geometry and concatenated tiles", + geometry: geom.MultiPolygon{polyA, polyB}, + tmIDs: []tms20.TMID{1}, + callResults: []map[tms20.TMID]SnapResult{ + {1: {Geometry: polyA, Tiles: []pointindex.Quadrant{tileA}}}, + {1: {Geometry: polyB, Tiles: []pointindex.Quadrant{tileB}}}, + }, + want: map[tms20.TMID]SnapResult{ + 1: {Geometry: geom.MultiPolygon{polyA, polyB}, Tiles: []pointindex.Quadrant{tileA, tileB}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + callCount := 0 + f := func(_ geom.Polygon, _ []tms20.TMID) map[tms20.TMID]SnapResult { + if callCount >= len(tt.callResults) { + t.Fatalf("unexpected call %d to f", callCount+1) + } + result := tt.callResults[callCount] + callCount++ + return result + } + + got := processGeometry(tt.geometry, tt.tmIDs, f) + + if callCount != len(tt.callResults) { + t.Errorf("f called %d times, want %d", callCount, len(tt.callResults)) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("processGeometry() = %#v, want %#v", got, tt.want) + } + }) + } +} diff --git a/processing/processingmvt.go b/processing/processingmvt.go new file mode 100644 index 0000000..f90cb8f --- /dev/null +++ b/processing/processingmvt.go @@ -0,0 +1,46 @@ +package processing + +// Orchestrating functionality for the mvt command + +import ( + "fmt" + + "github.com/pdok/texel/tile" +) + +// Orchestrator for the tile creation pipeline +func BuildAndWriteMVTTiles(source MVTSource, target MVTTarget, tableName string) error { + // Build key index (columns) just once. + keyIndex := tile.BuildKeyDictionary(source.AttributeColumnNames()) + + tiles, err := source.ListTiles() + if err != nil { + return err + } + + for _, tc := range tiles { + encFeatRows, err := source.GetFeaturesForTile(tc.X, tc.Y) + if err != nil { + return fmt.Errorf("getting features for tile (%d,%d): %w", tc.X, tc.Y, err) + } + featIDs := make([]int64, len(encFeatRows)) + for i, encFeatRow := range encFeatRows { + featIDs[i] = encFeatRow.FeatureID + } + + attributes, err := source.GetAttributesForFeatures(featIDs) + if err != nil { + return fmt.Errorf("getting attributes for tile (%d,%d): %w", tc.X, tc.Y, err) + } + + data, err := tile.BuildMVTTile(tableName, keyIndex, encFeatRows, attributes) + if err != nil { + return fmt.Errorf("building tile (%d,%d): %w", tc.X, tc.Y, err) + } + + if err := target.WriteTile(tc.X, tc.Y, data); err != nil { + return fmt.Errorf("writing tile (%d,%d): %w", tc.X, tc.Y, err) + } + } + return nil +} diff --git a/snap/snap.go b/snap/snap.go index 226d130..ae6fd0e 100644 --- a/snap/snap.go +++ b/snap/snap.go @@ -71,8 +71,6 @@ func SnapPolygon(polygon geom.Polygon, tileMatrixSet tms20.TileMatrixSet, tmIDs var tilesbbox []pointindex.Quadrant if config.EncodeTiles { tilesbbox = ix.GetPrimitiveQBBox(pointindex.Level(tmIDsByLevels[level])) //nolint:gosec // G115 These are numbers < 40 - } else { - tilesbbox = []pointindex.Quadrant{} } newGeometry := geomhelp.PolygonSliceToGeom(newPolygons) newPolygonsPerTileMatrixID[tmIDsByLevels[level]] = processing.SnapResult{Geometry: newGeometry, Tiles: tilesbbox} diff --git a/tile/assemble.go b/tile/assemble.go new file mode 100644 index 0000000..1ffc0fd --- /dev/null +++ b/tile/assemble.go @@ -0,0 +1,282 @@ +package tile + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/go-spatial/geom" + "github.com/go-spatial/geom/encoding/mvt" + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" + oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // needed: matches the reflection-based marshaling vector_tile.pb.go relies on + "github.com/pdok/texel/pointindex" +) + +const ( + precision = 4096 + mvtVersion = 2 +) + +type EncodedGeometry struct { + Encoding []uint32 + GeometryType int32 + XTile uint + YTile uint +} + +// EncodedFeatureRow pairs a feature's identifier with its encoded, tile-relative geometry +// as read back from the "
_encoded" table. +type EncodedFeatureRow struct { + FeatureID int64 + Geom EncodedGeometry +} + +// Transform geometry to tile extent, then encode. We assume the geometry is +// snapped to the proposed grid, in which case makevalid operations should not +// be necessary. +func MvtEncodeGeometry(q pointindex.Quadrant, g geom.Geometry) EncodedGeometry { + ext := q.Extent() + preparedGeo := mvt.PrepareGeo(g, &ext, float64(precision)) + + // This should not be necessary. + // sg, err := convert.ToTegola(preparedGeo) + // tegolaGeo, err := validate.CleanGeometry(context.TODO(), sg, &ext) + // validatedGeo := convert.ToGeom(tegolaGeo) + + encgeom, geomtype, err := EncodeGeometry(preparedGeo) + if err != nil { + panic(err) + } + + xTile, yTile := q.Coords() + + return EncodedGeometry{ + Encoding: encgeom, + GeometryType: int32(geomtype), + XTile: xTile, + YTile: yTile, + } +} + +// Serialze tile into protobuf format +func MarshalTile(t *vectorTile.Tile) ([]byte, error) { + return oldproto.Marshal(t) +} + +// Internal representation of attributes: read table[fid][columnName]=value. +type InternalAttributeTable map[int64]map[string]any + +// Build indices from column names +func BuildKeyDictionary(columnNames []string) map[string]uint32 { + index := make(map[string]uint32, len(columnNames)) + for i, name := range columnNames { + if i > int(^uint32(0)) { + panic("Number of keys does not fit in uint32") + } + index[name] = uint32(i) //nolint:gosec // G115 + } + return index +} + +// Build index table for values. This function guarantees that all indices from +// 0 to len(index)-1 all appear exactly once. Absent attributes are not recorded. +func BuildValueDictionary(attributesPerFeature InternalAttributeTable) map[any]uint32 { + index := make(map[any]uint32) + for _, attributes := range attributesPerFeature { + for _, v := range attributes { + if v == nil { + continue + } + if _, ok := index[v]; ok { + continue + } + l := len(index) + if l > int(^uint32(0)) { + panic("Number of values does not fit in uint32") + } + index[v] = uint32(l) + } + } + return index +} + +// Convert index map to slice. Assumes all indices 0, 1, ..., len(index)-1 are +// used exactly once. Necessary since iteration over maps is in arbitrary order. +// This is a generic function since we need it with K = string and K = any. +func orderedByIndex[K comparable](index map[K]uint32) []K { + ordered := make([]K, len(index)) + for k, i := range index { + ordered[i] = k + } + return ordered +} + +// Construct tags for a single feature. `keyIndex` and `valueIndex` are built +// by above functions by considering all features. `attributes` are the +// attributes of the current feature. Absent values are skipped. Panics if a +// key of value is not present. +func buildTags(attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) ([]uint32, error) { + tags := make([]uint32, 0, 2*len(attributes)) + for key, val := range attributes { + if val == nil { + continue + } + + kidx, ok := keyIndex[key] + if !ok { + return nil, fmt.Errorf("did not find key (%v) in key dictionary", key) + } + + vidx, ok := valueIndex[val] + if !ok { + return nil, fmt.Errorf("did not find value (%v) in value dictionary", val) + } + + tags = append(tags, kidx, vidx) + } + return tags, nil +} + +// Construct feature according to protobuf format. +func BuildFeature(featureID int64, geom EncodedGeometry, attributes map[string]any, keyIndex map[string]uint32, valueIndex map[any]uint32) (*vectorTile.Tile_Feature, error) { + tags, err := buildTags(attributes, keyIndex, valueIndex) + if err != nil { + return nil, fmt.Errorf("feature %d: %w", featureID, err) + } + + //nolint:gosec // G115 feature ids are expected to be non-negative + id := uint64(featureID) + gtype := vectorTile.Tile_GeomType(geom.GeometryType) + + return &vectorTile.Tile_Feature{ + Id: &id, + Tags: tags, + Type: >ype, + Geometry: geom.Encoding, + }, nil +} + +// Construct layer according to protobuf format +func BuildLayer(name string, features []*vectorTile.Tile_Feature, keyIndex map[string]uint32, valueIndex map[any]uint32) *vectorTile.Tile_Layer { + keys := orderedByIndex(keyIndex) + + rawValues := orderedByIndex(valueIndex) + values := make([]*vectorTile.Tile_Value, len(rawValues)) + for i, v := range rawValues { + values[i] = vectorTileValue(v) + } + + version := uint32(mvtVersion) + extent := uint32(precision) + layerName := name + + return &vectorTile.Tile_Layer{ + Version: &version, + Name: &layerName, + Features: features, + Keys: keys, + Values: values, + Extent: &extent, + } +} + +// Construct tile according to protobuf format +func BuildTile(layers ...*vectorTile.Tile_Layer) *vectorTile.Tile { + return &vectorTile.Tile{Layers: layers} +} + +// Main exported function that encodes a tile of a single layer. Assumes +// geometry and keyIndex have already been encoded. Returns marshalled byte +// string. +func BuildMVTTile(layerName string, keyIndex map[string]uint32, encFeatRows []EncodedFeatureRow, attributes InternalAttributeTable) ([]byte, error) { + valueIndex := BuildValueDictionary(attributes) + + features := make([]*vectorTile.Tile_Feature, 0, len(encFeatRows)) + for _, encodedFeature := range encFeatRows { + featureID := encodedFeature.FeatureID + attrs := attributesForFeature(attributes, featureID) + feature, err := BuildFeature(featureID, encodedFeature.Geom, attrs, keyIndex, valueIndex) + if err != nil { + return nil, err + } + features = append(features, feature) + } + + layer := BuildLayer(layerName, features, keyIndex, valueIndex) + + data, err := MarshalTile(BuildTile(layer)) + if err != nil { + return nil, fmt.Errorf("marshaling tile: %w", err) + } + return data, nil +} + +// attributesForFeature extracts a single feature's attributes from the map construct. +func attributesForFeature(attributes InternalAttributeTable, featureID int64) map[string]any { + if attrs, ok := attributes[featureID]; ok { + return attrs + } + return map[string]any{} +} + +// Convert value to Tile_Value. Copied from go-spatial/encoding/mvt/layer.go +func vectorTileValue(i any) *vectorTile.Tile_Value { //nolint:cyclop, funlen + tv := new(vectorTile.Tile_Value) + switch t := i.(type) { + default: + buff := new(bytes.Buffer) + err := binary.Write(buff, binary.BigEndian, t) + // We are going to ignore the value and return an empty TileValue + if err == nil { + tv.XXX_unrecognized = buff.Bytes() + } + + case string: + tv.StringValue = &t + + case fmt.Stringer: + str := t.String() + tv.StringValue = &str + + case bool: + tv.BoolValue = &t + + case int8: + intv := int64(t) + tv.SintValue = &intv + + case int16: + intv := int64(t) + tv.SintValue = &intv + + case int32: + intv := int64(t) + tv.SintValue = &intv + + case int64: + tv.IntValue = &t + + case uint8: + intv := int64(t) + tv.SintValue = &intv + + case uint16: + intv := int64(t) + tv.SintValue = &intv + + case uint32: + intv := int64(t) + tv.SintValue = &intv + + case uint64: + tv.UintValue = &t + + case float32: + tv.FloatValue = &t + + case float64: + tv.DoubleValue = &t + + } // switch + return tv +} diff --git a/tile/assemble_test.go b/tile/assemble_test.go new file mode 100644 index 0000000..79aecfb --- /dev/null +++ b/tile/assemble_test.go @@ -0,0 +1,236 @@ +package tile + +import ( + "sort" + "testing" + + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" + oldproto "github.com/golang/protobuf/proto" //nolint:staticcheck // matches MarshalTile's use of the reflection-based marshaler + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Quick test BuildKeyDictionary +func TestBuildKeyDictionary(t *testing.T) { + tests := []struct { + name string + columnNames []string + want map[string]uint32 + }{ + { + name: "empty slice yields empty map", + columnNames: []string{}, + want: map[string]uint32{}, + }, + { + name: "single column", + columnNames: []string{"a"}, + want: map[string]uint32{"a": 0}, + }, + { + name: "multiple columns keep their position as index", + columnNames: []string{"a", "b", "c"}, + want: map[string]uint32{"a": 0, "b": 1, "c": 2}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildKeyDictionary(tt.columnNames) + assert.Equal(t, tt.want, got) + }) + } +} + +// indexedKeys returns the keys of index sorted by their assigned uint32, +// i.e. it undoes the (arbitrary, since built from a map) assignment of +// indices so tests can assert on order-independent content. +func indexedKeys(index map[any]uint32) []any { + keys := make([]any, len(index)) + for k, i := range index { + keys[i] = k + } + return keys +} + +func TestBuildValueDictionary(t *testing.T) { + tests := []struct { + name string + attributesPerFeature InternalAttributeTable + wantValues []any // expected distinct, non-nil values (order-independent) + }{ + { + name: "empty table yields empty dictionary", + attributesPerFeature: InternalAttributeTable{}, + wantValues: []any{}, + }, + { + name: "single feature, single attribute", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo"}, + }, + wantValues: []any{"foo"}, + }, + { + name: "shared values across features are only counted once", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo"}, + 2: {"name": "foo"}, + 3: {"name": "bar"}, + }, + wantValues: []any{"foo", "bar"}, + }, + { + name: "nil values are skipped", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo", "optional": nil}, + }, + wantValues: []any{"foo"}, + }, + { + name: "mixed value types are all indexed", + attributesPerFeature: InternalAttributeTable{ + 1: {"name": "foo", "count": int64(3), "active": true}, + }, + wantValues: []any{"foo", int64(3), true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildValueDictionary(tt.attributesPerFeature) + + // Every index from 0..len(got)-1 must be used exactly once. + assert.Len(t, got, len(tt.wantValues)) + gotValues := indexedKeys(got) + assert.ElementsMatch(t, tt.wantValues, gotValues) + }) + } +} + +func TestBuildTags(t *testing.T) { + keyIndex := map[string]uint32{"name": 0, "count": 1} + valueIndex := map[any]uint32{"foo": 0, int64(3): 1} + + t.Run("empty attributes yield empty tags", func(t *testing.T) { + tags, err := buildTags(map[string]any{}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Empty(t, tags) + }) + + t.Run("single attribute produces a key/value index pair", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo"}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Equal(t, []uint32{0, 0}, tags) + }) + + t.Run("nil-valued attribute is skipped", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo", "count": nil}, keyIndex, valueIndex) + require.NoError(t, err) + assert.Equal(t, []uint32{0, 0}, tags) + }) + + t.Run("multiple attributes produce all pairs", func(t *testing.T) { + tags, err := buildTags(map[string]any{"name": "foo", "count": int64(3)}, keyIndex, valueIndex) + require.NoError(t, err) + + // Tags is built from map iteration, so pair order is not + // guaranteed; compare as a set of (key, value) pairs instead. + require.Len(t, tags, 4) + pairs := map[[2]uint32]bool{} + for i := 0; i < len(tags); i += 2 { + pairs[[2]uint32{tags[i], tags[i+1]}] = true + } + assert.True(t, pairs[[2]uint32{0, 0}]) + assert.True(t, pairs[[2]uint32{1, 1}]) + }) + + t.Run("missing key returns an error", func(t *testing.T) { + _, err := buildTags(map[string]any{"unknown": "foo"}, keyIndex, valueIndex) + require.Error(t, err) + }) + + t.Run("missing value returns an error", func(t *testing.T) { + _, err := buildTags(map[string]any{"name": "unindexed value"}, keyIndex, valueIndex) + require.Error(t, err) + }) +} + +func TestBuildLayer(t *testing.T) { + t.Run("happy path building layer", func(t *testing.T) { + keyIndex := map[string]uint32{"b": 1, "a": 0} + valueIndex := map[any]uint32{"second": 1, "first": 0} + features := []*vectorTile.Tile_Feature{{}, {}} + + layer := BuildLayer("mylayer", features, keyIndex, valueIndex) + + assert.Equal(t, "mylayer", layer.GetName()) + assert.Equal(t, uint32(mvtVersion), layer.GetVersion()) + assert.Equal(t, uint32(precision), layer.GetExtent()) + assert.Same(t, features[0], layer.GetFeatures()[0]) + assert.Same(t, features[1], layer.GetFeatures()[1]) + + // Keys/values must be ordered according to their assigned index, + // regardless of the (arbitrary) map iteration order used to build them. + assert.Equal(t, []string{"a", "b"}, layer.GetKeys()) + require.Len(t, layer.GetValues(), 2) + assert.Equal(t, "first", layer.GetValues()[0].GetStringValue()) + assert.Equal(t, "second", layer.GetValues()[1].GetStringValue()) + }) + t.Run("empty layer", func(t *testing.T) { + layer := BuildLayer("empty", nil, map[string]uint32{}, map[any]uint32{}) + + assert.Equal(t, "empty", layer.GetName()) + assert.Empty(t, layer.GetKeys()) + assert.Empty(t, layer.GetValues()) + assert.Empty(t, layer.GetFeatures()) + }) +} + +func TestBuildMVTTile(t *testing.T) { + t.Run("happy path roundtrips features, tags and attribute values", func(t *testing.T) { + keyIndex := BuildKeyDictionary([]string{"name"}) + attributes := InternalAttributeTable{ + 1: {"name": "foo"}, + 2: {"name": "bar"}, + } + encFeatRows := []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + {FeatureID: 2, Geom: EncodedGeometry{GeometryType: int32(vectorTile.Tile_POINT)}}, + } + + data, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) + require.NoError(t, err) + require.NotEmpty(t, data) + + got := &vectorTile.Tile{} + require.NoError(t, oldproto.Unmarshal(data, got)) + require.Len(t, got.GetLayers(), 1) + + layer := got.GetLayers()[0] + assert.Equal(t, "mylayer", layer.GetName()) + require.Len(t, layer.GetFeatures(), 2) + + // Collect the string values assigned to each feature via its tags, + // independent of feature/value ordering. + gotNames := make([]string, 0, 2) + for _, feature := range layer.GetFeatures() { + require.Len(t, feature.GetTags(), 2) + valueIdx := feature.GetTags()[1] + gotNames = append(gotNames, layer.GetValues()[valueIdx].GetStringValue()) + } + sort.Strings(gotNames) + assert.Equal(t, []string{"bar", "foo"}, gotNames) + }) + + t.Run("error building a feature's tags propagates", func(t *testing.T) { + keyIndex := map[string]uint32{"name": 0} + attributes := InternalAttributeTable{ + 1: {"unknown-key": "foo"}, + } + encFeatRows := []EncodedFeatureRow{ + {FeatureID: 1, Geom: EncodedGeometry{}}, + } + + _, err := BuildMVTTile("mylayer", keyIndex, encFeatRows, attributes) + require.Error(t, err) + }) +} diff --git a/tile/go_spatial_feature.go b/tile/go_spatial_encoding.go similarity index 55% rename from tile/go_spatial_feature.go rename to tile/go_spatial_encoding.go index 7d348a6..5835943 100644 --- a/tile/go_spatial_feature.go +++ b/tile/go_spatial_encoding.go @@ -1,12 +1,11 @@ package tile -// This file was copied from go-spatial/geom/encoding/mvt/feature.go -// Modifications made: -// - Made EncodeGeometry an exported function. -// - Made file self-contained by adding type definitions from -// go-spatial/geom/encoding/mvt/vector_tile/vector_tile.pb.go -// - Made file self-contained by defining the debug variable -// - Removed unused parameter "context" from all functions. +// This is the geometry encoding logic copied from go-spatial/geom/encoding/mvt +// It was necessary to take this step, since usually EncodeGeometry is not an +// exported function. Modifications made: +// - Added `const debug = false` which is usually present in another file. +// - Addressed linter issues +// - Removed `context` parameter. import ( "errors" @@ -14,59 +13,11 @@ import ( "log" "github.com/go-spatial/geom" + vectorTile "github.com/go-spatial/geom/encoding/mvt/vector_tile" "github.com/go-spatial/geom/encoding/wkt" "github.com/go-spatial/geom/winding" ) -// START copied from vector_tile.pb.go - -type GSTileGeomType int32 - -const ( - TileUNKNOWN GSTileGeomType = 0 - TilePOINT GSTileGeomType = 1 - TileLINESTRING GSTileGeomType = 2 - TilePOLYGON GSTileGeomType = 3 -) - -var TileGeomTypeName = map[int32]string{ - 0: "UNKNOWN", - 1: "POINT", - 2: "LINESTRING", - 3: "POLYGON", -} - -var TileGeomTypeValue = map[string]int32{ - "UNKNOWN": 0, - "POINT": 1, - "LINESTRING": 2, - "POLYGON": 3, -} - -var ( - ErrNilFeature = errors.New("feature is nil") - ErrUnknownGeometryType = errors.New("unknown geometry type") - ErrNilGeometryType = errors.New("geometry is nil") -) - -type GSTileFeature struct { - Id *uint64 `protobuf:"varint,1,opt,name=id,def=0" json:"id,omitempty"` - // Tags of this feature are encoded as repeated pairs of - // integers. - // A detailed description of tags is located in sections - // 4.2 and 4.4 of the specification - Tags []uint32 `protobuf:"varint,2,rep,packed,name=tags" json:"tags,omitempty"` - // The type of geometry stored in this feature. - Type *GSTileGeomType `protobuf:"varint,3,opt,name=type,enum=vector_tile.Tile_GeomType,def=0" json:"type,omitempty"` - // Contains a stream of commands and parameters (vertices). - // A detailed description on geometry encoding is located in - // section 4.3 of the specification. - Geometry []uint32 `protobuf:"varint,4,rep,packed,name=geometry" json:"geometry,omitempty"` - XXXunrecognized []byte `json:"-"` -} - -// END copied from vector_tile.pb.go - // Definition needed for compilation const debug = false @@ -80,70 +31,12 @@ const debug = false // Feature describes a feature of a Layer. A layer will contain multiple features // each of which has a geometry describing the interesting thing, and the metadata // associated with it. -type Feature struct { //nolint:recvcheck - ID *uint64 - Tags map[string]any - Geometry geom.Geometry -} - -func (f Feature) String() string { - g, err := wkt.EncodeString(f.Geometry) - if err != nil { - return fmt.Sprintf("encoding error for geom geom, %v", err) - } - - if f.ID != nil { - return fmt.Sprintf("{Feature: %v, GEO: %v, Tags: %+v}", *f.ID, g, f.Tags) - } - - return fmt.Sprintf("{Feature: GEO: %v, Tags: %+v}", g, f.Tags) -} - -// NewFeatures returns one or more features for the given Geometry -func NewFeatures(geo geom.Geometry, tags map[string]any) (f []Feature) { - if geo == nil { - return f // return empty feature set for a nil geometry - } - - if g, ok := geo.(geom.Collection); ok { - geos := g.Geometries() - for i := range geos { - f = append(f, NewFeatures(geos[i], tags)...) - } - return f - } - f = append(f, Feature{ - Tags: tags, - Geometry: geo, - }) - - return f -} - -// VTileFeature will return a vectorTile.Feature that would represent the Feature -func (f *Feature) VTileFeature(keys []string, vals []any) (tf *GSTileFeature, err error) { - tf = new(GSTileFeature) - tf.Id = f.ID - - if tf.Tags, err = keyvalTagsMap(keys, vals, f); err != nil { - return tf, err - } - - geo, gtype, err := EncodeGeometry(f.Geometry) - if err != nil { - return tf, err - } - - if len(geo) == 0 { - return nil, nil - } - - tf.Geometry = geo - tf.Type = >ype - - return tf, nil -} +var ( + ErrNilFeature = errors.New("feature is nil") + ErrUnknownGeometryType = errors.New("unknown geometry type") + ErrNilGeometryType = errors.New("geometry is nil") +) // These values came from: https://github.com/mapbox/vector-tile-spec/tree/master/2.1 const ( @@ -349,9 +242,9 @@ func (c *cursor) encodePolygon(geo geom.Polygon) []uint32 { // EncodeGeometry will take a geom.Geometry and encode it according to the // mapbox vector_tile spec. -func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, err error) { +func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp vectorTile.Tile_GeomType, err error) { if geometry == nil { - return nil, TileUNKNOWN, ErrNilGeometryType + return nil, vectorTile.Tile_UNKNOWN, ErrNilGeometryType } c := NewCursor() @@ -359,17 +252,17 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er switch t := geometry.(type) { case geom.Point: g = append(g, c.MoveTo(t)...) - return g, TilePOINT, nil + return g, vectorTile.Tile_POINT, nil case geom.MultiPoint: g = append(g, c.MoveTo(t.Points()...)...) - return g, TilePOINT, nil + return g, vectorTile.Tile_POINT, nil case geom.LineString: points := t.Vertices() g = append(g, c.MoveTo(points[0])...) g = append(g, c.LineTo(points[1:]...)...) - return g, TileLINESTRING, nil + return g, vectorTile.Tile_LINESTRING, nil case geom.MultiLineString: lines := t.LineStrings() @@ -378,159 +271,31 @@ func EncodeGeometry(geometry geom.Geometry) (g []uint32, vtyp GSTileGeomType, er g = append(g, c.MoveTo(points[0])...) g = append(g, c.LineTo(points[1:]...)...) } - return g, TileLINESTRING, nil + return g, vectorTile.Tile_LINESTRING, nil case geom.Polygon: g = append(g, c.encodePolygon(t)...) - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil case geom.MultiPolygon: polygons := t.Polygons() for _, p := range polygons { g = append(g, c.encodePolygon(p)...) } - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil case *geom.MultiPolygon: if t == nil { - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil } polygons := t.Polygons() for _, p := range polygons { g = append(g, c.encodePolygon(p)...) } - return g, TilePOLYGON, nil + return g, vectorTile.Tile_POLYGON, nil default: - return nil, TileUNKNOWN, ErrUnknownGeometryType + return nil, vectorTile.Tile_UNKNOWN, ErrUnknownGeometryType } } - -// keyvalTagsMap will return the tags map as expected by the mapbox tile spec. It takes -// a keyMap and a valueMap that list the order of the expected keys and values. It will -// return a vector map that refers to these two maps. -func keyvalTagsMap(keyMap []string, valueMap []any, f *Feature) (tags []uint32, err error) { //nolint: cyclop,funlen - if f == nil { - return nil, ErrNilFeature - } - - var kidx, vidx int64 - - for key, val := range f.Tags { - - kidx, vidx = -1, -1 // Set to known not found value. - - for i, k := range keyMap { - if k != key { - continue // move to the next key - } - kidx = int64(i) - break // we found a match - } - - if kidx == -1 { - log.Printf("did not find key (%v) in keymap.", key) - return tags, fmt.Errorf("did not find key (%v) in keymap", key) - } - - // if val is nil we skip it for now - // https://github.com/mapbox/vector-tile-spec/issues/62 - if val == nil { - continue - } - - for i, v := range valueMap { - switch tv := val.(type) { - default: - return tags, fmt.Errorf("value (%[1]v) of type (%[1]T) for key (%[2]v) is not supported", tv, key) - case string: - vmt, ok := v.(string) // Make sure the type of the Value map matches the type of the Tag's value - if !ok || vmt != tv { // and that the values match - continue // if they don't match move to the next value. - } - case fmt.Stringer: - vmt, ok := v.(fmt.Stringer) - if !ok || vmt.String() != tv.String() { - continue - } - case int: - vmt, ok := v.(int) - if !ok || vmt != tv { - continue - } - case int8: - vmt, ok := v.(int8) - if !ok || vmt != tv { - continue - } - case int16: - vmt, ok := v.(int16) - if !ok || vmt != tv { - continue - } - case int32: - vmt, ok := v.(int32) - if !ok || vmt != tv { - continue - } - case int64: - vmt, ok := v.(int64) - if !ok || vmt != tv { - continue - } - case uint: - vmt, ok := v.(uint) - if !ok || vmt != tv { - continue - } - case uint8: - vmt, ok := v.(uint8) - if !ok || vmt != tv { - continue - } - case uint16: - vmt, ok := v.(uint16) - if !ok || vmt != tv { - continue - } - case uint32: - vmt, ok := v.(uint32) - if !ok || vmt != tv { - continue - } - case uint64: - vmt, ok := v.(uint64) - if !ok || vmt != tv { - continue - } - - case float32: - vmt, ok := v.(float32) - if !ok || vmt != tv { - continue - } - case float64: - vmt, ok := v.(float64) - if !ok || vmt != tv { - continue - } - case bool: - vmt, ok := v.(bool) - if !ok || vmt != tv { - continue - } - } // Values Switch Statement - // if the values match let's record the index. - vidx = int64(i) - break // we found our value no need to continue on. - } // range on value - - if vidx == -1 { // None of the values matched. - return tags, fmt.Errorf("did not find a value: %v in valuemap", val) - } - tags = append(tags, uint32(kidx), uint32(vidx)) //nolint: gosec // G115 casting sems unavoidable - } // Move to the next tag key and value. - - return tags, nil -} diff --git a/tile/tile.go b/tile/tile.go deleted file mode 100644 index 2310eb8..0000000 --- a/tile/tile.go +++ /dev/null @@ -1,40 +0,0 @@ -package tile - -import ( - "github.com/go-spatial/geom" - "github.com/go-spatial/geom/encoding/mvt" - "github.com/pdok/texel/pointindex" -) - -const precision = 4096 - -type EncodedGeometry struct { - Encoding []uint32 - GeometryType int32 - XTile uint - YTile uint -} - -func MvtEncodeGeometry(q pointindex.Quadrant, g geom.Geometry) EncodedGeometry { - ext := q.Extent() - preparedGeo := mvt.PrepareGeo(g, &ext, float64(precision)) - - // This should not be necessary. - // sg, err := convert.ToTegola(preparedGeo) - // tegolaGeo, err := validate.CleanGeometry(context.TODO(), sg, &ext) - // validatedGeo := convert.ToGeom(tegolaGeo) - - encgeom, geomtype, err := EncodeGeometry(preparedGeo) - if err != nil { - panic(err) - } - - xTile, yTile := q.Coords() - - return EncodedGeometry{ - Encoding: encgeom, - GeometryType: int32(geomtype), - XTile: xTile, - YTile: yTile, - } -}