diff --git a/encoding/mvt/README.md b/encoding/mvt/README.md index 90e928c..d46b9b4 100644 --- a/encoding/mvt/README.md +++ b/encoding/mvt/README.md @@ -13,9 +13,11 @@ type Layer struct { } func MarshalGzipped(layers Layers) ([]byte, error) +func MarshalBrotli(layers Layers) ([]byte, error) func Marshal(layers Layers) ([]byte, error) func UnmarshalGzipped(data []byte) (Layers, error) +func UnmarshalBrotli(data []byte) (Layers, error) func Unmarshal(data []byte) (Layers, error) ``` diff --git a/encoding/mvt/marshal.go b/encoding/mvt/marshal.go index 22640f2..be0c0a8 100644 --- a/encoding/mvt/marshal.go +++ b/encoding/mvt/marshal.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" + "github.com/andybalholm/brotli" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/mvt/vectortile" "github.com/paulmach/orb/geojson" @@ -39,6 +40,28 @@ func MarshalGzipped(layers Layers) ([]byte, error) { return buf.Bytes(), nil } +// MarshalBrotli will marshal the layers into Mapbox Vector Tile format +// and brotli compress the result. +func MarshalBrotli(layers Layers) ([]byte, error) { + data, err := Marshal(layers) + if err != nil { + return nil, err + } + + buf := bytes.NewBuffer(nil) + brwriter := brotli.NewWriter(buf) + + if _, err = brwriter.Write(data); err != nil { + return nil, err + } + + if err := brwriter.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + // MarshalToVectorTile will take a set of layers and encode them into a Mapbox Vector Tile proto structure. // Features that have a nil geometry, for some reason, will be skipped and not included. func MarshalToVectorTile(layers Layers) (*vectortile.Tile, error) { diff --git a/encoding/mvt/marshal_test.go b/encoding/mvt/marshal_test.go index e30812d..e78c276 100644 --- a/encoding/mvt/marshal_test.go +++ b/encoding/mvt/marshal_test.go @@ -69,6 +69,61 @@ func TestMarshalUnmarshalGzipped_Full(t *testing.T) { compareOrbGeometry(t, result.Geometry, expected, xe, ye) } +func TestMarshalUnmarshalBrotli_Full(t *testing.T) { + tile := maptile.New(8956, 12223, 15) + ls := orb.LineString{ + {-81.60346275, 41.50998572}, + {-81.6033669, 41.50991259}, + {-81.60355599, 41.50976036}, + {-81.6040648, 41.50936811}, + {-81.60404411, 41.50935405}, + } + expected := ls.Clone() + + f := geojson.NewFeature(ls) + f.Properties = geojson.Properties{ + "source": "openstreetmap.org", + "kind": "path", + "name": "Uptown Alley", + "landuse_kind": "retail", + "sort_rank": float64(354), + "kind_detail": "pedestrian", + "min_zoom": float64(13), + "id": float64(246698394), + } + + fc := geojson.NewFeatureCollection() + fc.Append(f) + + layers := Layers{NewLayer("roads", fc)} + + // project to the tile coords + layers.ProjectToTile(tile) + + // marshal + encoded, err := MarshalBrotli(layers) + if err != nil { + t.Fatalf("marshal error: %v", err) + } + + // unmarshal + decoded, err := UnmarshalBrotli(encoded) + if err != nil { + t.Fatalf("unmarshal error: %v", err) + } + + // project back + decoded.ProjectToWGS84(tile) + + // compare the results + result := decoded[0].Features[0] + compareProperties(t, result.Properties, f.Properties) + + // compare geometry + xe, ye := tileEpsilon(tile) + compareOrbGeometry(t, result.Geometry, expected, xe, ye) +} + func TestMarshalUnmarshalForGeometryCollection(t *testing.T) { tile := maptile.New(8956, 12223, 15) outerRing := orb.Ring{ @@ -109,23 +164,39 @@ func TestMarshalUnmarshalForGeometryCollection(t *testing.T) { // project to the tile coords layers.ProjectToTile(tile) - // marshal - encoded, err := MarshalGzipped(layers) + // marshal gzip + encodedGzip, err := MarshalGzipped(layers) if err != nil { - t.Fatalf("marshal error: %v", err) + t.Fatalf("marshal gzip error: %v", err) } - // unmarshal - decoded, err := UnmarshalGzipped(encoded) + // unmarshal gzip + decodedGzip, err := UnmarshalGzipped(encodedGzip) if err != nil { - t.Fatalf("unmarshal error: %v", err) + t.Fatalf("unmarshal gzip error: %v", err) + } + + // compare the results + results := decodedGzip[0].Features + compareProperties(t, results[0].Properties, f.Properties) + + // marshal brotli + encodedBrotli, err := MarshalBrotli(layers) + if err != nil { + t.Fatalf("marshal brotli error: %v", err) + } + + // unmarshal brotli + decodedBrotli, err := UnmarshalBrotli(encodedBrotli) + if err != nil { + t.Fatalf("unmarshal brotli error: %v", err) } // project back - decoded.ProjectToWGS84(tile) + decodedBrotli.ProjectToWGS84(tile) // compare the results - results := decoded[0].Features + results = decodedBrotli[0].Features compareProperties(t, results[0].Properties, f.Properties) // compare geometry diff --git a/encoding/mvt/unmarshal.go b/encoding/mvt/unmarshal.go index 7454850..343a48b 100644 --- a/encoding/mvt/unmarshal.go +++ b/encoding/mvt/unmarshal.go @@ -5,8 +5,10 @@ import ( "compress/gzip" "errors" "fmt" + "io" "io/ioutil" + "github.com/andybalholm/brotli" "github.com/paulmach/orb" "github.com/paulmach/orb/encoding/mvt/vectortile" "github.com/paulmach/orb/geojson" @@ -31,6 +33,19 @@ func UnmarshalGzipped(data []byte) (Layers, error) { return Unmarshal(decoded) } +// UnmarshalBrotli takes brotli compressed Mapbox Vector Tile (MVT) data and uncompresses it +// before decoding it into a set of layers, It does not project the coordinates. +func UnmarshalBrotli(data []byte) (Layers, error) { + brreader := brotli.NewReader(bytes.NewBuffer(data)) + + decoded, err := io.ReadAll(brreader) + if err != nil { + return nil, fmt.Errorf("failed to unzip: %v", err) + } + + return Unmarshal(decoded) +} + // Unmarshal takes Mapbox Vector Tile (MVT) data and converts into a // set of layers, It does not project the coordinates. func Unmarshal(data []byte) (Layers, error) { diff --git a/go.mod b/go.mod index 8f99949..be47132 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,9 @@ module github.com/paulmach/orb -go 1.15 +go 1.22 require ( + github.com/andybalholm/brotli v1.2.0 github.com/gogo/protobuf v1.3.2 github.com/paulmach/protoscan v0.2.1 go.mongodb.org/mongo-driver v1.11.4 diff --git a/go.sum b/go.sum index d1a75cb..382e7c2 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -11,10 +13,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8= @@ -30,6 +30,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -68,13 +70,11 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=