From 6f45315c75eb32b7280c8673891eb2ae028acfb5 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 11 Nov 2022 10:56:50 +0100 Subject: [PATCH] WIP: Add code to generalize data using different strategies This commit adds the code to generalize various types of data using different strategies. The following strategies work on a tile-by-tile basis and operate on polygons: The "vector-union" strategy buffers and unionizes polygons using vector operations. The "raster-union" strategy does a similar thing but does it in raster space which is much faster. First the polygons are rendered into a raster, an open/close operation is called (which basically does the same thing as the buffering in vector space) and finally the resulting raster is vectorized again. The "builtup" strategy is intended to derive a layer of builtup areas from landuse=residential/industrial etc. as well as building cover and dense road networks. This still needs some work... Also a new "discrete-isolation" strategy which rates places based on some importance metric. (This is not tile-based.) The new "rivers" strategy finds important rivers, this is still very much work in progress. For the raster support this adds two new library dependency: CImg and potrace. The functionality is accessed through a new command line program called osm2pgsql-gen. It reads the same Lua config file that osm2pgsql reads. Call it with -h to get some usage information. This program is for testing only, eventually the functionality should be accessible from osm2pgsql itself. See also https://osm2pgsql.org/generalization/ . --- .../actions/ubuntu-prerequisites/action.yml | 2 + .github/actions/win-install/action.yml | 3 +- .github/workflows/ci.yml | 4 +- .github/workflows/test-install.yml | 2 + CMakeLists.txt | 33 +- README.md | 5 +- src/gen/canvas.cpp | 130 ++++ src/gen/canvas.hpp | 88 +++ src/gen/gen-base.cpp | 108 +++ src/gen/gen-base.hpp | 95 +++ src/gen/gen-create.cpp | 45 ++ src/gen/gen-create.hpp | 26 + src/gen/gen-discrete-isolation.cpp | 147 ++++ src/gen/gen-discrete-isolation.hpp | 37 + src/gen/gen-rivers.cpp | 347 +++++++++ src/gen/gen-rivers.hpp | 43 ++ src/gen/gen-tile-builtup.cpp | 280 +++++++ src/gen/gen-tile-builtup.hpp | 52 ++ src/gen/gen-tile-raster.cpp | 249 +++++++ src/gen/gen-tile-raster.hpp | 49 ++ src/gen/gen-tile-vector.cpp | 94 +++ src/gen/gen-tile-vector.hpp | 37 + src/gen/gen-tile.cpp | 49 ++ src/gen/gen-tile.hpp | 36 + src/gen/osm2pgsql-gen.cpp | 683 ++++++++++++++++++ src/gen/params.cpp | 125 ++++ src/gen/params.hpp | 96 +++ src/gen/raster.cpp | 66 ++ src/gen/raster.hpp | 64 ++ src/gen/tracer.cpp | 108 +++ src/gen/tracer.hpp | 76 ++ src/pgsql.hpp | 35 +- tests/test-pgsql.cpp | 12 - 33 files changed, 3205 insertions(+), 21 deletions(-) create mode 100644 src/gen/canvas.cpp create mode 100644 src/gen/canvas.hpp create mode 100644 src/gen/gen-base.cpp create mode 100644 src/gen/gen-base.hpp create mode 100644 src/gen/gen-create.cpp create mode 100644 src/gen/gen-create.hpp create mode 100644 src/gen/gen-discrete-isolation.cpp create mode 100644 src/gen/gen-discrete-isolation.hpp create mode 100644 src/gen/gen-rivers.cpp create mode 100644 src/gen/gen-rivers.hpp create mode 100644 src/gen/gen-tile-builtup.cpp create mode 100644 src/gen/gen-tile-builtup.hpp create mode 100644 src/gen/gen-tile-raster.cpp create mode 100644 src/gen/gen-tile-raster.hpp create mode 100644 src/gen/gen-tile-vector.cpp create mode 100644 src/gen/gen-tile-vector.hpp create mode 100644 src/gen/gen-tile.cpp create mode 100644 src/gen/gen-tile.hpp create mode 100644 src/gen/osm2pgsql-gen.cpp create mode 100644 src/gen/params.cpp create mode 100644 src/gen/params.hpp create mode 100644 src/gen/raster.cpp create mode 100644 src/gen/raster.hpp create mode 100644 src/gen/tracer.cpp create mode 100644 src/gen/tracer.hpp diff --git a/.github/actions/ubuntu-prerequisites/action.yml b/.github/actions/ubuntu-prerequisites/action.yml index 12743319a..8a7eb0c91 100644 --- a/.github/actions/ubuntu-prerequisites/action.yml +++ b/.github/actions/ubuntu-prerequisites/action.yml @@ -16,10 +16,12 @@ runs: - name: Install software run: | sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ pandoc \ diff --git a/.github/actions/win-install/action.yml b/.github/actions/win-install/action.yml index bf5171099..3ebc475b3 100644 --- a/.github/actions/win-install/action.yml +++ b/.github/actions/win-install/action.yml @@ -5,9 +5,8 @@ runs: steps: - name: Install packages - run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows + run: vcpkg install cimg:x64-windows bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows shell: bash - - name: Install psycopg2 and beahve run: python -m pip install psycopg2 behave shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b76af0dc6..45a1bcde7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - name: Install prerequisites run: | - brew install lua boost postgis pandoc + brew install lua boost postgis pandoc cimg potrace pip3 install psycopg2 behave pg_ctl -D /usr/local/var/postgres init pg_ctl -D /usr/local/var/postgres start @@ -45,6 +45,7 @@ jobs: env: CC: gcc-10 CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: ON POSTGRESQL_VERSION: 9.6 @@ -79,6 +80,7 @@ jobs: env: CC: gcc-10 CXX: g++-10 + EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug LUA_VERSION: 5.3 LUAJIT_OPTION: OFF POSTGRESQL_VERSION: 10 diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index 3fe0d6790..cba48c0f9 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -37,12 +37,14 @@ jobs: run: | sudo apt-get purge -yq postgresql* sudo apt-get install -yq --no-install-suggests --no-install-recommends \ + cimg-dev \ libboost-filesystem-dev \ libboost-system-dev \ libbz2-dev \ libexpat1-dev \ liblua${LUA_VERSION}-dev \ libluajit-5.1-dev \ + libpotrace-dev \ libpq-dev \ libproj-dev \ lua${LUA_VERSION} \ diff --git a/CMakeLists.txt b/CMakeLists.txt index f92a35771..2d7552c08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,6 +205,11 @@ include_directories(SYSTEM ${PostgreSQL_INCLUDE_DIRS}) find_package(Threads) +find_path(POTRACE_INCLUDE_DIR potracelib.h) +find_library(POTRACE_LIBRARY NAMES potrace) + +find_path(CIMG_INCLUDE_DIR CImg.h) + ############### Libraries are found now ######################## set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES}) @@ -276,6 +281,32 @@ add_subdirectory(src) add_executable(osm2pgsql src/osm2pgsql.cpp) target_link_libraries(osm2pgsql osm2pgsql_lib ${LIBS}) +if (${POTRACE_LIBRARY} STREQUAL "POTRACE_LIBRARY-NOTFOUND" OR ${CIMG_INCLUDE_DIR} STREQUAL "CIMG_INCLUDE_DIR-NOTFOUND") + message(STATUS "Did not find cimg and/or potrace library. Not building osm2pgsql-gen.") +else() + if (WITH_LUA) + message(STATUS "Found cimg and potrace library. Building osm2pgsql-gen.") + include_directories(SYSTEM ${CIMG_INCLUDE_DIR}) + include_directories(SYSTEM ${POTRACE_INCLUDE_DIR}) + add_executable(osm2pgsql-gen src/gen/osm2pgsql-gen.cpp + src/gen/canvas.cpp + src/gen/gen-base.cpp + src/gen/gen-create.cpp + src/gen/gen-discrete-isolation.cpp + src/gen/gen-rivers.cpp + src/gen/gen-tile-builtup.cpp + src/gen/gen-tile-raster.cpp + src/gen/gen-tile-vector.cpp + src/gen/gen-tile.cpp + src/gen/params.cpp + src/gen/raster.cpp + src/gen/tracer.cpp) + target_link_libraries(osm2pgsql-gen osm2pgsql_lib ${LIBS} ${POTRACE_LIBRARY}) + else() + message(STATUS "No Lua. Not building osm2pgsql-gen.") + endif() +endif() + ############################################################# # Optional "clang-tidy" target ############################################################# @@ -287,7 +318,7 @@ find_program(CLANG_TIDY if (CLANG_TIDY) message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}") - file(GLOB CT_CHECK_FILES src/*.cpp tests/*cpp) + file(GLOB CT_CHECK_FILES src/*.cpp src/*/*.cpp tests/*cpp) add_custom_target(clang-tidy ${CLANG_TIDY} diff --git a/README.md b/README.md index 3264c0ad3..db1b67287 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Required libraries are * [zlib](https://www.zlib.net/) * [Boost libraries](https://www.boost.org/), including geometry, system and filesystem +* [CImg](https://cimg.eu/) (Optional, see README-gen.md) +* [potrace](https://potrace.sourceforge.net/) (Optional, see README-gen.md) * [PostgreSQL](https://www.postgresql.org/) client libraries * [Lua](https://www.lua.org/) (Optional, used for Lua tag transforms and the flex output) @@ -80,7 +82,7 @@ On a Debian or Ubuntu system, this can be done with: ```sh sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \ - libboost-filesystem-dev libexpat1-dev zlib1g-dev \ + libboost-filesystem-dev libexpat1-dev zlib1g-dev libpotrace-dev cimg-dev \ libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc ``` @@ -88,6 +90,7 @@ On a Fedora system, use ```sh sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \ + potrace-devel cimg-devel \ bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc ``` diff --git a/src/gen/canvas.cpp b/src/gen/canvas.cpp new file mode 100644 index 000000000..faa402e54 --- /dev/null +++ b/src/gen/canvas.cpp @@ -0,0 +1,130 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "raster.hpp" + +cimg_library::CImg canvas_t::create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const +{ + cimg_library::CImg points{static_cast(pl.size()), 2}; + + int n = 0; + for (auto const point : pl) { + auto const tp = tile.to_tile_coords(point, m_extent); + points(n, 0) = static_cast(static_cast(m_buffer) + tp.x()); + points(n, 1) = + static_cast(static_cast(m_buffer + m_extent) - tp.y()); + ++n; + } + + return points; +} + +std::size_t canvas_t::draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile) +{ + if (polygon.inners().empty()) { + m_rast.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + return polygon.outer().size(); + } + + std::size_t num_points = polygon.outer().size(); + m_temp.draw_polygon(create_pointlist(polygon.outer(), tile), &White); + for (auto const &inner : polygon.inners()) { + num_points += inner.size(); + m_temp.draw_polygon(create_pointlist(inner, tile), &Black); + } + m_rast |= m_temp; + + return num_points; +} + +std::size_t canvas_t::draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile) +{ + m_rast.draw_line(create_pointlist(linestring, tile), &White); + return linestring.size(); +} + +std::size_t canvas_t::draw(geom::geometry_t const &geometry, tile_t const &tile) +{ + if (geometry.is_linestring()) { + auto const &linestring = geometry.get(); + return draw_linestring(linestring, tile); + } + + if (geometry.is_polygon()) { + auto const &polygon = geometry.get(); + return draw_polygon(polygon, tile); + } + + if (geometry.is_multipolygon()) { + auto const &mp = geometry.get(); + std::size_t num_points = 0; + for (auto const &p : mp) { + num_points += draw_polygon(p, tile); + } + return num_points; + } + + // XXX other geometry types? + + return 0; +} + +void canvas_t::save(std::string const &filename) const +{ + m_rast.save(filename.c_str()); +} + +std::string canvas_t::to_wkb(tile_t const &tile, double margin) const +{ + std::string wkb; + wkb.reserve(61 + 2 + m_rast.size()); + + // header + wkb_raster_header header{}; + header.nBands = 1; + header.scaleX = tile.extent() / static_cast(m_extent); + header.scaleY = -header.scaleX; + header.ipX = tile.xmin(margin); + header.ipY = tile.ymax(margin); + header.width = m_extent + 2 * m_buffer; + header.height = header.width; + add_raster_header(&wkb, header); + + // band + wkb_raster_band band{}; + band.bits = 4; + add_raster_band(&wkb, band); + + // rasterdata + wkb.append(reinterpret_cast(m_rast.data()), m_rast.size()); + + assert(wkb.size() == 61 + 2 + m_rast.size()); + + return wkb; +} + +void canvas_t::merge(canvas_t const &other) { m_rast |= other.m_rast; } + +std::string to_hex(std::string const &in) +{ + std::string result; + char const *const lookup_hex = "0123456789ABCDEF"; + + for (const auto c : in) { + unsigned int const num = static_cast(c); + result += lookup_hex[(num >> 4U) & 0xfU]; + result += lookup_hex[num & 0xfU]; + } + + return result; +} diff --git a/src/gen/canvas.hpp b/src/gen/canvas.hpp new file mode 100644 index 000000000..bc439b947 --- /dev/null +++ b/src/gen/canvas.hpp @@ -0,0 +1,88 @@ +#ifndef OSM2PGSQL_CANVAS_HPP +#define OSM2PGSQL_CANVAS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "geom.hpp" +#include "tile.hpp" + +#define cimg_display 0 // NOLINT(cppcoreguidelines-macro-usage) +#include "CImg.h" + +#include + +/** + * This class wraps the image class from the CImg library. + */ +class canvas_t +{ +public: + static void info() { cimg_library::cimg::info(); } + + /** + * Create a new image canvas. It will be quadratic and have the width and + * height extent + 2*buffer. + */ + canvas_t(std::size_t extent, std::size_t buffer) + : m_extent(extent), + m_buffer(buffer), m_rast{size(), size(), 1, 1, 0}, m_temp{size(), size(), + 1, 1, 0} + {} + + unsigned int size() const noexcept + { + return static_cast(m_extent + 2 * m_buffer); + } + + unsigned char const *begin() const noexcept { return m_rast.begin(); } + unsigned char const *end() const noexcept { return m_rast.end(); } + + std::size_t draw(geom::geometry_t const &geometry, tile_t const &tile); + + unsigned char operator()(int x, int y) const noexcept + { + return m_rast(x, y, 0, 0); + } + + void open_close(unsigned int buffer_size) + { + m_rast.dilate(buffer_size).erode(buffer_size * 2).dilate(buffer_size); + } + + void save(std::string const &filename) const; + + std::string to_wkb(tile_t const &tile, double margin) const; + + void merge(canvas_t const &other); + +private: + constexpr static unsigned char const Black = 0; + constexpr static unsigned char const White = 255; + + using image_type = cimg_library::CImg; + + cimg_library::CImg create_pointlist(geom::point_list_t const &pl, + tile_t const &tile) const; + + std::size_t draw_polygon(geom::polygon_t const &polygon, + tile_t const &tile); + + std::size_t draw_linestring(geom::linestring_t const &linestring, + tile_t const &tile); + + std::size_t m_extent; + std::size_t m_buffer; + image_type m_rast; + image_type m_temp; +}; // class canvas_t + +std::string to_hex(std::string const &in); + +#endif // OSM2PGSQL_CANVAS_HPP diff --git a/src/gen/gen-base.cpp b/src/gen/gen-base.cpp new file mode 100644 index 000000000..0aa728371 --- /dev/null +++ b/src/gen/gen-base.cpp @@ -0,0 +1,108 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include "format.hpp" +#include "params.hpp" + +#include + +gen_base_t::gen_base_t(pg_conn_t *connection, params_t *params) +: m_connection(connection), m_params(params) +{ + assert(connection); + assert(params); + + auto const schema = params->get_identifier("schema"); + if (schema.empty()) { + params->set("schema", "public"); + } + + if (params->has("src_table")) { + auto const src_table = get_params().get_identifier("src_table"); + params->set("src", qualified_name(schema, src_table)); + } + + if (params->has("dest_table")) { + auto const dest_table = get_params().get_identifier("dest_table"); + params->set("dest", qualified_name(schema, dest_table)); + } + + if (!params->has("geom_column")) { + params->set("geom_column", "geom"); + } + + m_debug = get_params().get_bool("debug", false); +} + +std::string gen_base_t::name() { return get_params().get_string("name", ""); } + +static pg_result_t dbexec_internal( + pg_conn_t const &connection, std::string const &templ, + fmt::dynamic_format_arg_store const &format_store) +{ + try { + auto const sql = fmt::vformat(templ, format_store); + return connection.exec(sql); + } catch (fmt::format_error const &e) { + log_error("Missing parameter for template: '{}'", templ); + throw; + } +} + +pg_result_t gen_base_t::dbexec(std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +pg_result_t gen_base_t::dbexec(params_t const &tmp_params, + std::string const &templ) +{ + fmt::dynamic_format_arg_store format_store; + for (auto const &[key, value] : get_params()) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + for (auto const &[key, value] : tmp_params) { + format_store.push_back(fmt::arg(key.c_str(), to_string(value))); + } + return dbexec_internal(connection(), templ, format_store); +} + +void gen_base_t::raster_table_preprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, "SELECT DropRasterConstraints('{schema}'::name," + " '{TABLE}'::name, 'rast'::name)"); +} + +void gen_base_t::raster_table_postprocess(std::string const &table) +{ + params_t tmp_params; + tmp_params.set("TABLE", table); + + dbexec(tmp_params, R"(SELECT AddRasterConstraints('{schema}'::name,)" + R"( '{TABLE}'::name, 'rast'::name))"); + dbexec(tmp_params, R"(ALTER TABLE "{schema}"."{TABLE}")" + R"( VALIDATE CONSTRAINT enforce_max_extent_rast)"); + dbexec(tmp_params, R"(ANALYZE "{schema}"."{TABLE}")"); +} + +void gen_base_t::merge_timers(gen_base_t const &other) +{ + for (std::size_t n = 0; n < m_timers.size(); ++n) { + m_timers[n] += other.m_timers[n]; + } +} diff --git a/src/gen/gen-base.hpp b/src/gen/gen-base.hpp new file mode 100644 index 000000000..b6c6f32d9 --- /dev/null +++ b/src/gen/gen-base.hpp @@ -0,0 +1,95 @@ +#ifndef OSM2PGSQL_GEN_BASE_HPP +#define OSM2PGSQL_GEN_BASE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include +#include + +class params_t; +class tile_t; + +/** + * Base class for generalization strategies. + */ +class gen_base_t +{ +public: + virtual ~gen_base_t() = default; + + /// Process data. Used for non-tile-based generalizers. + virtual void process() {} + + /// Process one tile. Used for tile-based generalizers. + virtual void process(tile_t const & /*tile*/) {} + + /// Optional postprocessing after all tiles. + virtual void post() {} + + /// Get the name of the generalization strategy. + virtual std::string_view strategy() const noexcept = 0; + + virtual bool on_tiles() const noexcept { return false; } + + void merge_timers(gen_base_t const &other); + + std::vector const &timers() const noexcept + { + return m_timers; + } + + bool debug() const noexcept { return m_debug; } + + std::string name(); + + template + void log_gen(ARGS... args) + { + if (m_debug) { + log_debug(args...); + } + } + +protected: + gen_base_t(pg_conn_t *connection, params_t *params); + + pg_conn_t &connection() noexcept { return *m_connection; } + + std::size_t add_timer(char const *name) + { + m_timers.emplace_back(name); + return m_timers.size() - 1; + } + + util::timer_t &timer(std::size_t n) noexcept { return m_timers[n]; } + + params_t const &get_params() const noexcept { return *m_params; } + + pg_result_t dbexec(std::string const &templ); + + pg_result_t dbexec(params_t const &tmp_params, std::string const &templ); + + void raster_table_preprocess(std::string const &table); + + void raster_table_postprocess(std::string const &table); + +private: + std::vector m_timers; + pg_conn_t *m_connection; + params_t *m_params; + bool m_debug = false; +}; // class gen_base_t + +#endif // OSM2PGSQL_GEN_BASE_HPP diff --git a/src/gen/gen-create.cpp b/src/gen/gen-create.cpp new file mode 100644 index 000000000..4486338a0 --- /dev/null +++ b/src/gen/gen-create.cpp @@ -0,0 +1,45 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-create.hpp" + +#include "gen-discrete-isolation.hpp" +#include "gen-rivers.hpp" +#include "gen-tile-builtup.hpp" +#include "gen-tile-raster.hpp" +#include "gen-tile-vector.hpp" +#include "params.hpp" + +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params) +{ + auto generalizer = [&]() -> std::unique_ptr { + if (strategy == "builtup") { + return std::make_unique(connection, params); + } + if (strategy == "discrete-isolation") { + return std::make_unique(connection, params); + } + if (strategy == "raster-union") { + return std::make_unique(connection, + params); + } + if (strategy == "rivers") { + return std::make_unique(connection, params); + } + if (strategy == "vector-union") { + return std::make_unique(connection, + params); + } + throw fmt_error("Unknown generalization strategy '{}'.", strategy); + }(); + + return generalizer; +} diff --git a/src/gen/gen-create.hpp b/src/gen/gen-create.hpp new file mode 100644 index 000000000..54dacbef4 --- /dev/null +++ b/src/gen/gen-create.hpp @@ -0,0 +1,26 @@ +#ifndef OSM2PGSQL_GEN_CREATE_HPP +#define OSM2PGSQL_GEN_CREATE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include +#include + +class params_t; +class pg_conn_t; + +/// Instantiate a generalizer for the specified strategy. +std::unique_ptr create_generalizer(std::string const &strategy, + pg_conn_t *connection, + params_t *params); + +#endif // OSM2PGSQL_GEN_CREATE_HPP diff --git a/src/gen/gen-discrete-isolation.cpp b/src/gen/gen-discrete-isolation.cpp new file mode 100644 index 000000000..7f50c5be5 --- /dev/null +++ b/src/gen/gen-discrete-isolation.cpp @@ -0,0 +1,147 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-discrete-isolation.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" + +#include +#include + +gen_di_t::gen_di_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_di(add_timer("di")), + m_timer_reorder(add_timer("reorder")), m_timer_write(add_timer("write")) +{ + params->check_identifier_with_default("id_column", "id"); + params->check_identifier_with_default("importance_column", "importance"); +} + +void gen_di_t::process() +{ + struct feature + { + // input: unique id of the feature + uint64_t id; + + // input: importance of the feature (positive, larger is more imporant) + double importance; + + // input: x/y coordinate of the feature + double x; + double y; + + // output: discrete isolation + double di; + + // output: rank for importance + uint32_t irank; + }; + + log_gen("Reading data from database..."); + + std::vector data; + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT {id_column}, {importance_column}, + ST_X({geom_column}), ST_Y({geom_column}) +FROM {src} WHERE {importance_column} > 0 +)"); + + data.reserve(result.num_tuples()); + for (int i = 0; i < result.num_tuples(); ++i) { + data.push_back({std::strtoull(result.get_value(i, 0), nullptr, 10), + std::strtod(result.get_value(i, 1), nullptr), + std::strtod(result.get_value(i, 2), nullptr), + std::strtod(result.get_value(i, 3), nullptr), 0.0, + 0}); + } + } + timer(m_timer_get).stop(); + log_gen("Read {} features", data.size()); + + if (data.size() < 2) { + log_gen("Found fewer than two features. Nothing to do."); + return; + } + + log_gen("Sorting data by importance..."); + timer(m_timer_sort).start(); + { + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.importance > b.importance; + }); + { + uint32_t n = 0; + for (auto &item : data) { + item.irank = n++; + } + } + } + timer(m_timer_sort).stop(); + + log_gen("Calculating discrete isolation..."); + timer(m_timer_di).start(); + { + std::vector> coords; + coords.reserve(data.size()); + for (auto const &d : data) { + coords.emplace_back(d.x, d.y); + } + + for (std::size_t n = 1; n < data.size(); ++n) { + if (n % 10000 == 0) { + log_gen(" {}", n); + } + double min = 100000000000000.0; + for (std::size_t m = 0; m < n; ++m) { + double const dx = coords[m].first - coords[n].first; + double const dy = coords[m].second - coords[n].second; + double const dist = dx * dx + dy * dy; + if (dist < min) { + min = dist; + } + } + data[n].di = sqrt(min); + } + data[0].di = data[1].di + 1; + } + timer(m_timer_di).stop(); + + log_gen("Sorting data by discrete isolation..."); + timer(m_timer_reorder).start(); + std::sort(data.begin(), data.end(), + [](feature const &a, feature const &b) noexcept { + return a.di > b.di; + }); + timer(m_timer_reorder).stop(); + + log_gen("Writing results to destination table..."); + dbexec("PREPARE update (int, real, int4, int8) AS" + " UPDATE {src} SET dirank = $1, discr_iso = $2, irank = $3" + " WHERE {id_column} = $4"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + std::size_t n = 0; + for (auto const &d : data) { + connection().exec_prepared("update", n++, d.di, d.irank, d.id); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {src}"); + + log_gen("Done."); +} diff --git a/src/gen/gen-discrete-isolation.hpp b/src/gen/gen-discrete-isolation.hpp new file mode 100644 index 000000000..90b5ba578 --- /dev/null +++ b/src/gen/gen-discrete-isolation.hpp @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP +#define OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_di_t : public gen_base_t +{ +public: + gen_di_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override + { + return "discrete-isolation"; + } + +private: + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_di; + std::size_t m_timer_reorder; + std::size_t m_timer_write; +}; + +#endif // OSM2PGSQL_GEN_DISCRETE_ISOLATION_HPP diff --git a/src/gen/gen-rivers.cpp b/src/gen/gen-rivers.cpp new file mode 100644 index 000000000..0835bdd16 --- /dev/null +++ b/src/gen/gen-rivers.cpp @@ -0,0 +1,347 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-rivers.hpp" + +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "util.hpp" +#include "wkb.hpp" + +#include +#include +#include +#include +#include + +gen_rivers_t::gen_rivers_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_area(add_timer("area")), + m_timer_prep(add_timer("prep")), m_timer_get(add_timer("get")), + m_timer_sort(add_timer("sort")), m_timer_net(add_timer("net")), + m_timer_remove(add_timer("remove")), m_timer_width(add_timer("width")), + m_timer_write(add_timer("write")), + m_delete_existing(params->has("delete_existing")) +{ + params->check_identifier_with_default("src_areas", "waterway_areas"); + params->check_identifier_with_default("id_column", "way_id"); + params->check_identifier_with_default("width_column", "width"); + params->check_identifier_with_default("name_column", "name"); + + params->set("qualified_src_areas", + qualified_name(get_params().get_string("schema"), + get_params().get_string("src_areas"))); +} + +/// The data for a graph edge in the waterway network. +struct edge_t +{ + // All the points in this edge + geom::linestring_t points; + + // Edges can be made from (part) of one or more OSM ways, this is the id + // of one of them. + osmid_t id = 0; + + // The width of the river along this edge + double width = 0.0; +}; + +bool operator<(edge_t const &a, edge_t const &b) noexcept +{ + assert(a.points.size() > 1 && b.points.size() > 1); + if (a.points[0] == b.points[0]) { + return a.points[1] < b.points[1]; + } + return a.points[0] < b.points[0]; +} + +bool operator<(edge_t const &a, geom::point_t b) noexcept +{ + assert(!a.points.empty()); + return a.points[0] < b; +} + +bool operator<(geom::point_t a, edge_t const &b) noexcept +{ + assert(!b.points.empty()); + return a < b.points[0]; +} + +static void +follow_chain_and_set_width(edge_t const &edge, std::vector *edges, + std::map const &node_order, + geom::linestring_t *seen) +{ + assert(!edge.points.empty()); + + auto const seen_it = + std::find(seen->cbegin(), seen->cend(), edge.points[0]); + if (seen_it != seen->cend()) { + return; // loop detected + } + + seen->push_back(edge.points[0]); + + assert(edge.points.size() > 1); + auto const next_point = edge.points.back(); + if (node_order.at(next_point) > 1) { + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (std::next(s) == e) { + if (s->width < edge.width) { + s->width = edge.width; + follow_chain_and_set_width(*s, edges, node_order, seen); + } + } else { + for (auto it = s; it != e; ++it) { + assert(it->points[0] == next_point); + if (it->width < edge.width) { + it->width = edge.width; + auto seen2 = *seen; + follow_chain_and_set_width(*it, edges, node_order, &seen2); + } + } + } + } +} + +static void assemble_edge(edge_t *edge, std::vector *edges, + std::map const &node_order) + +{ + assert(edge); + assert(edges); + while (true) { + assert(edge->points.size() > 1); + geom::point_t const next_point = edge->points.back(); + + auto const count = node_order.at(next_point); + if (count != 2) { + return; + } + + auto const [s, e] = + std::equal_range(edges->begin(), edges->end(), next_point); + + if (s == e) { + return; + } + assert(e == std::next(s)); + + auto const it = s; + if (it->points.size() == 1 || &*it == edge) { + return; + } + + if (it->points[0] != next_point) { + return; + } + assert(it != edges->end()); + + edge->width = std::max(edge->width, it->width); + + if (it->points.size() == 2) { + edge->points.push_back(it->points.back()); + it->points.resize(1); + it->points.shrink_to_fit(); + } else { + edge->points.insert(edge->points.end(), + std::next(it->points.begin()), + it->points.end()); + it->points.resize(1); + it->points.shrink_to_fit(); + return; + } + } +} + +/// Get some stats from source table +void gen_rivers_t::get_stats() +{ + auto const result = + dbexec("SELECT count(*), sum(ST_NumPoints(geom)) FROM {src}"); + + m_num_waterways = strtoul(result.get_value(0, 0), nullptr, 10); + m_num_points = strtoul(result.get_value(0, 1), nullptr, 10); + + log_gen("Found {} waterways with {} points.", m_num_waterways, + m_num_points); +} + +static std::string const & +get_name(std::unordered_map const &names, osmid_t id) +{ + static std::string const empty; + auto const it = names.find(id); + if (it == names.end()) { + return empty; + } + return it->second; +} + +void gen_rivers_t::process() +{ + log_gen("Calculate waterway area width..."); + timer(m_timer_area).start(); + dbexec(R"(UPDATE {qualified_src_areas} SET width =)" + R"( (ST_MaximumInscribedCircle("{geom_column}")).radius * 2)" + R"( WHERE width IS NULL)"); + dbexec("ANALYZE {qualified_src_areas}"); + timer(m_timer_area).stop(); + + log_gen("Get 'width' from areas onto lines..."); + timer(m_timer_prep).start(); + dbexec(R"( +WITH _covered_lines AS ( + SELECT "{geom_column}" AS geom, "{id_column}" AS wid FROM {src} w + WHERE ST_NumPoints(w."{geom_column}") > 2 AND ST_CoveredBy(w."{geom_column}", + (SELECT ST_Union("{geom_column}") FROM {qualified_src_areas} a + WHERE ST_Intersects(w."{geom_column}", a."{geom_column}"))) +), _intersections AS ( + SELECT w.wid, ST_Intersection(a.geom, w.geom) AS inters, + ST_Length(w.geom) AS wlength, a.width AS width + FROM _covered_lines w, {qualified_src_areas} a + WHERE ST_Intersects(w.geom, a.geom) +), _lines AS ( + SELECT wid, wlength, ST_Length(inters) * width AS lenwidth FROM _intersections + WHERE ST_GeometryType(inters) IN ('ST_LineString', 'ST_MultiLineString') +), _glines AS ( + SELECT wid, sum(lenwidth) / wlength AS width FROM _lines + GROUP BY wid, wlength +) +UPDATE {src} a SET width = l.width + FROM _glines l WHERE l.wid = a."{id_column}" AND a.width IS NULL + )"); + timer(m_timer_prep).stop(); + + log_gen("Reading waterway lines from database..."); + get_stats(); + + // This vector will initially contain all segments (connection between + // two points) from waterway ways. They will later be assembled into + // graph edges connecting points where the waterways network branches. + std::vector edges; + edges.reserve(m_num_points - m_num_waterways); + + // This stores the order of each node in our graph, i.e. the number of + // connections this node has. Order 1 are beginning or end of a waterway, + // order 2 is just the continuing waterway, order >= 3 is a branching + // point. + std::map node_order; + + // This is where we keep the names of all waterways indexed by their + // way id. + std::unordered_map names; + + timer(m_timer_get).start(); + { + auto const result = dbexec(R"( +SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}" + FROM {src}; +)"); + + for (int i = 0; i < result.num_tuples(); ++i) { + auto const id = std::strtol(result.get_value(i, 0), nullptr, 10); + auto const width = std::strtod(result.get_value(i, 1), nullptr); + auto const name = result.get(i, 2); + if (!name.empty()) { + names.emplace(id, name); + } + auto const geom = ewkb_to_geom(decode_hex(result.get_value(i, 3))); + + if (geom.is_linestring()) { + auto const &ls = geom.get(); + geom::for_each_segment(ls, + [&](geom::point_t a, geom::point_t b) { + if (a != b) { + auto &f = edges.emplace_back(); + f.points.push_back(a); + f.points.push_back(b); + f.id = id; + f.width = width; + node_order[a]++; + node_order[b]++; + } + }); + } + } + } + timer(m_timer_get).stop(); + log_gen("Read {} segments, {} unique points, and {} names.", edges.size(), + node_order.size(), names.size()); + + if (edges.size() < 2) { + log_gen("Found fewer than two segments. Nothing to do."); + return; + } + + log_gen("Sorting segments..."); + timer(m_timer_sort).start(); + std::sort(edges.begin(), edges.end()); + timer(m_timer_sort).stop(); + + log_gen("Assembling edges from segments..."); + timer(m_timer_net).start(); + for (auto &edge : edges) { + if (edge.points.size() > 1) { + assemble_edge(&edge, &edges, node_order); + } + } + timer(m_timer_net).stop(); + + log_gen("Removing now empty edges..."); + timer(m_timer_remove).start(); + { + auto const last = + std::remove_if(edges.begin(), edges.end(), [](edge_t const &edge) { + return edge.points.size() == 1; + }); + edges.erase(last, edges.end()); + std::sort(edges.begin(), edges.end()); + } + timer(m_timer_remove).stop(); + + log_gen("Network has {} edges.", edges.size()); + + log_gen("Propagating 'width' property downstream..."); + timer(m_timer_width).start(); + for (auto &edge : edges) { + assert(!edge.points.empty()); + geom::linestring_t seen; + follow_chain_and_set_width(edge, &edges, node_order, &seen); + } + timer(m_timer_width).stop(); + + if (m_delete_existing) { + dbexec("TRUNCATE {dest}"); + } + + log_gen("Writing results to destination table..."); + dbexec("PREPARE ins (int8, real, text, geometry) AS" + " INSERT INTO {dest} ({id_column}, width, name, geom)" + " VALUES ($1, $2, $3, $4)"); + + timer(m_timer_write).start(); + connection().exec("BEGIN"); + for (auto &edge : edges) { + geom::geometry_t const geom{std::move(edge.points), 3857}; + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("ins", edge.id, edge.width, + get_name(names, edge.id), binary_param(wkb)); + } + connection().exec("COMMIT"); + timer(m_timer_write).stop(); + + dbexec("ANALYZE {dest}"); + + log_gen("Done."); +} diff --git a/src/gen/gen-rivers.hpp b/src/gen/gen-rivers.hpp new file mode 100644 index 000000000..f59afe72a --- /dev/null +++ b/src/gen/gen-rivers.hpp @@ -0,0 +1,43 @@ +#ifndef OSM2PGSQL_GEN_RIVERS_HPP +#define OSM2PGSQL_GEN_RIVERS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +#include + +class gen_rivers_t : public gen_base_t +{ +public: + gen_rivers_t(pg_conn_t *connection, params_t *params); + + void process() override; + + std::string_view strategy() const noexcept override { return "rivers"; } + +private: + void get_stats(); + + std::size_t m_timer_area; + std::size_t m_timer_prep; + std::size_t m_timer_get; + std::size_t m_timer_sort; + std::size_t m_timer_net; + std::size_t m_timer_remove; + std::size_t m_timer_width; + std::size_t m_timer_write; + + std::size_t m_num_waterways = 0; + std::size_t m_num_points = 0; + bool m_delete_existing; +}; + +#endif // OSM2PGSQL_GEN_RIVERS_HPP diff --git a/src/gen/gen-tile-builtup.cpp b/src/gen/gen-tile-builtup.cpp new file mode 100644 index 000000000..96fb22e96 --- /dev/null +++ b/src/gen/gen-tile-builtup.cpp @@ -0,0 +1,280 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-builtup.hpp" + +#include "canvas.hpp" +#include "geom-functions.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_builtup_t::gen_tile_builtup_t(pg_conn_t *connection, params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + m_schema = get_params().get_identifier("schema"); + m_source_tables = + osmium::split_string(get_params().get_string("src_tables"), ','); + + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + + auto const buffer_sizes = + osmium::split_string(get_params().get_string("buffer_size"), ','); + for (auto const &bs : buffer_sizes) { + m_buffer_sizes.push_back(std::strtoul(bs.c_str(), nullptr, 10)); + } + + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + m_min_area = get_params().get_double("min_area", 0.0); + + if (get_params().has("area_column")) { + m_has_area_column = true; + get_params().get_identifier("area_column"); + } + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}_{}", m_image_table, table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + id SERIAL PRIMARY KEY NOT NULL, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + } + + if (params->get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if (m_image_extent < 1024U) { + throw std::runtime_error{"width must be at least 1024"}; + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw std::runtime_error{"width must be power of 2"}; + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + int n = 0; + for (auto const &src_table : m_source_tables) { + params_t tmp_params; + tmp_params.set("N", std::to_string(n++)); + tmp_params.set("SRC", qualified_name(m_schema, src_table)); + + dbexec(tmp_params, R"( +PREPARE get_geoms_{N} (real, real, real, real) AS + SELECT "{geom_column}", '' AS param + FROM {SRC} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + } + + if (m_has_area_column) { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{area_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + dbexec(R"( +PREPARE insert_geoms (geometry, int, int) AS + INSERT INTO {dest} ("{geom_column}", x, y) + VALUES ({geom_sql}, $2, $3) +)"); + } +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const &table, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}_{}\" (zoom, x, y, rast)" + " VALUES ({}, {}, {}, '{}')", + table_prefix, table, variant, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::string table; +}; + +} // anonymous namespace + +using canvas_list_t = std::vector; + +static void draw_from_db(double margin, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + int prep = 0; + auto const box = tile.box(margin); + for (auto &cc : *canvas_list) { + std::string const statement = "get_geoms_" + fmt::to_string(prep++); + auto const result = + conn->exec_prepared(statement.c_str(), box.min_x(), box.min_y(), + box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + cc.canvas.draw(geom, tile); + } + } +} + +void gen_tile_builtup_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + for (auto const &table : m_source_tables) { + canvas_list.push_back( + {canvas_t{m_image_extent, m_image_buffer}, table}); + } + + if (canvas_list.empty()) { + throw std::runtime_error{"No source tables?!"}; + } + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, &canvas_list, &connection(), tile); + timer(m_timer_draw).stop(); + + std::size_t n = 0; + for (auto &[canvas, table] : canvas_list) { + log_gen("Handling table='{}'", table); + + if (!m_image_path.empty()) { + // Save input images for debugging + save_image_to_file(canvas, tile, m_image_path, table, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input images in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "i", m_image_table); + } + + if (m_buffer_sizes[n] > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_sizes[n] * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_sizes[n]); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, table, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, table, + "o", m_image_table); + } + + ++n; + } + + log_gen("Merge bitmaps..."); + for (std::size_t n = 1; n < canvas_list.size(); ++n) { + canvas_list[0].canvas.merge(canvas_list[n].canvas); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = + tracer.trace(canvas_list[0].canvas, tile, m_min_area); + timer(m_timer_vectorize).stop(); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = to_hex(geom_to_ewkb(geom)); + if (m_has_area_column) { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y(), + geom::area(geom)); + } else { + connection().exec_prepared("insert_geoms", wkb, tile.x(), tile.y()); + } + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); +} + +void gen_tile_builtup_t::post() +{ + if (!m_image_table.empty()) { + for (auto const &table : m_source_tables) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}_{}", m_image_table, table, variant)); + } + } + } + dbexec("ANALYZE {dest}"); +} diff --git a/src/gen/gen-tile-builtup.hpp b/src/gen/gen-tile-builtup.hpp new file mode 100644 index 000000000..d130e18f3 --- /dev/null +++ b/src/gen/gen-tile-builtup.hpp @@ -0,0 +1,52 @@ +#ifndef OSM2PGSQL_GEN_TILE_BUILTUP_HPP +#define OSM2PGSQL_GEN_TILE_BUILTUP_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include +#include + +class gen_tile_builtup_t final : public gen_tile_t +{ +public: + gen_tile_builtup_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_builtup_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override { return "builtup"; } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::vector m_source_tables; + std::string m_image_path; + std::string m_schema; + std::string m_dest_table; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + std::vector m_buffer_sizes; + int m_turdsize = 2; + double m_min_area = 0.0; + bool m_has_area_column; +}; + +#endif // OSM2PGSQL_GEN_TILE_BUILTUP_HPP diff --git a/src/gen/gen-tile-raster.cpp b/src/gen/gen-tile-raster.cpp new file mode 100644 index 000000000..e349f46c2 --- /dev/null +++ b/src/gen/gen-tile-raster.cpp @@ -0,0 +1,249 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-raster.hpp" + +#include "canvas.hpp" +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "raster.hpp" +#include "tile.hpp" +#include "tracer.hpp" +#include "wkb.hpp" + +#include + +static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept +{ + return ((value + multiple - 1U) / multiple) * multiple; +} + +gen_tile_raster_union_t::gen_tile_raster_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")), + m_timer_simplify(add_timer("simplify")), + m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write")) +{ + m_margin = get_params().get_double("margin"); + m_image_extent = uint_in_range(*params, "image_extent", 1024, 65536, 2048); + m_image_buffer = + uint_in_range(*params, "image_buffer", 0, m_image_extent, 0); + m_buffer_size = uint_in_range(*params, "buffer_size", 1, 65536, 10); + m_turdsize = static_cast( + uint_in_range(*params, "turdsize", 0, 65536, m_turdsize)); + + if (get_params().has("img_path")) { + m_image_path = get_params().get_string("img_path"); + } + + if (get_params().has("img_table")) { + m_image_table = get_params().get_string("img_table"); + + for (char const variant : {'i', 'o'}) { + auto const table_name = + fmt::format("{}_{}", m_image_table, variant); + connection->exec(R"( +CREATE TABLE IF NOT EXISTS "{}" ( + type TEXT, + zoom INT4, + x INT4, + y INT4, + rast RASTER +) +)", + table_name); + raster_table_preprocess(table_name); + } + } + + if (get_params().get_bool("make_valid")) { + params->set( + "geom_sql", + "(ST_Dump(ST_CollectionExtract(ST_MakeValid($1), 3))).geom"); + } else { + params->set("geom_sql", "$1"); + } + + if (m_image_extent < 1024U) { + throw std::runtime_error{"width must be at least 1024"}; + } + + if ((m_image_extent & (m_image_extent - 1)) != 0) { + throw std::runtime_error{"width must be power of 2"}; + } + + m_image_buffer = + round_up(static_cast(m_margin * + static_cast(m_image_extent)), + 64U); + m_margin = static_cast(m_image_buffer) / + static_cast(m_image_extent); + + log_gen("Image extent: {}px, buffer: {}px, margin: {}", m_image_extent, + m_image_buffer, m_margin); + + if (with_group_by()) { + dbexec(R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", "{group_by_column}" + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y, "{group_by_column}") + VALUES ({geom_sql}, $2, $3, $4) +)"); + } else { + dbexec(R"( +PREPARE get_geoms (real, real, real, real) AS + SELECT "{geom_column}", NULL AS param + FROM {src} + WHERE "{geom_column}" && ST_MakeEnvelope($1, $2, $3, $4, 3857) +)"); + dbexec(R"( +PREPARE insert_geoms (geometry, int, int, text) AS + INSERT INTO {dest} ("{geom_column}", x, y) VALUES ({geom_sql}, $2, $3) +)"); + } +} + +static void save_image_to_table(pg_conn_t *connection, canvas_t const &canvas, + tile_t const &tile, double margin, + std::string const ¶m, char const *variant, + std::string const &table_prefix) +{ + auto const wkb = to_hex(canvas.to_wkb(tile, margin)); + + connection->exec("INSERT INTO \"{}_{}\" (type, zoom, x, y, rast)" + " VALUES ('{}', {}, {}, {}, '{}')", + table_prefix, variant, param, tile.zoom(), tile.x(), + tile.y(), wkb); +} + +namespace { + +struct param_canvas_t +{ + canvas_t canvas; + std::size_t points = 0; + + param_canvas_t(unsigned int image_extent, unsigned int image_buffer) + : canvas(image_extent, image_buffer) + {} +}; + +} // anonymous namespace + +using canvas_list_t = std::unordered_map; + +static void draw_from_db(double margin, unsigned int image_extent, + unsigned int image_buffer, canvas_list_t *canvas_list, + pg_conn_t *conn, tile_t const &tile) +{ + auto const box = tile.box(margin); + auto const result = conn->exec_prepared( + "get_geoms", box.min_x(), box.min_y(), box.max_x(), box.max_y()); + + for (int n = 0; n < result.num_tuples(); ++n) { + std::string param = result.get_value(n, 1); + auto const geom = ewkb_to_geom(decode_hex(result.get_value(n, 0))); + + auto const [it, success] = canvas_list->try_emplace( + std::move(param), image_extent, image_buffer); + + it->second.points += it->second.canvas.draw(geom, tile); + } +} + +void gen_tile_raster_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + canvas_list_t canvas_list; + + log_gen("Read from database and draw polygons..."); + timer(m_timer_draw).start(); + draw_from_db(m_margin, m_image_extent, m_image_buffer, &canvas_list, + &connection(), tile); + timer(m_timer_draw).stop(); + + for (auto &cp : canvas_list) { + auto const ¶m = cp.first; + auto &[canvas, points] = cp.second; + log_gen("Handling param='{}'", param); + + if (!m_image_path.empty()) { + // Save input image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "i", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store input image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "i", m_image_table); + } + + if (m_buffer_size > 0) { + log_gen("Generalize (buffer={} Mercator units)...", + m_buffer_size * tile.extent() / + static_cast(m_image_extent)); + timer(m_timer_simplify).start(); + canvas.open_close(m_buffer_size); + timer(m_timer_simplify).stop(); + } + + if (!m_image_path.empty()) { + // Save output image for debugging + save_image_to_file(canvas, tile, m_image_path, param, "o", + m_image_extent, m_margin); + } + + if (!m_image_table.empty()) { + // Store output image in database for debugging + save_image_to_table(&connection(), canvas, tile, m_margin, param, + "o", m_image_table); + } + + tracer_t tracer{m_image_extent, m_image_buffer, m_turdsize}; + + log_gen("Vectorize..."); + timer(m_timer_vectorize).start(); + auto const geometries = tracer.trace(canvas, tile); + timer(m_timer_vectorize).stop(); + + log_gen("Reduced from {} points to {} points ({:.1f} %)", points, + tracer.num_points(), + static_cast(tracer.num_points()) / + static_cast(points) * 100); + + log_gen("Write geometries to destination table..."); + timer(m_timer_write).start(); + for (auto const &geom : geometries) { + auto const wkb = geom_to_ewkb(geom); + connection().exec_prepared("insert_geoms", binary_param{wkb}, + tile.x(), tile.y(), param); + } + timer(m_timer_write).stop(); + log_gen("Inserted {} generalized polygons", geometries.size()); + } +} + +void gen_tile_raster_union_t::post() +{ + if (!m_image_table.empty()) { + for (char const variant : {'i', 'o'}) { + raster_table_postprocess( + fmt::format("{}_{}", m_image_table, variant)); + } + } + dbexec("ANALYZE {dest}"); +} diff --git a/src/gen/gen-tile-raster.hpp b/src/gen/gen-tile-raster.hpp new file mode 100644 index 000000000..21d26b6df --- /dev/null +++ b/src/gen/gen-tile-raster.hpp @@ -0,0 +1,49 @@ +#ifndef OSM2PGSQL_GEN_TILE_RASTER_HPP +#define OSM2PGSQL_GEN_TILE_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include +#include + +class gen_tile_raster_union_t final : public gen_tile_t +{ +public: + gen_tile_raster_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_raster_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "raster-union"; + } + +private: + std::size_t m_timer_draw; + std::size_t m_timer_simplify; + std::size_t m_timer_vectorize; + std::size_t m_timer_write; + + std::string m_image_path; + std::string m_image_table; + double m_margin = 0.0; + std::size_t m_image_extent = 2048; + std::size_t m_image_buffer = 0; + unsigned int m_buffer_size = 10; + int m_turdsize = 2; +}; + +#endif // OSM2PGSQL_GEN_TILE_RASTER_HPP diff --git a/src/gen/gen-tile-vector.cpp b/src/gen/gen-tile-vector.cpp new file mode 100644 index 000000000..62b1e715b --- /dev/null +++ b/src/gen/gen-tile-vector.cpp @@ -0,0 +1,94 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile-vector.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +gen_tile_vector_union_t::gen_tile_vector_union_t(pg_conn_t *connection, + params_t *params) +: gen_tile_t(connection, params), m_timer_simplify(add_timer("simplify")) +{ + if (!get_params().has("margin")) { + params->set("margin", 0.0); + } else { + // We don't need the result, just checking that this is a real number + get_params().get_double("margin"); + } + + if (!get_params().has("buffer_size")) { + params->set("buffer_size", static_cast(10)); + } else { + // We don't need the result, just checking that this is an integer + get_params().get_int64("buffer_size"); + } + + if (with_group_by()) { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{group_by_column}" AS col, "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT col, ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT col, ST_Union(geom) AS geom + FROM buffered GROUP BY col + ), + unbuffered AS ( + SELECT col, ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{group_by_column}", "{geom_column}") + SELECT $2, $3, col, (ST_Dump(geom)).geom FROM unbuffered +)"); + } else { + dbexec(R"( +PREPARE gen_geoms (int, int, int) AS + WITH gen_tile_input AS ( + SELECT "{geom_column}" AS geom FROM {src} + WHERE "{geom_column}" && ST_TileEnvelope($1, $2, $3, margin => {margin}) + ), + buffered AS ( + SELECT ST_Buffer(geom, {buffer_size}) AS geom + FROM gen_tile_input + ), + merged AS ( + SELECT ST_Union(geom) AS geom + FROM buffered + ), + unbuffered AS ( + SELECT ST_Buffer(ST_Buffer(geom, -2 * {buffer_size}), {buffer_size}) AS geom + FROM merged + ) + INSERT INTO {dest} (x, y, "{geom_column}") + SELECT $2, $3, (ST_Dump(geom)).geom FROM unbuffered +)"); + } +} + +void gen_tile_vector_union_t::process(tile_t const &tile) +{ + delete_existing(tile); + + log_gen("Generalize..."); + timer(m_timer_simplify).start(); + auto const result = connection().exec_prepared("gen_geoms", tile.zoom(), + tile.x(), tile.y()); + timer(m_timer_simplify).stop(); + log_gen("Inserted {} generalized polygons", result.affected_rows()); +} + +void gen_tile_vector_union_t::post() { dbexec("ANALYZE {dest}"); } diff --git a/src/gen/gen-tile-vector.hpp b/src/gen/gen-tile-vector.hpp new file mode 100644 index 000000000..1aa9867a1 --- /dev/null +++ b/src/gen/gen-tile-vector.hpp @@ -0,0 +1,37 @@ +#ifndef OSM2PGSQL_GEN_TILE_VECTOR_HPP +#define OSM2PGSQL_GEN_TILE_VECTOR_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include + +class gen_tile_vector_union_t final : public gen_tile_t +{ +public: + gen_tile_vector_union_t(pg_conn_t *connection, params_t *params); + + ~gen_tile_vector_union_t() override = default; + + void process(tile_t const &tile) override; + + void post() override; + + std::string_view strategy() const noexcept override + { + return "vector-union"; + } + +private: + std::size_t m_timer_simplify; +}; + +#endif // OSM2PGSQL_GEN_TILE_VECTOR_HPP diff --git a/src/gen/gen-tile.cpp b/src/gen/gen-tile.cpp new file mode 100644 index 000000000..8cc0ed773 --- /dev/null +++ b/src/gen/gen-tile.cpp @@ -0,0 +1,49 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-tile.hpp" + +#include "logging.hpp" +#include "params.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +gen_tile_t::gen_tile_t(pg_conn_t *connection, params_t *params) +: gen_base_t(connection, params), m_timer_delete(add_timer("delete")) +{ + m_with_group_by = !get_params().get_identifier("group_by_column").empty(); + + if (get_params().get_bool("delete_existing")) { + m_delete_existing = true; + dbexec("PREPARE del_geoms (int, int) AS" + " DELETE FROM {dest} WHERE x=$1 AND y=$2"); + } +} + +void gen_tile_t::delete_existing(tile_t const &tile) +{ + if (!m_delete_existing) { + return; + } + + if (debug()) { + log_gen("Delete geometries from destination table..."); + } + + timer(m_timer_delete).start(); + auto const result = + connection().exec_prepared("del_geoms", tile.x(), tile.y()); + timer(m_timer_delete).stop(); + + if (debug()) { + log_gen("Deleted {} rows.", result.affected_rows()); + } +} diff --git a/src/gen/gen-tile.hpp b/src/gen/gen-tile.hpp new file mode 100644 index 000000000..626fbf161 --- /dev/null +++ b/src/gen/gen-tile.hpp @@ -0,0 +1,36 @@ +#ifndef OSM2PGSQL_GEN_TILE_HPP +#define OSM2PGSQL_GEN_TILE_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "gen-base.hpp" + +/** + * Base class for generalizations based on tiles. + */ +class gen_tile_t : public gen_base_t +{ +public: + bool on_tiles() const noexcept override { return true; } + +protected: + gen_tile_t(pg_conn_t *connection, params_t *params); + + void delete_existing(tile_t const &tile); + + bool with_group_by() const noexcept { return m_with_group_by; } + +private: + std::size_t m_timer_delete; + bool m_delete_existing = false; + bool m_with_group_by = false; +}; + +#endif // OSM2PGSQL_GEN_TILE_HPP diff --git a/src/gen/osm2pgsql-gen.cpp b/src/gen/osm2pgsql-gen.cpp new file mode 100644 index 000000000..353557fc1 --- /dev/null +++ b/src/gen/osm2pgsql-gen.cpp @@ -0,0 +1,683 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +/** + * \file + * + * This program is used for accessing generalization functionality. It is + * experimental and might or might not be integrated into osm2pgsql itself + * in the future. + */ + +#include "canvas.hpp" +#include "debug-output.hpp" +#include "expire-output.hpp" +#include "flex-lua-expire-output.hpp" +#include "flex-lua-geom.hpp" +#include "flex-lua-table.hpp" +#include "flex-table.hpp" +#include "format.hpp" +#include "gen-base.hpp" +#include "gen-create.hpp" +#include "logging.hpp" +#include "lua-init.hpp" +#include "lua-setup.hpp" +#include "lua-utils.hpp" +#include "options.hpp" +#include "params.hpp" +#include "pgsql-capabilities.hpp" +#include "pgsql.hpp" +#include "tile.hpp" +#include "util.hpp" +#include "version.hpp" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +constexpr std::size_t const max_force_single_thread = 4; + +// Lua can't call functions on C++ objects directly. This macro defines simple +// C "trampoline" functions which are called from Lua which get the current +// context (the genproc_t object) and call the respective function on the +// context object. +// NOLINTNEXTLINE(cppcoreguidelines-macro-usage) +#define TRAMPOLINE(func_name, lua_name) \ + static int lua_trampoline_##func_name(lua_State *lua_state) \ + { \ + try { \ + return static_cast(luaX_get_context(lua_state)) \ + ->func_name(); \ + } catch (std::exception const &e) { \ + return luaL_error(lua_state, "Error in '" #lua_name "': %s\n", \ + e.what()); \ + } catch (...) { \ + return luaL_error(lua_state, \ + "Unknown error in '" #lua_name "'.\n"); \ + } \ + } + +static void show_help() +{ + fmt::print(R"(osm2pgsql-gen [OPTIONS] +Generalization of OSM data. + +This program is EXPERIMENTAL and might change without notice. + +Options: + -h|--help Print this help text and stop + -a|--append Run in append mode + -c|--create Run in create mode (default) + -j|--jobs Number of parallel jobs (default 1) + -l|--log-level=LEVEL Log level (debug, info (default), warn, error) + --log-sql Log SQL commands + --cimg-info Call info() function of CImg library + +Database options: + -d|--database=DB The name of the PostgreSQL database to connect to or + a PostgreSQL conninfo string. + -U|--username=NAME PostgreSQL user name. + -W|--password Force password prompt. + -H|--host=HOST Database server host name or socket location. + -P|--port=PORT Database server port. +)"); +} + +static char const *const short_options = "acd:hH:j:l:P:S:U:W"; + +static std::array const long_options = { + {{"help", no_argument, nullptr, 'h'}, + {"append", no_argument, nullptr, 'a'}, + {"create", no_argument, nullptr, 'c'}, + {"jobs", required_argument, nullptr, 'j'}, + {"database", required_argument, nullptr, 'd'}, + {"user", required_argument, nullptr, 'U'}, + {"host", required_argument, nullptr, 'H'}, + {"port", required_argument, nullptr, 'P'}, + {"password", no_argument, nullptr, 'W'}, + {"log-level", required_argument, nullptr, 'l'}, + {"style", required_argument, nullptr, 'S'}, + {"cimg-info", no_argument, nullptr, 200}, + {"log-sql", no_argument, nullptr, 201}, + {nullptr, 0, nullptr, 0}}}; + +struct tile_extent +{ + uint32_t xmin = 0; + uint32_t ymin = 0; + uint32_t xmax = 0; + uint32_t ymax = 0; + bool valid = false; +}; + +static bool table_is_empty(pg_conn_t const &db_connection, + std::string const &schema, std::string const &table) +{ + auto const result = db_connection.exec("SELECT 1 FROM {} LIMIT 1", + qualified_name(schema, table)); + return result.num_tuples() == 0; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + std::string const &schema, + std::string const &table, + std::string const &column, uint32_t zoom) +{ + if (table_is_empty(db_connection, schema, table)) { + return {}; + } + + auto const result = db_connection.exec( + "SELECT ST_XMin(e), ST_YMin(e), ST_XMax(e), ST_YMax(e)" + " FROM ST_EstimatedExtent('{}', '{}', '{}') AS e", + schema, table, column); + + if (result.num_tuples() == 0 || result.is_null(0, 0)) { + return {}; + } + + double const extent_xmin = strtod(result.get_value(0, 0), nullptr); + double const extent_ymin = strtod(result.get_value(0, 1), nullptr); + double const extent_xmax = strtod(result.get_value(0, 2), nullptr); + double const extent_ymax = strtod(result.get_value(0, 3), nullptr); + log_debug("Extent: ({} {}, {} {})", extent_xmin, extent_ymin, extent_xmax, + extent_ymax); + + return {osmium::geom::mercx_to_tilex(zoom, extent_xmin), + osmium::geom::mercy_to_tiley(zoom, extent_ymax), + osmium::geom::mercx_to_tilex(zoom, extent_xmax), + osmium::geom::mercy_to_tiley(zoom, extent_ymin), true}; +} + +static tile_extent get_extent_from_db(pg_conn_t const &db_connection, + params_t const ¶ms, uint32_t zoom) +{ + auto const schema = params.get_string("schema", "public"); + std::string table; + if (params.has("src_table")) { + table = params.get_string("src_table"); + } else if (params.has("src_tables")) { + table = params.get_string("src_tables"); + auto const n = table.find(','); + if (n != std::string::npos) { + table.resize(n); + } + } else { + throw std::runtime_error{"Need 'src_table' or 'src_tables' param."}; + } + auto const geom_column = params.get_string("geom_column", "geom"); + return get_extent_from_db(db_connection, schema, table, geom_column, zoom); +} + +static std::vector> +get_tiles_from_table(pg_conn_t const &connection, std::string const &table) +{ + std::vector> tiles; + + auto const result = connection.exec(R"(SELECT x, y FROM "{}")", table); + + for (int n = 0; n < result.num_tuples(); ++n) { + char *end = nullptr; + auto const x = std::strtoul(result.get_value(n, 0), &end, 10); + auto const y = std::strtoul(result.get_value(n, 1), &end, 10); + tiles.emplace_back(x, y); + } + + return tiles; +} + +static uint32_t parse_uint32(param_value_t const &val, char const *context, + uint32_t default_value) +{ + if (std::holds_alternative(val)) { + return default_value; + } + if (!std::holds_alternative(val)) { + throw fmt_error("Invalid value for {}.", context); + } + return static_cast(std::get(val)); +} + +class tile_processor_t +{ +public: + tile_processor_t(gen_base_t *generalizer, std::size_t num_tiles) + : m_generalizer(generalizer), m_num_tiles(num_tiles) + {} + + void operator()(tile_t const &tile) + { + log_debug("Processing tile {}/{}/{} ({} of {})...", tile.zoom(), + tile.x(), tile.y(), ++m_count, m_num_tiles); + m_generalizer->process(tile); + } + +private: + gen_base_t *m_generalizer; + std::size_t m_count = 0; + std::size_t m_num_tiles; +}; + +void run_tile_gen(std::string const &conninfo, gen_base_t *master_generalizer, + params_t params, uint32_t zoom, + std::vector> *queue, + std::mutex *mut, unsigned int n) +{ + get_logger().init_thread(n); + + log_debug("Started generalizer thread for '{}'.", + master_generalizer->strategy()); + pg_conn_t db_connection{conninfo}; + std::string const strategy{master_generalizer->strategy()}; + auto generalizer = create_generalizer(strategy, &db_connection, ¶ms); + + while (true) { + std::pair p; + { + std::lock_guard const guard{*mut}; + if (queue->empty()) { + master_generalizer->merge_timers(*generalizer); + break; + } + p = queue->back(); + queue->pop_back(); + } + + generalizer->process({zoom, p.first, p.second}); + } + log_debug("Shutting down generalizer thread."); +} + +class genproc_t +{ +public: + genproc_t(std::string const &filename, std::string conninfo, bool append, + uint32_t jobs); + + int app_define_table() + { +#if 0 + if (m_calling_context != calling_context::main) { + throw std::runtime_error{ + "Database tables have to be defined in the" + " main Lua code, not in any of the callbacks."}; + } +#endif + + return setup_flex_table(m_lua_state.get(), &m_tables, &m_expire_outputs, + true, m_append); + } + + int app_define_expire_output() + { + return setup_flex_expire_output(m_lua_state.get(), &m_expire_outputs); + } + + int app_run_gen() + { + log_debug("Running configured generalizer (run {})...", ++m_gen_run); + + if (lua_type(lua_state(), 1) != LUA_TSTRING) { + throw std::runtime_error{"Argument #1 to 'run_gen' must be a " + "string naming the strategy."}; + } + + std::string const strategy = lua_tostring(lua_state(), 1); + log_debug("Generalizer strategy '{}'", strategy); + + if (lua_type(lua_state(), 2) != LUA_TTABLE) { + throw std::runtime_error{"Argument #2 to 'run_gen' must be a " + "table with parameters."}; + } + + auto params = parse_params(); + + write_to_debug_log(params, "Params (config):"); + + log_debug("Connecting to database..."); + pg_conn_t db_connection{m_conninfo}; + + log_debug("Creating generalizer..."); + auto generalizer = + create_generalizer(strategy, &db_connection, ¶ms); + + log_debug("Generalizer '{}' ({}) initialized.", generalizer->name(), + generalizer->strategy()); + + if (m_append) { + params.set("delete_existing", true); + } + + write_to_debug_log(params, "Params (after initialization):"); + + if (generalizer->on_tiles()) { + process_tiles(db_connection, params, generalizer.get()); + } else { + generalizer->process(); + } + + log_debug("Running generalizer postprocessing..."); + generalizer->post(); + + log_debug("Generalizer processing done."); + + log_debug("Timers:"); + for (auto const &timer : generalizer->timers()) { + log_debug(fmt::format( + " {:10} {:>10L}", timer.name() + ":", + std::chrono::duration_cast( + timer.elapsed()))); + } + log_debug("Finished generalizer '{}' (run {}).", generalizer->name(), + m_gen_run); + + return 0; + } + + int app_run_sql() + { + if (lua_type(lua_state(), 1) != LUA_TTABLE) { + throw std::runtime_error{"Argument #1 to 'run_sql' must be a " + "table with parameters."}; + } + + std::string const description = + luaX_get_table_string(lua_state(), "description", 1, "Argument #1"); + std::string const sql = + luaX_get_table_string(lua_state(), "sql", 1, "Argument #1"); + + log_debug("Running SQL command: {}.", description); + + util::timer_t timer_sql; + pg_conn_t const db_connection{m_conninfo}; + db_connection.exec(sql); + log_info("SQL command took {}.", + util::human_readable_duration(timer_sql.stop())); + + return 0; + } + + void run(); + +private: + params_t parse_params() + { + params_t params; + + lua_pushnil(lua_state()); + while (lua_next(lua_state(), 2) != 0) { + if (lua_type(lua_state(), -2) != LUA_TSTRING) { + throw std::runtime_error{"Argument #2 must have string keys"}; + } + auto const *key = lua_tostring(lua_state(), -2); + + switch (lua_type(lua_state(), -1)) { + case LUA_TSTRING: + params.set(key, lua_tostring(lua_state(), -1)); + break; + case LUA_TNUMBER: +#if LUA_VERSION_NUM >= 503 + if (lua_isinteger(lua_state(), -1)) { + params.set(key, static_cast( + lua_tointeger(lua_state(), -1))); + } else { + params.set(key, static_cast( + lua_tonumber(lua_state(), -1))); + } +#else + params.set(key, + static_cast(lua_tonumber(lua_state(), -1))); +#endif + break; + case LUA_TBOOLEAN: + params.set(key, + static_cast(lua_toboolean(lua_state(), -1))); + break; + case LUA_TNIL: + break; + default: + throw std::runtime_error{"Argument #2 must have string values"}; + } + + lua_pop(lua_state(), 1); + } + return params; + } + + void process_tiles(pg_conn_t const &db_connection, params_t const ¶ms, + gen_base_t *generalizer) + { + uint32_t const zoom = parse_uint32(params.get("zoom"), "zoom", 0); + std::vector> tile_list; + if (m_append) { + auto const table = params.get_string("expire_list"); + log_debug("Running generalizer for expire list from table '{}'...", + table); + tile_list = get_tiles_from_table(db_connection, table); + log_debug("Truncating table '{}'...", table); + db_connection.exec("TRUNCATE {}", table); + } else { + auto const extent = get_extent_from_db(db_connection, params, zoom); + + if (extent.valid) { + auto const num_tiles = (extent.xmax - extent.xmin + 1) * + (extent.ymax - extent.ymin + 1); + log_debug("Running generalizer for bounding box x{}-{}, y{}-{}" + " on zoom={}...", + extent.xmin, extent.xmax, extent.ymin, extent.ymax, + zoom); + tile_list.reserve(num_tiles); + for (unsigned x = extent.xmin; x <= extent.xmax; ++x) { + for (unsigned y = extent.ymin; y <= extent.ymax; ++y) { + tile_list.emplace_back(x, y); + } + } + } else { + log_debug("Source table empty, nothing to do."); + } + } + log_debug("Need to process {} tiles.", tile_list.size()); + if (m_jobs == 1 || tile_list.size() < max_force_single_thread) { + log_debug("Running in single-threaded mode."); + tile_processor_t tp{generalizer, tile_list.size()}; + while (!tile_list.empty()) { + auto [x, y] = tile_list.back(); + tp({zoom, x, y}); + tile_list.pop_back(); + } + } else { + log_debug("Running in multi-threaded mode."); + std::mutex mut; + std::vector threads; + for (unsigned int n = 1; n <= m_jobs; ++n) { + threads.emplace_back(run_tile_gen, m_conninfo, generalizer, + params, zoom, &tile_list, &mut, n); + } + for (auto &t : threads) { + t.join(); + } + } + } + + lua_State *lua_state() const noexcept { return m_lua_state.get(); } + + std::shared_ptr m_lua_state{ + luaL_newstate(), [](lua_State *state) { lua_close(state); }}; + + std::vector m_tables; + std::vector m_expire_outputs; + + std::string m_conninfo; + std::size_t m_gen_run = 0; + uint32_t m_jobs; + bool m_append; +}; // class genproc_t + +TRAMPOLINE(app_define_table, define_table) +TRAMPOLINE(app_define_expire_output, define_expire_output) +TRAMPOLINE(app_run_gen, run_gen) +TRAMPOLINE(app_run_sql, run_sql) + +genproc_t::genproc_t(std::string const &filename, std::string conninfo, + bool append, uint32_t jobs) +: m_conninfo(std::move(conninfo)), m_jobs(jobs), m_append(append) +{ + setup_lua_environment(lua_state(), filename, append); + + luaX_add_table_func(lua_state(), "define_table", + lua_trampoline_app_define_table); + luaX_add_table_func(lua_state(), "define_expire_output", + lua_trampoline_app_define_expire_output); + + luaX_add_table_func(lua_state(), "run_gen", lua_trampoline_app_run_gen); + luaX_add_table_func(lua_state(), "run_sql", lua_trampoline_app_run_sql); + + init_geometry_class(lua_state()); + + // Load compiled in init.lua + if (luaL_dostring(lua_state(), lua_init())) { + throw fmt_error("Internal error in Lua setup: {}.", + lua_tostring(lua_state(), -1)); + } + + // Load user config file + luaX_set_context(lua_state(), this); + if (luaL_dofile(lua_state(), filename.c_str())) { + throw fmt_error("Error loading lua config: {}.", + lua_tostring(lua_state(), -1)); + } + + write_expire_output_list_to_debug_log(m_expire_outputs); + write_table_list_to_debug_log(m_tables); +} + +void genproc_t::run() +{ + lua_getglobal(lua_state(), "osm2pgsql"); + lua_getfield(lua_state(), -1, "process_gen"); + + if (lua_isnil(lua_state(), -1)) { + log_warn("No function 'osm2pgsql.process_gen()'. Nothing to do."); + return; + } + + if (luaX_pcall(lua_state(), 0, 0)) { + throw fmt_error( + "Failed to execute Lua function 'osm2pgsql.process_gen': {}.", + lua_tostring(lua_state(), -1)); + } +} + +int main(int argc, char *argv[]) +{ + try { + database_options_t database_options; + std::string log_level; + std::string style; + uint32_t jobs = 1; + bool pass_prompt = false; + bool append = false; + + int c = 0; + // NOLINTNEXTLINE(concurrency-mt-unsafe) + while (-1 != (c = getopt_long(argc, argv, short_options, + long_options.data(), nullptr))) { + switch (c) { + case 'h': + show_help(); + return 0; + case 'a': // --append + append = true; + break; + case 'c': // --create + append = false; + break; + case 'j': // --jons + jobs = + std::clamp(std::strtoul(optarg, nullptr, 10), 1UL, 256UL); + break; + case 'd': // --database + database_options.db = optarg; + break; + case 'U': // --username + database_options.username = optarg; + break; + case 'W': // --password + pass_prompt = true; + break; + case 'H': // --host + database_options.host = optarg; + break; + case 'P': // --port + database_options.port = optarg; + break; + case 'l': + log_level = optarg; + break; + case 'S': + style = optarg; + break; + case 200: + canvas_t::info(); + return 0; + case 201: + get_logger().enable_sql(); + break; + default: + log_error("Unknown argument"); + return 2; + } + } + + if (log_level == "debug") { + get_logger().set_level(log_level::debug); + } else if (log_level == "info" || log_level.empty()) { + get_logger().set_level(log_level::info); + } else if (log_level == "warn") { + get_logger().set_level(log_level::warn); + } else if (log_level == "error") { + get_logger().set_level(log_level::error); + } else { + log_error("Unknown log level: {}. " + "Use 'debug', 'info', 'warn', or 'error'.", + log_level); + return 2; + } + + if (style.empty()) { + log_error("Need --style/-S option"); + return 2; + } + + if (jobs < 1 || jobs > 32) { + log_error("The --jobs/-j parameter must be between 1 and 32."); + return 2; + } + + util::timer_t timer_overall; + + log_info("osm2pgsql-gen version {}", get_osm2pgsql_version()); + log_warn("This is an EXPERIMENTAL extension to osm2pgsql."); + + if (append) { + log_debug("Running in append mode."); + } else { + log_debug("Running in create mode."); + } + + if (jobs == 1) { + log_debug("Running in single-threaded mode."); + } else { + log_debug( + "Running in multi-threaded mode with a maximum of {} threads.", + jobs); + } + + if (pass_prompt) { + database_options.password = util::get_password(); + } + auto const conninfo = build_conninfo(database_options); + + log_debug("Checking database capabilities..."); + { + pg_conn_t const db_connection{conninfo}; + init_database_capabilities(db_connection); + } + + genproc_t gen{style, conninfo, append, jobs}; + gen.run(); + + osmium::MemoryUsage const mem; + log_info("Memory: {}MB current, {}MB peak", mem.current(), mem.peak()); + + log_info("osm2pgsql-gen took {} overall.", + util::human_readable_duration(timer_overall.stop())); + } catch (std::exception const &e) { + log_error("{}", e.what()); + return 1; + } catch (...) { + log_error("Unknown exception."); + return 1; + } + + return 0; +} diff --git a/src/gen/params.cpp b/src/gen/params.cpp new file mode 100644 index 000000000..b50bae946 --- /dev/null +++ b/src/gen/params.cpp @@ -0,0 +1,125 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "params.hpp" + +#include "format.hpp" +#include "logging.hpp" +#include "overloaded.hpp" +#include "pgsql.hpp" + +std::string to_string(param_value_t const &value) +{ + return std::visit( + overloaded{[](null_param_t) { return std::string{}; }, + [](std::string val) { return val; }, + [](auto const &val) { return fmt::to_string(val); }}, + value); +} + +param_value_t params_t::get(std::string const &key) const +{ + return m_map.at(key); +} + +bool params_t::has(std::string const &key) const noexcept +{ + return m_map.count(key) > 0; +} + +bool params_t::get_bool(std::string const &key, bool default_value) const +{ + return get_by_type(key, default_value); +} + +int64_t params_t::get_int64(std::string const &key, int64_t default_value) const +{ + return get_by_type(key, default_value); +} + +double params_t::get_double(std::string const &key, double default_value) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (std::holds_alternative(it->second)) { + return std::get(it->second); + } + + if (std::holds_alternative(it->second)) { + return static_cast(std::get(it->second)); + } + + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), key); +} + +std::string params_t::get_string(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + throw fmt_error("Missing parameter '{}' on generalizer.", key); + } + return to_string(it->second); +} + +std::string params_t::get_string(std::string const &key, + std::string const &default_value) const +{ + return get_by_type(key, default_value); +} + +std::string params_t::get_identifier(std::string const &key) const +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + return {}; + } + std::string result = to_string(it->second); + check_identifier(result, key.c_str()); + return result; +} + +void params_t::check_identifier_with_default(std::string const &key, + std::string default_value) +{ + auto const it = m_map.find(key); + if (it == m_map.end()) { + m_map.emplace(key, std::move(default_value)); + } else { + check_identifier(to_string(it->second), key.c_str()); + } +} + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value) +{ + int64_t const value = params.get_int64(key, default_value); + if (value < 0 || value > std::numeric_limits::max()) { + throw fmt_error("Invalid value '{}' for {}.", value, key); + } + auto uvalue = static_cast(value); + if (uvalue < min || uvalue > max) { + throw fmt_error("Invalid value '{}' for {}.", value, key); + } + return uvalue; +} + +void write_to_debug_log(params_t const ¶ms, char const *message) +{ + if (!get_logger().debug_enabled()) { + return; + } + log_debug(message); + for (auto const &[key, value] : params) { + log_debug(" {}={}", key, to_string(value)); + } +} diff --git a/src/gen/params.hpp b/src/gen/params.hpp new file mode 100644 index 000000000..f0f766f81 --- /dev/null +++ b/src/gen/params.hpp @@ -0,0 +1,96 @@ +#ifndef OSM2PGSQL_PARAMS_HPP +#define OSM2PGSQL_PARAMS_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "logging.hpp" + +#include +#include +#include + +/// A "NULL" value for a parameter. Same as not set. +using null_param_t = std::monostate; + +/// A parameter value can have one of several types. +using param_value_t = + std::variant; + +/// Convert a parameter value into a string. +std::string to_string(param_value_t const &value); + +/** + * A collection of parameters. + */ +class params_t +{ +public: + template + void set(K &&key, V &&value) + { + m_map.insert_or_assign(std::forward(key), std::forward(value)); + } + + template + void remove(K &&key) + { + m_map.erase(std::forward(key)); + } + + bool has(std::string const &key) const noexcept; + + param_value_t get(std::string const &key) const; + + bool get_bool(std::string const &key, bool default_value = false) const; + + int64_t get_int64(std::string const &key, int64_t default_value = 0) const; + + double get_double(std::string const &key, double default_value = 0.0) const; + + std::string get_string(std::string const &key) const; + + std::string get_string(std::string const &key, + std::string const &default_value) const; + + std::string get_identifier(std::string const &key) const; + + void check_identifier_with_default(std::string const &key, + std::string default_value); + + auto begin() const noexcept { return m_map.begin(); } + + auto end() const noexcept { return m_map.end(); } + +private: + template + T get_by_type(std::string const &key, T default_value) const + { + auto const it = m_map.find(key); + if (it == m_map.end()) { + return default_value; + } + + if (!std::holds_alternative(it->second)) { + throw fmt_error("Invalid value '{}' for {}.", to_string(it->second), + key); + } + return std::get(it->second); + } + + std::map m_map; +}; // class params_t + +void write_to_debug_log(params_t const ¶ms, char const *message); + +unsigned int uint_in_range(params_t const ¶ms, std::string const &key, + unsigned int min, unsigned int max, + unsigned int default_value); + +#endif // OSM2PGSQL_PARAMS_HPP diff --git a/src/gen/raster.cpp b/src/gen/raster.cpp new file mode 100644 index 000000000..e552c7cd2 --- /dev/null +++ b/src/gen/raster.cpp @@ -0,0 +1,66 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "raster.hpp" + +#include "canvas.hpp" +#include "format.hpp" +#include "pgsql.hpp" +#include "tile.hpp" + +#include + +template +void append(std::string *str, T value) +{ + str->append(reinterpret_cast(&value), sizeof(T)); +} + +void add_raster_header(std::string *wkb, wkb_raster_header const &data) +{ + append(wkb, data.endianness); + append(wkb, data.version); + append(wkb, data.nBands); + append(wkb, data.scaleX); + append(wkb, data.scaleY); + append(wkb, data.ipX); + append(wkb, data.ipY); + append(wkb, data.skewX); + append(wkb, data.skewY); + append(wkb, data.srid); + append(wkb, data.width); + append(wkb, data.height); +} + +void add_raster_band(std::string *wkb, wkb_raster_band const &data) +{ + append(wkb, data.bits); + append(wkb, data.nodata); +} + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin) +{ + std::string name{fmt::format("{}-{}-{}-{}{}{}.", path, tile.x(), tile.y(), + param, param.empty() ? "" : "-", variant)}; + + // write image file + canvas.save(name + "png"); + + // write world file + auto const pixel_size = tile.extent() / image_extent; + name += "wld"; + auto *file = std::fopen(name.c_str(), "w"); + fmt::print(file, "{0}\n0.0\n0.0\n-{0}\n{1}\n{2}\n", pixel_size, + tile.xmin(margin) + pixel_size / 2, + tile.ymax(margin) - pixel_size / 2); + (void)std::fclose(file); +} diff --git a/src/gen/raster.hpp b/src/gen/raster.hpp new file mode 100644 index 000000000..82a86ea0b --- /dev/null +++ b/src/gen/raster.hpp @@ -0,0 +1,64 @@ +#ifndef OSM2PGSQL_RASTER_HPP +#define OSM2PGSQL_RASTER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include +#include + +class canvas_t; +class pg_conn_t; +class tile_t; + +/** + * \file + * + * Helper functions for creating raster images in PostgreSQL/PostGIS. + * https://trac.osgeo.org/postgis/wiki/WKTRaster/RFC/RFC2_V0WKBFormat + */ + +struct wkb_raster_header +{ + uint8_t endianness = +#if __BYTE_ORDER == __LITTLE_ENDIAN + 1 // Little Endian +#else + 0 // Big Endian +#endif + ; + uint16_t version = 0; + uint16_t nBands = 0; + double scaleX = 0.0; + double scaleY = 0.0; + double ipX = 0.0; + double ipY = 0.0; + double skewX = 0.0; + double skewY = 0.0; + int32_t srid = 3857; + uint16_t width = 0; + uint16_t height = 0; +}; + +struct wkb_raster_band +{ + uint8_t bits = 0; + uint8_t nodata = 0; +}; + +void add_raster_header(std::string *wkb, wkb_raster_header const &data); + +void add_raster_band(std::string *wkb, wkb_raster_band const &data); + +void save_image_to_file(canvas_t const &canvas, tile_t const &tile, + std::string const &path, std::string const ¶m, + char const *variant, unsigned int image_extent, + double margin); + +#endif // OSM2PGSQL_RASTER_HPP diff --git a/src/gen/tracer.cpp b/src/gen/tracer.cpp new file mode 100644 index 000000000..5321c3eae --- /dev/null +++ b/src/gen/tracer.cpp @@ -0,0 +1,108 @@ +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "tracer.hpp" + +#include "geom-boost-adaptor.hpp" + +#include +#include +#include + +geom::point_t tracer_t::make_point(potrace_dpoint_t const &p) const noexcept +{ + return {p.x - static_cast(m_buffer), + static_cast(m_extent + m_buffer) - p.y}; +} + +std::vector +tracer_t::trace(canvas_t const &canvas, tile_t const &tile, double min_area) +{ + prepare(canvas); + + m_state.reset(potrace_trace(m_param.get(), &m_bitmap)); + if (!m_state || m_state->status != POTRACE_STATUS_OK) { + throw std::runtime_error{"potrace failed"}; + } + + return build_geometries(tile, m_state->plist, min_area); +} + +void tracer_t::reset() +{ + m_bits.clear(); + m_state.reset(); + m_num_points = 0; +} + +void tracer_t::prepare(canvas_t const &canvas) noexcept +{ + std::size_t const size = canvas.size(); + assert(size % bits_per_word == 0); + + m_bits.reserve((size * size) / bits_per_word); + + unsigned char const *d = canvas.begin(); + while (d != canvas.end()) { + potrace_word w = 0x1U & *d++; + for (std::size_t n = 1; n < bits_per_word; ++n) { + w <<= 1U; + assert(d != canvas.end()); + w |= 0x1U & *d++; + } + m_bits.push_back(w); + } + + m_bitmap = {int(size), int(size), int(size / bits_per_word), m_bits.data()}; +} + +std::vector +tracer_t::build_geometries(tile_t const &tile, potrace_path_t const *plist, + double min_area) noexcept +{ + std::vector geometries; + if (!plist) { + return geometries; + } + + for (potrace_path_t const *path = plist; path != nullptr; + path = path->next) { + + geom::ring_t ring; + + auto const n = path->curve.n; + assert(path->curve.tag[n - 1] == POTRACE_CORNER); + ring.push_back(tile.to_world_coords(make_point(path->curve.c[n - 1][2]), + m_extent)); + for (int i = 0; i < n; ++i) { + assert(path->curve.tag[i] == POTRACE_CORNER); + auto const &c = path->curve.c[i]; + ring.push_back(tile.to_world_coords(make_point(c[1]), m_extent)); + ring.push_back(tile.to_world_coords(make_point(c[2]), m_extent)); + } + + auto const ring_area = + std::abs(static_cast(boost::geometry::area(ring))); + if (ring_area >= min_area) { + m_num_points += ring.size(); + + if (path->sign == '+') { + geometries.emplace_back(geom::polygon_t{}, 3857) + .get() + .outer() = std::move(ring); + } else { + assert(!geometries.empty()); + geometries.back().get().add_inner_ring( + std::move(ring)); + } + } + } + + return geometries; +} diff --git a/src/gen/tracer.hpp b/src/gen/tracer.hpp new file mode 100644 index 000000000..6026070e6 --- /dev/null +++ b/src/gen/tracer.hpp @@ -0,0 +1,76 @@ +#ifndef OSM2PGSQL_TRACER_HPP +#define OSM2PGSQL_TRACER_HPP + +/** + * SPDX-License-Identifier: GPL-2.0-or-later + * + * This file is part of osm2pgsql (https://osm2pgsql.org/). + * + * Copyright (C) 2006-2023 by the osm2pgsql developer community. + * For a full list of authors see the git log. + */ + +#include "canvas.hpp" +#include "geom.hpp" +#include "tile.hpp" + +#include + +#include +#include + +class tracer_t +{ +public: + tracer_t(std::size_t extent, std::size_t buffer, int turdsize) + : m_param(potrace_param_default()), m_extent(extent), m_buffer(buffer) + { + m_param->alphamax = 0.0; + m_param->turdsize = turdsize; + } + + std::vector + trace(canvas_t const &canvas, tile_t const &tile, double min_area = 0.0); + + void reset(); + + std::size_t num_points() const noexcept { return m_num_points; } + +private: + static constexpr auto const bits_per_word = sizeof(potrace_word) * 8; + + geom::point_t make_point(potrace_dpoint_t const &p) const noexcept; + + struct potrace_param_deleter + { + void operator()(potrace_param_t *ptr) const noexcept + { + potrace_param_free(ptr); + } + }; + + struct potrace_state_deleter + { + void operator()(potrace_state_t *ptr) const noexcept + { + potrace_state_free(ptr); + } + }; + + void prepare(canvas_t const &canvas) noexcept; + + std::vector build_geometries(tile_t const &tile, + potrace_path_t const *plist, + double min_area) noexcept; + + std::vector m_bits; + potrace_bitmap_t m_bitmap{}; + std::unique_ptr m_param; + std::unique_ptr m_state; + std::size_t m_extent; + std::size_t m_buffer; + std::size_t m_num_points = 0; + +}; // class tracer_t + +#endif // OSM2PGSQL_TRACER_HPP diff --git a/src/pgsql.hpp b/src/pgsql.hpp index 782d79ae8..3736a115b 100644 --- a/src/pgsql.hpp +++ b/src/pgsql.hpp @@ -126,6 +126,20 @@ class pg_result_t std::unique_ptr m_result; }; +/** + * Wrapper class for query parameters that should be sent to the database + * as binary parameter. + */ +class binary_param : public std::string_view +{ +public: + using std::string_view::string_view; + + binary_param(std::string const &str) + : std::string_view(str.data(), str.size()) + {} +}; + /** * PostgreSQL connection. * @@ -229,6 +243,8 @@ class pg_conn_t return 0; } else if constexpr (std::is_same_v) { return 0; + } else if constexpr (std::is_same_v) { + return 0; } return 1; } @@ -240,12 +256,18 @@ class pg_conn_t * strings. */ template - static char const *to_str(std::vector *data, T const ¶m) + static char const *to_str(std::vector *data, int *length, + int *bin, T const ¶m) { if constexpr (std::is_same_v) { return param; } else if constexpr (std::is_same_v) { + *length = param.size(); return param.c_str(); + } else if constexpr (std::is_same_v) { + *length = param.size(); + *bin = 1; + return param.data(); } return data->emplace_back(fmt::to_string(param)).c_str(); } @@ -275,16 +297,21 @@ class pg_conn_t std::vector exec_params; exec_params.reserve(total_buffers_needed); + std::array lengths = {0}; + std::array bins = {0}; + // This array holds the pointers to all parameter strings, either // to the original string parameters or to the recently converted // in the exec_params vector. + std::size_t n = 0; + std::size_t m = 0; std::array param_ptrs = { - to_str>(&exec_params, + to_str>(&exec_params, &lengths[n++], &bins[m++], std::forward(params))...}; return exec_prepared_internal(stmt, sizeof...(params), - param_ptrs.data(), nullptr, nullptr, - result_as_binary ? 1 : 0); + param_ptrs.data(), lengths.data(), + bins.data(), result_as_binary ? 1 : 0); } struct pg_conn_deleter_t diff --git a/tests/test-pgsql.cpp b/tests/test-pgsql.cpp index f394fc224..4d9806ed4 100644 --- a/tests/test-pgsql.cpp +++ b/tests/test-pgsql.cpp @@ -56,18 +56,6 @@ TEST_CASE("exec with invalid SQL should fail") REQUIRE_THROWS(conn.exec("XYZ")); } -TEST_CASE("exec_prepared without parameters should work") -{ - auto const conn = db.db().connect(); - conn.exec("PREPARE test AS SELECT 42"); - - auto const result = conn.exec_prepared("test"); - REQUIRE(result.status() == PGRES_TUPLES_OK); - REQUIRE(result.num_fields() == 1); - REQUIRE(result.num_tuples() == 1); - REQUIRE(result.get(0, 0) == "42"); -} - TEST_CASE("exec_prepared with single string parameters should work") { auto const conn = db.db().connect();