Skip to content

Commit 0613222

Browse files
committed
WIP: Add code to generalize data using different strategies
This commit adds the code to generalize various types of data using different strategies. The following strategies work on a tile-by-tile basis and operate on polygons: The "vector-union" strategy buffers and unionizes polygons using vector operations. The "raster-union" strategy does a similar thing but does it in raster space which is much faster. First the polygons are rendered into a raster, an open/close operation is called (which basically does the same thing as the buffering in vector space) and finally the resulting raster is vectorized again. The "builtup" strategy is intended to derive a layer of builtup areas from landuse=residential/industrial etc. as well as building cover and dense road networks. This still needs some work... Also a new "discrete-isolation" strategy which rates places based on some importance metric. (This is not tile-based.) The new "rivers" strategy finds important rivers, this is still very much work in progress. For the raster support this adds two new library dependency: CImg and potrace. The functionality is accessed through a new command line program called osm2pgsql-gen. It reads the same Lua config file that osm2pgsql reads. Call it with -h to get some usage information. This program is for testing only, eventually the functionality should be accessible from osm2pgsql itself. See also https://osm2pgsql.org/generalization/ .
1 parent 69c6b4e commit 0613222

36 files changed

+3241
-30
lines changed

.github/actions/ubuntu-prerequisites/action.yml

+2
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ runs:
1616
- name: Install software
1717
run: |
1818
sudo apt-get install -yq --no-install-suggests --no-install-recommends \
19+
cimg-dev \
1920
libboost-filesystem-dev \
2021
libboost-system-dev \
2122
libbz2-dev \
2223
libexpat1-dev \
24+
libpotrace-dev \
2325
libpq-dev \
2426
libproj-dev \
2527
pandoc \

.github/actions/win-install/action.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ runs:
55

66
steps:
77
- name: Install packages
8-
run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows
8+
run: vcpkg install cimg:x64-windows bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows
99
shell: bash
10-
1110
- name: Install psycopg2 and beahve
1211
run: python -m pip install psycopg2 behave
1312
shell: bash

.github/workflows/ci.yml

+3-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414

1515
- name: Install prerequisites
1616
run: |
17-
brew install lua boost postgis pandoc
17+
brew install lua boost postgis pandoc cimg potrace
1818
pip3 install psycopg2 behave
1919
pg_ctl -D /usr/local/var/postgres init
2020
pg_ctl -D /usr/local/var/postgres start
@@ -45,6 +45,7 @@ jobs:
4545
env:
4646
CC: gcc-10
4747
CXX: g++-10
48+
EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug
4849
LUA_VERSION: 5.3
4950
LUAJIT_OPTION: ON
5051
POSTGRESQL_VERSION: 9.6
@@ -79,6 +80,7 @@ jobs:
7980
env:
8081
CC: gcc-10
8182
CXX: g++-10
83+
EXTRA_FLAGS: -Wno-unused-but-set-parameter # workaround for GCC bug
8284
LUA_VERSION: 5.3
8385
LUAJIT_OPTION: OFF
8486
POSTGRESQL_VERSION: 10

.github/workflows/test-install.yml

+4-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ jobs:
2222
POSTGRESQL_VERSION: 12
2323
POSTGIS_VERSION: 3
2424
BUILD_TYPE: Release
25-
CXXFLAGS: -pedantic -Wextra -Werror
25+
# -Wno-unused-but-set-parameter is a workaround for GCC bug
26+
CXXFLAGS: -pedantic -Wextra -Werror -Wno-unused-but-set-parameter
2627
PREFIX: /usr/local
2728
OSMURL: https://download.geofabrik.de/europe/monaco-latest.osm.pbf
2829
OSMFILE: monaco-latest.osm.pbf
@@ -37,12 +38,14 @@ jobs:
3738
run: |
3839
sudo apt-get purge -yq postgresql*
3940
sudo apt-get install -yq --no-install-suggests --no-install-recommends \
41+
cimg-dev \
4042
libboost-filesystem-dev \
4143
libboost-system-dev \
4244
libbz2-dev \
4345
libexpat1-dev \
4446
liblua${LUA_VERSION}-dev \
4547
libluajit-5.1-dev \
48+
libpotrace-dev \
4649
libpq-dev \
4750
libproj-dev \
4851
lua${LUA_VERSION} \

CMakeLists.txt

+32-1
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,11 @@ include_directories(SYSTEM ${PostgreSQL_INCLUDE_DIRS})
205205

206206
find_package(Threads)
207207

208+
find_path(POTRACE_INCLUDE_DIR potracelib.h)
209+
find_library(POTRACE_LIBRARY NAMES potrace)
210+
211+
find_path(CIMG_INCLUDE_DIR CImg.h)
212+
208213
############### Libraries are found now ########################
209214

210215
set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES})
@@ -276,6 +281,32 @@ add_subdirectory(src)
276281
add_executable(osm2pgsql src/osm2pgsql.cpp)
277282
target_link_libraries(osm2pgsql osm2pgsql_lib ${LIBS})
278283

284+
if (${POTRACE_LIBRARY} STREQUAL "POTRACE_LIBRARY-NOTFOUND" OR ${CIMG_INCLUDE_DIR} STREQUAL "CIMG_INCLUDE_DIR-NOTFOUND")
285+
message(STATUS "Did not find cimg and/or potrace library. Not building osm2pgsql-gen.")
286+
else()
287+
if (WITH_LUA)
288+
message(STATUS "Found cimg and potrace library. Building osm2pgsql-gen.")
289+
include_directories(SYSTEM ${CIMG_INCLUDE_DIR})
290+
include_directories(SYSTEM ${POTRACE_INCLUDE_DIR})
291+
add_executable(osm2pgsql-gen src/gen/osm2pgsql-gen.cpp
292+
src/gen/canvas.cpp
293+
src/gen/gen-base.cpp
294+
src/gen/gen-create.cpp
295+
src/gen/gen-discrete-isolation.cpp
296+
src/gen/gen-rivers.cpp
297+
src/gen/gen-tile-builtup.cpp
298+
src/gen/gen-tile-raster.cpp
299+
src/gen/gen-tile-vector.cpp
300+
src/gen/gen-tile.cpp
301+
src/gen/params.cpp
302+
src/gen/raster.cpp
303+
src/gen/tracer.cpp)
304+
target_link_libraries(osm2pgsql-gen osm2pgsql_lib ${LIBS} ${POTRACE_LIBRARY})
305+
else()
306+
message(STATUS "No Lua. Not building osm2pgsql-gen.")
307+
endif()
308+
endif()
309+
279310
#############################################################
280311
# Optional "clang-tidy" target
281312
#############################################################
@@ -287,7 +318,7 @@ find_program(CLANG_TIDY
287318
if (CLANG_TIDY)
288319
message(STATUS "Looking for clang-tidy - found ${CLANG_TIDY}")
289320

290-
file(GLOB CT_CHECK_FILES src/*.cpp tests/*cpp)
321+
file(GLOB CT_CHECK_FILES src/*.cpp src/*/*.cpp tests/*cpp)
291322

292323
add_custom_target(clang-tidy
293324
${CLANG_TIDY}

README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@ Required libraries are
4949
* [zlib](https://www.zlib.net/)
5050
* [Boost libraries](https://www.boost.org/), including geometry, system and
5151
filesystem
52+
* [CImg](https://cimg.eu/) (Optional, see README-gen.md)
53+
* [potrace](https://potrace.sourceforge.net/) (Optional, see README-gen.md)
5254
* [PostgreSQL](https://www.postgresql.org/) client libraries
5355
* [Lua](https://www.lua.org/) (Optional, used for Lua tag transforms
5456
and the flex output)
@@ -80,14 +82,15 @@ On a Debian or Ubuntu system, this can be done with:
8082

8183
```sh
8284
sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \
83-
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
85+
libboost-filesystem-dev libexpat1-dev zlib1g-dev libpotrace-dev cimg-dev \
8486
libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc
8587
```
8688

8789
On a Fedora system, use
8890

8991
```sh
9092
sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \
93+
potrace-devel cimg-devel \
9194
bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc
9295
```
9396

src/gen/canvas.cpp

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/**
2+
* SPDX-License-Identifier: GPL-2.0-or-later
3+
*
4+
* This file is part of osm2pgsql (https://osm2pgsql.org/).
5+
*
6+
* Copyright (C) 2006-2023 by the osm2pgsql developer community.
7+
* For a full list of authors see the git log.
8+
*/
9+
10+
#include "canvas.hpp"
11+
#include "raster.hpp"
12+
13+
cimg_library::CImg<int> canvas_t::create_pointlist(geom::point_list_t const &pl,
14+
tile_t const &tile) const
15+
{
16+
cimg_library::CImg<int> points{static_cast<unsigned int>(pl.size()), 2};
17+
18+
int n = 0;
19+
for (auto const point : pl) {
20+
auto const tp = tile.to_tile_coords(point, m_extent);
21+
points(n, 0) = static_cast<double>(m_buffer) + tp.x();
22+
points(n, 1) = static_cast<double>(m_buffer + m_extent) - tp.y();
23+
++n;
24+
}
25+
26+
return points;
27+
}
28+
29+
std::size_t canvas_t::draw_polygon(geom::polygon_t const &polygon,
30+
tile_t const &tile)
31+
{
32+
if (polygon.inners().empty()) {
33+
m_rast.draw_polygon(create_pointlist(polygon.outer(), tile), &White);
34+
return polygon.outer().size();
35+
}
36+
37+
std::size_t num_points = polygon.outer().size();
38+
m_temp.draw_polygon(create_pointlist(polygon.outer(), tile), &White);
39+
for (auto const &inner : polygon.inners()) {
40+
num_points += inner.size();
41+
m_temp.draw_polygon(create_pointlist(inner, tile), &Black);
42+
}
43+
m_rast |= m_temp;
44+
45+
return num_points;
46+
}
47+
48+
std::size_t canvas_t::draw_linestring(geom::linestring_t const &linestring,
49+
tile_t const &tile)
50+
{
51+
m_rast.draw_line(create_pointlist(linestring, tile), &White);
52+
return linestring.size();
53+
}
54+
55+
std::size_t canvas_t::draw(geom::geometry_t const &geometry, tile_t const &tile)
56+
{
57+
if (geometry.is_linestring()) {
58+
auto const &linestring = geometry.get<geom::linestring_t>();
59+
return draw_linestring(linestring, tile);
60+
}
61+
62+
if (geometry.is_polygon()) {
63+
auto const &polygon = geometry.get<geom::polygon_t>();
64+
return draw_polygon(polygon, tile);
65+
}
66+
67+
if (geometry.is_multipolygon()) {
68+
auto const &mp = geometry.get<geom::multipolygon_t>();
69+
std::size_t num_points = 0;
70+
for (auto const &p : mp) {
71+
num_points += draw_polygon(p, tile);
72+
}
73+
return num_points;
74+
}
75+
76+
// XXX other geometry types?
77+
78+
return 0;
79+
}
80+
81+
void canvas_t::save(std::string const &filename) const
82+
{
83+
m_rast.save(filename.c_str());
84+
}
85+
86+
std::string canvas_t::to_wkb(tile_t const &tile, double margin) const
87+
{
88+
std::string wkb;
89+
wkb.reserve(61 + 2 + m_rast.size());
90+
91+
// header
92+
wkb_raster_header header{};
93+
header.nBands = 1;
94+
header.scaleX = tile.extent() / m_extent;
95+
header.scaleY = -header.scaleX;
96+
header.ipX = tile.xmin(margin);
97+
header.ipY = tile.ymax(margin);
98+
header.width = m_extent + 2 * m_buffer;
99+
header.height = header.width;
100+
add_raster_header(&wkb, header);
101+
102+
// band
103+
wkb_raster_band band{};
104+
band.bits = 4;
105+
add_raster_band(&wkb, band);
106+
107+
// rasterdata
108+
wkb.append(reinterpret_cast<char const *>(m_rast.data()), m_rast.size());
109+
110+
assert(wkb.size() == 61 + 2 + m_rast.size());
111+
112+
return wkb;
113+
}
114+
115+
void canvas_t::merge(canvas_t const &other) { m_rast |= other.m_rast; }
116+
117+
std::string to_hex(std::string const &in)
118+
{
119+
std::string result;
120+
char const *const lookup_hex = "0123456789ABCDEF";
121+
122+
for (const auto c : in) {
123+
unsigned int const num = static_cast<unsigned char>(c);
124+
result += lookup_hex[(num >> 4U) & 0xfU];
125+
result += lookup_hex[num & 0xfU];
126+
}
127+
128+
return result;
129+
}

src/gen/canvas.hpp

+88
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#ifndef OSM2PGSQL_CANVAS_HPP
2+
#define OSM2PGSQL_CANVAS_HPP
3+
4+
/**
5+
* SPDX-License-Identifier: GPL-2.0-or-later
6+
*
7+
* This file is part of osm2pgsql (https://osm2pgsql.org/).
8+
*
9+
* Copyright (C) 2006-2023 by the osm2pgsql developer community.
10+
* For a full list of authors see the git log.
11+
*/
12+
13+
#include "geom.hpp"
14+
#include "tile.hpp"
15+
16+
#define cimg_display 0 // NOLINT(cppcoreguidelines-macro-usage)
17+
#include "CImg.h"
18+
19+
#include <cstddef>
20+
21+
/**
22+
* This class wraps the image class from the CImg library.
23+
*/
24+
class canvas_t
25+
{
26+
public:
27+
static void info() { cimg_library::cimg::info(); }
28+
29+
/**
30+
* Create a new image canvas. It will be quadratic and have the width and
31+
* height extent + 2*buffer.
32+
*/
33+
canvas_t(std::size_t extent, std::size_t buffer)
34+
: m_extent(extent),
35+
m_buffer(buffer), m_rast{size(), size(), 1, 1, 0}, m_temp{size(), size(),
36+
1, 1, 0}
37+
{}
38+
39+
unsigned int size() const noexcept
40+
{
41+
return static_cast<unsigned int>(m_extent + 2 * m_buffer);
42+
}
43+
44+
unsigned char const *begin() const noexcept { return m_rast.begin(); }
45+
unsigned char const *end() const noexcept { return m_rast.end(); }
46+
47+
std::size_t draw(geom::geometry_t const &geometry, tile_t const &tile);
48+
49+
unsigned char operator()(int x, int y) const noexcept
50+
{
51+
return m_rast(x, y, 0, 0);
52+
}
53+
54+
void open_close(unsigned int buffer_size)
55+
{
56+
m_rast.dilate(buffer_size).erode(buffer_size * 2).dilate(buffer_size);
57+
}
58+
59+
void save(std::string const &filename) const;
60+
61+
std::string to_wkb(tile_t const &tile, double margin) const;
62+
63+
void merge(canvas_t const &other);
64+
65+
private:
66+
constexpr static unsigned char const Black = 0;
67+
constexpr static unsigned char const White = 255;
68+
69+
using image_type = cimg_library::CImg<unsigned char>;
70+
71+
cimg_library::CImg<int> create_pointlist(geom::point_list_t const &pl,
72+
tile_t const &tile) const;
73+
74+
std::size_t draw_polygon(geom::polygon_t const &polygon,
75+
tile_t const &tile);
76+
77+
std::size_t draw_linestring(geom::linestring_t const &linestring,
78+
tile_t const &tile);
79+
80+
std::size_t m_extent;
81+
std::size_t m_buffer;
82+
image_type m_rast;
83+
image_type m_temp;
84+
}; // class canvas_t
85+
86+
std::string to_hex(std::string const &in);
87+
88+
#endif // OSM2PGSQL_CANVAS_HPP

0 commit comments

Comments
 (0)