Skip to content

Commit dd1c8f3

Browse files
committed
WIP: Add code to generalize data using different strategies
This commit adds the code to generalize various types of data using different strategies. The following strategies work on a tile-by-tile basis and operate on polygons: The "vector-union" strategy buffers and unionizes polygons using vector operations. The "raster-union" strategy does a similar thing but does it in raster space which is much faster. First the polygons are rendered into a raster, an open/close operation is called (which basically does the same thing as the buffering in vector space) and finally the resulting raster is vectorized again. The "builtup" strategy is intended to derive a layer of builtup areas from landuse=residential/industrial etc. as well as building cover and dense road networks. This still needs some work... Also a new "discrete-isolation" strategy which rates places based on some importance metric. (This is not tile-based.) For the raster support this adds two new library dependency: CImg and potrace. The functionality is accessed through a new command line program called osm2pgsql-gen. Call it with -h to get some usage information. This program is for testing only, eventually the functionality should be accessible from osm2pgsql itself (using the Lua config file for configuration). Some additional information is in README-gen.md . See also https://osm2pgsql.org/generalization/ .
1 parent a910b86 commit dd1c8f3

30 files changed

+2378
-5
lines changed

.github/actions/ubuntu-prerequisites/action.yml

+2
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@ runs:
1616
- name: Install software
1717
run: |
1818
sudo apt-get install -yq --no-install-suggests --no-install-recommends \
19+
cimg-dev \
1920
libboost-filesystem-dev \
2021
libboost-system-dev \
2122
libbz2-dev \
2223
libexpat1-dev \
24+
libpotrace-dev \
2325
libpq-dev \
2426
libproj-dev \
2527
pandoc \

.github/actions/win-install/action.yml

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ runs:
55

66
steps:
77
- name: Install packages
8-
run: vcpkg install bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows
8+
run: vcpkg install cimg:x64-windows bzip2:x64-windows expat:x64-windows zlib:x64-windows proj4:x64-windows boost-geometry:x64-windows boost-system:x64-windows boost-filesystem:x64-windows boost-property-tree:x64-windows lua:x64-windows libpq:x64-windows
99
shell: bash
10-
1110
- name: Install psycopg2 and beahve
1211
run: python -m pip install psycopg2 behave
1312
shell: bash

.github/workflows/ci.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414

1515
- name: Install prerequisites
1616
run: |
17-
brew install lua boost postgis pandoc
17+
brew install lua boost postgis pandoc cimg potrace
1818
pip3 install psycopg2 behave
1919
pg_ctl -D /usr/local/var/postgres init
2020
pg_ctl -D /usr/local/var/postgres start

.github/workflows/test-install.yml

+2
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@ jobs:
2626
run: |
2727
sudo apt-get purge -yq postgresql*
2828
sudo apt-get install -yq --no-install-suggests --no-install-recommends \
29+
cimg-dev \
2930
libboost-filesystem-dev \
3031
libboost-system-dev \
3132
libbz2-dev \
3233
libexpat1-dev \
3334
liblua${LUA_VERSION}-dev \
3435
libluajit-5.1-dev \
36+
libpotrace-dev \
3537
libpq-dev \
3638
libproj-dev \
3739
lua${LUA_VERSION} \

CMakeLists.txt

+16
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,11 @@ include_directories(SYSTEM ${PostgreSQL_INCLUDE_DIRS})
210210

211211
find_package(Threads)
212212

213+
find_path(POTRACE_INCLUDE_DIR potracelib.h)
214+
find_library(POTRACE_LIBRARY NAMES potrace)
215+
216+
find_path(CIMG_INCLUDE_DIR CImg.h)
217+
213218
############### Libraries are found now ########################
214219

215220
set(LIBS ${Boost_LIBRARIES} ${PostgreSQL_LIBRARY} ${OSMIUM_LIBRARIES})
@@ -281,6 +286,17 @@ add_subdirectory(src)
281286
add_executable(osm2pgsql src/osm2pgsql.cpp)
282287
target_link_libraries(osm2pgsql osm2pgsql_lib ${LIBS})
283288

289+
if (${POTRACE_LIBRARY} STREQUAL "POTRACE_LIBRARY-NOTFOUND" OR ${CIMG_INCLUDE_DIR} STREQUAL "CIMG_INCLUDE_DIR-NOTFOUND")
290+
message(STATUS "Did not find cimg or potrace library. Not building osm2pgsql-gen.")
291+
else()
292+
message(STATUS "Found cimg and potrace library. Building osm2pgsql-gen.")
293+
include_directories(SYSTEM ${CIMG_INCLUDE_DIR})
294+
include_directories(SYSTEM ${POTRACE_INCLUDE_DIR})
295+
add_executable(osm2pgsql-gen src/osm2pgsql-gen.cpp src/canvas.cpp src/raster.cpp src/tracer.cpp
296+
src/gen-base.cpp src/gen-discrete-isolation.cpp src/gen-tile.cpp src/gen-tile-builtup.cpp src/gen-tile-raster.cpp src/gen-tile-vector.cpp)
297+
target_link_libraries(osm2pgsql-gen osm2pgsql_lib ${LIBS} ${POTRACE_LIBRARY})
298+
endif()
299+
284300
#############################################################
285301
# Optional "clang-tidy" target
286302
#############################################################

README-gen.md

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
2+
# Generalization
3+
4+
There is an experimental new program `osm2pgsql-gen`. It is only built if the
5+
CImg and potrace libraries are available.
6+
7+
This is part of a project to add generalization support to osm2pgsql. See
8+
https://osm2pgsql.org/generalization/ for more information.
9+
10+
Generalization is currently only supported for Web Mercator (EPSG 3857). This
11+
is by far the most common use case, we can look at extending this later if
12+
needed.
13+
14+
## Usage
15+
16+
Run osm2pgsql as usual. This example assumes you have a table called
17+
`landcover` with a polygon geometry column called `geom` and a text column
18+
called `type` with the type of the landcover (forest, grass, etc.).
19+
20+
Create a table `landcover_z10` with (at least) columns `geom` and `type` as
21+
well as integer columns `x`, and `y`. Then call this:
22+
23+
```
24+
PGDATABASE=mydb ./osm2pgsql-gen -t landcover -T landcover_z10 -g geom -G type -z 10 -m 0.125 -s raster-union
25+
```
26+
27+
The `landcover_z10` table will be filled with generalized polygons.
28+
29+
Database connection parameters have to be specified using [environment
30+
variables](https://www.postgresql.org/docs/current/libpq-envars.html).
31+
32+
Call `osm2pgsql-gen -h` for more command line options.
33+
34+
## Extent
35+
36+
You can process a single tile by setting `-x`, and `-y`. If `-X` and `-Y` are
37+
also set, all tiles between the arguments of `-x` and `-X`, and `-y` and `-Y`
38+
are processed. Without any of these, the program gets the extent from the data
39+
in the source table.
40+
41+
You can also use the option `-e, --expire-list=TABLE` to read the list of tiles
42+
from a database table.
43+
44+
In any case `-z` or `--zoom` sets the zoom level.
45+
46+
## Parameters
47+
48+
For a full list of parameters see `osm2pgsql-gen -h`.
49+
50+
* `-s, --strategy=STRATEGY`: Set the strategy used for generalization. See
51+
below for available strategies.
52+
* `-m, --margin=MARGIN`: This sets the margin around the tile as a fraction of
53+
the tile size. So a value of 0.1 sets a 10% boundary on each side of the tile,
54+
so as a result the tiles will overlap by 20% of their size. When using the
55+
`raster-union` strategy the margin will be rounded up to the nearest multiple
56+
of 64 pixels.
57+
* `-w, --width=WIDTH`: Size of the imaged rendered when using the `raster-union`
58+
strategy, not used in the `vector-union` strategy.
59+
* `-b, --buffer=BUFFER`: The amount by which the polygons will be buffered. For
60+
the `raster-union` strategy this is in pixels, for the `vector-union` strategy
61+
this is in Mercator map units.
62+
* `-g, --group-by-column=COL`: Set this to the column describing the type of
63+
polygon if any, the program will group the data by this column before
64+
generalization. If this is not specified, no grouping is performed and
65+
the destination table also doesn't need a column for this.
66+
* `--img-path=PATH`: Used to dump PNGs of the "before" and "after" images
67+
generated with the `raster-union` strategy. Use something like this:
68+
`--img-path=some/dir/path/img`. Resulting images will be in the
69+
directory `some/dir/path` and are named `img-X-Y-TYPE-[io].png` for
70+
input (`i`) or output (`o`) images. The `TYPE` is the value from the
71+
column specified with the `-G` option.
72+
* `--img-table=TABLE`: Used to dump "before" and "after" raster images to the
73+
database. The table will be created if it doesn't exist already.
74+
* `-e, --expire-list=TABLE`: Get list of tiles to expire from the specified
75+
table. If this is set the options `-x`, `-X`, `-y`, and `-Y` are ignored. The
76+
content of the table is not removed after processing!
77+
* `-p, --param=KEY=VALUE`: Set any parameter. This allows for easier
78+
experimentation with new parameters.
79+
80+
## Strategies
81+
82+
Some strategies work on a tile-by-tile basis (`vector-union`, `raster-union`,
83+
`builtup`), some work for all data at once (`discrete-isolation`).
84+
85+
* `vector-union`: Buffer and union polygons together in vector space to
86+
form generalized polygons for landcover and similar use.
87+
* `raster-union`: Buffer and union polygons together in raster space to
88+
form generalized polygons for landcover and similar use.
89+
See https://blog.jochentopf.com/2022-11-21-generalizing-polygons.html for
90+
details.
91+
* `builtup`: Aggregate data from several layers (such as landcover, buildings,
92+
and roads) to derive built-up areas.
93+
* `discrete-isolation`: Classify point geometries by importance to render
94+
only the most important places but still don't leave areas too empty.
95+
See https://blog.jochentopf.com/2022-12-19-selecting-settlements-to-display.html
96+
for some background.
97+

README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ Required libraries are
5050
* [zlib](https://www.zlib.net/)
5151
* [Boost libraries](https://www.boost.org/), including geometry, system and
5252
filesystem
53+
* [CImg](https://cimg.eu/) (Optional, see README-gen.md)
54+
* [potrace](https://potrace.sourceforge.net/) (Optional, see README-gen.md)
5355
* [PostgreSQL](https://www.postgresql.org/) client libraries
5456
* [Lua](https://www.lua.org/) (Optional, used for Lua tag transforms
5557
and the flex output)
@@ -81,14 +83,15 @@ On a Debian or Ubuntu system, this can be done with:
8183

8284
```sh
8385
sudo apt-get install make cmake g++ libboost-dev libboost-system-dev \
84-
libboost-filesystem-dev libexpat1-dev zlib1g-dev \
86+
libboost-filesystem-dev libexpat1-dev zlib1g-dev libpotrace-dev cimg-dev\
8587
libbz2-dev libpq-dev libproj-dev lua5.3 liblua5.3-dev pandoc
8688
```
8789

8890
On a Fedora system, use
8991

9092
```sh
9193
sudo dnf install cmake make gcc-c++ boost-devel expat-devel zlib-devel \
94+
potrace-devel cimg-devel\
9295
bzip2-devel postgresql-devel proj-devel proj-epsg lua-devel pandoc
9396
```
9497

src/canvas.cpp

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
/**
2+
* SPDX-License-Identifier: GPL-2.0-or-later
3+
*
4+
* This file is part of osm2pgsql (https://osm2pgsql.org/).
5+
*
6+
* Copyright (C) 2006-2022 by the osm2pgsql developer community.
7+
* For a full list of authors see the git log.
8+
*/
9+
10+
#include "canvas.hpp"
11+
#include "raster.hpp"
12+
13+
cimg_library::CImg<int> canvas_t::create_pointlist(geom::point_list_t const &pl,
14+
tile_t const &tile) const
15+
{
16+
cimg_library::CImg<int> points{static_cast<unsigned int>(pl.size()), 2};
17+
18+
int n = 0;
19+
for (auto const point : pl) {
20+
auto const tp = tile.to_tile_coords(point, m_extent);
21+
points(n, 0) = static_cast<double>(m_buffer) + tp.x();
22+
points(n, 1) = static_cast<double>(m_buffer + m_extent) - tp.y();
23+
++n;
24+
}
25+
26+
return points;
27+
}
28+
29+
std::size_t canvas_t::draw_polygon(geom::polygon_t const &polygon,
30+
tile_t const &tile)
31+
{
32+
if (polygon.inners().empty()) {
33+
m_rast.draw_polygon(create_pointlist(polygon.outer(), tile), &White);
34+
return polygon.outer().size();
35+
}
36+
37+
std::size_t num_points = polygon.outer().size();
38+
m_temp.draw_polygon(create_pointlist(polygon.outer(), tile), &White);
39+
for (auto const &inner : polygon.inners()) {
40+
num_points += inner.size();
41+
m_temp.draw_polygon(create_pointlist(inner, tile), &Black);
42+
}
43+
m_rast |= m_temp;
44+
45+
return num_points;
46+
}
47+
48+
std::size_t canvas_t::draw_linestring(geom::linestring_t const &linestring,
49+
tile_t const &tile)
50+
{
51+
m_rast.draw_line(create_pointlist(linestring, tile), &White);
52+
return linestring.size();
53+
}
54+
55+
std::size_t canvas_t::draw(geom::geometry_t const &geometry, tile_t const &tile)
56+
{
57+
if (geometry.is_linestring()) {
58+
auto const &linestring = geometry.get<geom::linestring_t>();
59+
return draw_linestring(linestring, tile);
60+
}
61+
62+
if (geometry.is_polygon()) {
63+
auto const &polygon = geometry.get<geom::polygon_t>();
64+
return draw_polygon(polygon, tile);
65+
}
66+
67+
if (geometry.is_multipolygon()) {
68+
auto const &mp = geometry.get<geom::multipolygon_t>();
69+
std::size_t num_points = 0;
70+
for (auto const &p : mp) {
71+
num_points += draw_polygon(p, tile);
72+
}
73+
return num_points;
74+
}
75+
76+
// XXX other geometry types?
77+
78+
return 0;
79+
}
80+
81+
void canvas_t::save(std::string const &filename) const
82+
{
83+
m_rast.save(filename.c_str());
84+
}
85+
86+
std::string canvas_t::to_wkb(tile_t const &tile, double margin) const
87+
{
88+
std::string wkb;
89+
wkb.reserve(61 + 2 + m_rast.size());
90+
91+
// header
92+
wkb_raster_header header{};
93+
header.nBands = 1;
94+
header.scaleX = tile.extent() / m_extent;
95+
header.scaleY = -header.scaleX;
96+
header.ipX = tile.xmin(margin);
97+
header.ipY = tile.ymax(margin);
98+
header.width = m_extent + 2 * m_buffer;
99+
header.height = header.width;
100+
add_raster_header(&wkb, header);
101+
102+
// band
103+
wkb_raster_band band{};
104+
band.bits = 4;
105+
add_raster_band(&wkb, band);
106+
107+
// rasterdata
108+
wkb.append(reinterpret_cast<char const *>(m_rast.data()), m_rast.size());
109+
110+
assert(wkb.size() == 61 + 2 + m_rast.size());
111+
112+
return wkb;
113+
}
114+
115+
void canvas_t::merge(canvas_t const &other) { m_rast |= other.m_rast; }
116+
117+
std::string to_hex(std::string const &in)
118+
{
119+
std::string result;
120+
char const *const lookup_hex = "0123456789ABCDEF";
121+
122+
for (const auto c : in) {
123+
unsigned int const num = static_cast<unsigned char>(c);
124+
result += lookup_hex[(num >> 4U) & 0xfU];
125+
result += lookup_hex[num & 0xfU];
126+
}
127+
128+
return result;
129+
}

0 commit comments

Comments
 (0)