Merge pull request #2175 from joto/gen-append-flag

All generalizers can now differentiate between create and append mode, use this to not run ANALYZE in append mode

src/gen/gen-base.cpp

@@ -14,8 +14,8 @@
 
 #include <fmt/args.h>
 
-gen_base_t::gen_base_t(pg_conn_t *connection, params_t *params)
-: m_connection(connection), m_params(params)
+gen_base_t::gen_base_t(pg_conn_t *connection, bool append, params_t *params)
+: m_connection(connection), m_params(params), m_append(append)
 {
     assert(connection);
     assert(params);

src/gen/gen-base.hpp

@@ -67,8 +67,10 @@ class gen_base_t
         }
     }
 
+    bool append_mode() const noexcept { return m_append; }
+
 protected:
-    gen_base_t(pg_conn_t *connection, params_t *params);
+    gen_base_t(pg_conn_t *connection, bool append, params_t *params);
 
     /**
      * Check that the 'src_table' and 'dest_table' parameters exist and that
@@ -106,6 +108,7 @@ class gen_base_t
     std::vector<util::timer_t> m_timers;
     pg_conn_t *m_connection;
     params_t *m_params;
+    bool m_append;
     bool m_debug = false;
 }; // class gen_base_t
 

src/gen/gen-create.cpp

@@ -18,24 +18,25 @@
 
 std::unique_ptr<gen_base_t> create_generalizer(std::string const &strategy,
                                                pg_conn_t *connection,
-                                               params_t *params)
+                                               bool append, params_t *params)
 {
     auto generalizer = [&]() -> std::unique_ptr<gen_base_t> {
         if (strategy == "builtup") {
-            return std::make_unique<gen_tile_builtup_t>(connection, params);
+            return std::make_unique<gen_tile_builtup_t>(connection, append,
+                                                        params);
         }
         if (strategy == "discrete-isolation") {
-            return std::make_unique<gen_di_t>(connection, params);
+            return std::make_unique<gen_di_t>(connection, append, params);
         }
         if (strategy == "raster-union") {
-            return std::make_unique<gen_tile_raster_union_t>(connection,
+            return std::make_unique<gen_tile_raster_union_t>(connection, append,
                                                              params);
         }
         if (strategy == "rivers") {
-            return std::make_unique<gen_rivers_t>(connection, params);
+            return std::make_unique<gen_rivers_t>(connection, append, params);
         }
         if (strategy == "vector-union") {
-            return std::make_unique<gen_tile_vector_union_t>(connection,
+            return std::make_unique<gen_tile_vector_union_t>(connection, append,
                                                              params);
         }
         throw fmt_error("Unknown generalization strategy '{}'.", strategy);

src/gen/gen-create.hpp

@@ -21,6 +21,6 @@ class pg_conn_t;
 /// Instantiate a generalizer for the specified strategy.
 std::unique_ptr<gen_base_t> create_generalizer(std::string const &strategy,
                                                pg_conn_t *connection,
-                                               params_t *params);
+                                               bool append, params_t *params);
 
 #endif // OSM2PGSQL_GEN_CREATE_HPP

src/gen/gen-discrete-isolation.cpp

@@ -17,8 +17,8 @@
 #include <algorithm>
 #include <vector>
 
-gen_di_t::gen_di_t(pg_conn_t *connection, params_t *params)
-: gen_base_t(connection, params), m_timer_get(add_timer("get")),
+gen_di_t::gen_di_t(pg_conn_t *connection, bool append, params_t *params)
+: gen_base_t(connection, append, params), m_timer_get(add_timer("get")),
   m_timer_sort(add_timer("sort")), m_timer_di(add_timer("di")),
   m_timer_reorder(add_timer("reorder")), m_timer_write(add_timer("write"))
 {
@@ -143,7 +143,9 @@ FROM {src} WHERE {importance_column} > 0
     connection().exec("COMMIT");
     timer(m_timer_write).stop();
 
-    dbexec("ANALYZE {src}");
+    if (!append_mode()) {
+        dbexec("ANALYZE {src}");
+    }
 
     log_gen("Done.");
 }

src/gen/gen-discrete-isolation.hpp

@@ -17,7 +17,7 @@
 class gen_di_t : public gen_base_t
 {
 public:
-    gen_di_t(pg_conn_t *connection, params_t *params);
+    gen_di_t(pg_conn_t *connection, bool append, params_t *params);
 
     void process() override;
 

src/gen/gen-rivers.cpp

@@ -22,13 +22,12 @@
 #include <unordered_map>
 #include <vector>
 
-gen_rivers_t::gen_rivers_t(pg_conn_t *connection, params_t *params)
-: gen_base_t(connection, params), m_timer_area(add_timer("area")),
+gen_rivers_t::gen_rivers_t(pg_conn_t *connection, bool append, params_t *params)
+: gen_base_t(connection, append, params), m_timer_area(add_timer("area")),
   m_timer_prep(add_timer("prep")), m_timer_get(add_timer("get")),
   m_timer_sort(add_timer("sort")), m_timer_net(add_timer("net")),
   m_timer_remove(add_timer("remove")), m_timer_width(add_timer("width")),
-  m_timer_write(add_timer("write")),
-  m_delete_existing(params->has("delete_existing"))
+  m_timer_write(add_timer("write"))
 {
     check_src_dest_table_params_exist();
 
@@ -197,7 +196,11 @@ void gen_rivers_t::process()
     dbexec(R"(UPDATE {qualified_src_areas} SET width =)"
            R"( (ST_MaximumInscribedCircle("{geom_column}")).radius * 2)"
            R"( WHERE width IS NULL)");
-    dbexec("ANALYZE {qualified_src_areas}");
+
+    if (!append_mode()) {
+        dbexec("ANALYZE {qualified_src_areas}");
+    }
+
     timer(m_timer_area).stop();
 
     log_gen("Get 'width' from areas onto lines...");
@@ -323,7 +326,7 @@ SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}"
     }
     timer(m_timer_width).stop();
 
-    if (m_delete_existing) {
+    if (append_mode()) {
         dbexec("TRUNCATE {dest}");
     }
 
@@ -343,7 +346,9 @@ SELECT "{id_column}", "{width_column}", "{name_column}", "{geom_column}"
     connection().exec("COMMIT");
     timer(m_timer_write).stop();
 
-    dbexec("ANALYZE {dest}");
+    if (!append_mode()) {
+        dbexec("ANALYZE {dest}");
+    }
 
     log_gen("Done.");
 }

src/gen/gen-rivers.hpp

@@ -17,7 +17,7 @@
 class gen_rivers_t : public gen_base_t
 {
 public:
-    gen_rivers_t(pg_conn_t *connection, params_t *params);
+    gen_rivers_t(pg_conn_t *connection, bool append, params_t *params);
 
     void process() override;
 
@@ -37,7 +37,6 @@ class gen_rivers_t : public gen_base_t
 
     std::size_t m_num_waterways = 0;
     std::size_t m_num_points = 0;
-    bool m_delete_existing;
 };
 
 #endif // OSM2PGSQL_GEN_RIVERS_HPP

src/gen/gen-tile-builtup.cpp

@@ -26,8 +26,9 @@ static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept
     return ((value + multiple - 1U) / multiple) * multiple;
 }
 
-gen_tile_builtup_t::gen_tile_builtup_t(pg_conn_t *connection, params_t *params)
-: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")),
+gen_tile_builtup_t::gen_tile_builtup_t(pg_conn_t *connection, bool append,
+                                       params_t *params)
+: gen_tile_t(connection, append, params), m_timer_draw(add_timer("draw")),
   m_timer_simplify(add_timer("simplify")),
   m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write"))
 {
@@ -278,5 +279,8 @@ void gen_tile_builtup_t::post()
             }
         }
     }
-    dbexec("ANALYZE {dest}");
+
+    if (!append_mode()) {
+        dbexec("ANALYZE {dest}");
+    }
 }

src/gen/gen-tile-builtup.hpp

@@ -19,7 +19,7 @@
 class gen_tile_builtup_t final : public gen_tile_t
 {
 public:
-    gen_tile_builtup_t(pg_conn_t *connection, params_t *params);
+    gen_tile_builtup_t(pg_conn_t *connection, bool append, params_t *params);
 
     ~gen_tile_builtup_t() override = default;
 

src/gen/gen-tile-raster.cpp

@@ -26,8 +26,8 @@ static std::size_t round_up(std::size_t value, std::size_t multiple) noexcept
 }
 
 gen_tile_raster_union_t::gen_tile_raster_union_t(pg_conn_t *connection,
-                                                 params_t *params)
-: gen_tile_t(connection, params), m_timer_draw(add_timer("draw")),
+                                                 bool append, params_t *params)
+: gen_tile_t(connection, append, params), m_timer_draw(add_timer("draw")),
   m_timer_simplify(add_timer("simplify")),
   m_timer_vectorize(add_timer("vectorize")), m_timer_write(add_timer("write"))
 {
@@ -256,5 +256,8 @@ void gen_tile_raster_union_t::post()
                 fmt::format("{}_{}", m_image_table, variant));
         }
     }
-    dbexec("ANALYZE {dest}");
+
+    if (!append_mode()) {
+        dbexec("ANALYZE {dest}");
+    }
 }

src/gen/gen-tile-raster.hpp

@@ -18,7 +18,8 @@
 class gen_tile_raster_union_t final : public gen_tile_t
 {
 public:
-    gen_tile_raster_union_t(pg_conn_t *connection, params_t *params);
+    gen_tile_raster_union_t(pg_conn_t *connection, bool append,
+                            params_t *params);
 
     ~gen_tile_raster_union_t() override = default;
 

src/gen/gen-tile-vector.cpp

@@ -15,8 +15,9 @@
 #include "tile.hpp"
 
 gen_tile_vector_union_t::gen_tile_vector_union_t(pg_conn_t *connection,
-                                                 params_t *params)
-: gen_tile_t(connection, params), m_timer_simplify(add_timer("simplify"))
+                                                 bool append, params_t *params)
+: gen_tile_t(connection, append, params),
+  m_timer_simplify(add_timer("simplify"))
 {
     check_src_dest_table_params_exist();
 
@@ -95,4 +96,9 @@ void gen_tile_vector_union_t::process(tile_t const &tile)
     log_gen("Inserted {} generalized polygons", result.affected_rows());
 }
 
-void gen_tile_vector_union_t::post() { dbexec("ANALYZE {dest}"); }
+void gen_tile_vector_union_t::post()
+{
+    if (!append_mode()) {
+        dbexec("ANALYZE {dest}");
+    }
+}

src/gen/gen-tile-vector.hpp

@@ -17,7 +17,8 @@
 class gen_tile_vector_union_t final : public gen_tile_t
 {
 public:
-    gen_tile_vector_union_t(pg_conn_t *connection, params_t *params);
+    gen_tile_vector_union_t(pg_conn_t *connection, bool append,
+                            params_t *params);
 
     ~gen_tile_vector_union_t() override = default;
 

src/gen/gen-tile.cpp

@@ -16,14 +16,13 @@
 
 #include <cstdlib>
 
-gen_tile_t::gen_tile_t(pg_conn_t *connection, params_t *params)
-: gen_base_t(connection, params), m_timer_delete(add_timer("delete")),
+gen_tile_t::gen_tile_t(pg_conn_t *connection, bool append, params_t *params)
+: gen_base_t(connection, append, params), m_timer_delete(add_timer("delete")),
   m_zoom(parse_zoom())
 {
     m_with_group_by = !get_params().get_identifier("group_by_column").empty();
 
-    if (get_params().get_bool("delete_existing")) {
-        m_delete_existing = true;
+    if (append_mode()) {
         dbexec("PREPARE del_geoms (int, int) AS"
                " DELETE FROM {dest} WHERE x=$1 AND y=$2");
     }
@@ -55,7 +54,7 @@ uint32_t gen_tile_t::parse_zoom()
 
 void gen_tile_t::delete_existing(tile_t const &tile)
 {
-    if (!m_delete_existing) {
+    if (!append_mode()) {
         return;
     }
 

src/gen/gen-tile.hpp

@@ -23,7 +23,7 @@ class gen_tile_t : public gen_base_t
     uint32_t get_zoom() const noexcept override { return m_zoom; }
 
 protected:
-    gen_tile_t(pg_conn_t *connection, params_t *params);
+    gen_tile_t(pg_conn_t *connection, bool append, params_t *params);
 
     uint32_t parse_zoom();
 
@@ -34,7 +34,6 @@ class gen_tile_t : public gen_base_t
 private:
     std::size_t m_timer_delete;
     uint32_t m_zoom;
-    bool m_delete_existing = false;
     bool m_with_group_by = false;
 };
 

src/gen/osm2pgsql-gen.cpp

@@ -207,7 +207,8 @@ void run_tile_gen(connection_params_t const &connection_params,
               master_generalizer->strategy());
     pg_conn_t db_connection{connection_params, "gen.tile"};
     std::string const strategy{master_generalizer->strategy()};
-    auto generalizer = create_generalizer(strategy, &db_connection, &params);
+    auto generalizer = create_generalizer(
+        strategy, &db_connection, master_generalizer->append_mode(), &params);
 
     while (true) {
         std::pair<uint32_t, uint32_t> p;
@@ -279,18 +280,14 @@ class genproc_t
             params.set("schema", m_dbschema);
         }
 
-        if (m_append) {
-            params.set("delete_existing", true);
-        }
-
         write_to_debug_log(params, "Params (config):");
 
         log_debug("Connecting to database...");
         pg_conn_t db_connection{m_connection_params, "gen.proc"};
 
         log_debug("Creating generalizer...");
         auto generalizer =
-            create_generalizer(strategy, &db_connection, &params);
+            create_generalizer(strategy, &db_connection, m_append, &params);
 
         log_info("Running generalizer '{}' ({})...", generalizer->name(),
                  generalizer->strategy());