From 44272349cfb10f6c81314922d8937aeb3128e068 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Thu, 27 Jan 2022 09:03:19 +0100 Subject: [PATCH 1/2] Work-in-progress: Add geometry functions in Lua / new insert syntax This is the future of geometry processing in Lua. Only the most important geometry functions are implemented so far. There is a new syntax for inserting rows into a table. Instead of "add_row" the "insert" function is used. "add_row" is still available and it works as before. But the new geometry functions only work with the "insert" command. The "insert" function also allows multiple geometry columns which wasn't possible with "add_row". Geometry creation functions: * as_point() (for nodes) * as_linestring() (for ways) * as_polygon() (for ways) * as_multilinestring() (for relations) * as_multipolygon() (for relations) * as_geometrycollection() (for relations) Geometry functions: * geometrytype() * srid() * transform(target_srid) * area() (for polygons) * centroid() (for polygons) * simplify(tolerance) (for linestrings) * split_multi() (for multi geometries) * #geom/__len() returns the number of geometries (0 for null geom, 1 for point/linestring/polygon, n for multi* and geometrycollection geoms) See the file flex-config/new-insert-syntax.lua for an example that shows how to use these. --- flex-config/new-insert-syntax.lua | 215 ++++++++ src/flex-table.cpp | 3 + src/flex-table.hpp | 8 + src/middle-ram.cpp | 7 + src/output-flex.cpp | 526 +++++++++++++++++++- src/output-flex.hpp | 26 +- tests/bdd/flex/collection.feature | 69 +++ tests/bdd/flex/geometry-processing.feature | 97 ++++ tests/bdd/flex/geometry-split-multi.feature | 42 ++ tests/bdd/flex/insert-area.feature | 67 +++ tests/bdd/flex/invalid-lua.feature | 12 + tests/bdd/flex/null-geom.feature | 85 ++++ 12 files changed, 1146 insertions(+), 11 deletions(-) create mode 100644 flex-config/new-insert-syntax.lua create mode 100644 tests/bdd/flex/collection.feature create mode 100644 tests/bdd/flex/geometry-processing.feature create mode 100644 tests/bdd/flex/geometry-split-multi.feature create mode 100644 tests/bdd/flex/insert-area.feature create mode 100644 tests/bdd/flex/invalid-lua.feature create mode 100644 tests/bdd/flex/null-geom.feature diff --git a/flex-config/new-insert-syntax.lua b/flex-config/new-insert-syntax.lua new file mode 100644 index 000000000..3eef0ebaa --- /dev/null +++ b/flex-config/new-insert-syntax.lua @@ -0,0 +1,215 @@ +-- This config example file is released into the Public Domain. + +-- This is Lua config showing the handling of custom geometries. +-- +-- This is "work in progress", changes are possible. +-- +-- Note that the insert() function is used instead of the old add_row()! + +-- The following magic can be used to wrap the internal "insert" function +local orig_insert = osm2pgsql.Table.insert +local function my_insert(table, data) + local json = require 'json' + inserted, message, column, object = orig_insert(table, data) + if not inserted then + for key, value in pairs(data) do + if type(value) == 'userdata' then + data[key] = tostring(value) + end + end + print("insert() failed: " .. message .. " column='" .. column .. "' object='" .. json.encode(object) .. "' data='" .. json.encode(data) .. "'") + end +end +osm2pgsql.Table.insert = my_insert + + +local tables = {} + +-- This table will get all nodes and areas with an "addr:street" tag, for +-- areas we'll calculate the centroid. +tables.addr = osm2pgsql.define_table({ + name = 'addr', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', projection = 4326 }, + } +}) + +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', projection = 4326 }, + { column = 'poly', type = 'polygon', projection = 4326 }, + { column = 'geom3857', type = 'linestring', projection = 3857 }, + { column = 'geomautoproj', type = 'linestring', projection = 3857 }, +}) + +tables.major_roads = osm2pgsql.define_way_table('major_roads', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring' }, + { column = 'sgeom', type = 'linestring' } +}) + +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry', not_null = true }, + { column = 'area4326', type = 'real' }, + { column = 'area3857', type = 'real' } +}) + +tables.routes = osm2pgsql.define_relation_table('routes', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'multilinestring', projection = 4326 }, +}) + +tables.route_parts = osm2pgsql.define_relation_table('route_parts', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', projection = 4326 }, +}) + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] +end + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + if object.tags['addr:street'] then + tables.addr:insert({ + tags = object.tags, + geom = object:as_point() + }) + end +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + if object.is_closed and object.tags['addr:street'] then + tables.addr:insert({ + tags = object.tags, + geom = object:as_polygon():centroid() + }) + end + + if object.tags.highway == 'motorway' or + object.tags.highway == 'trunk' or + object.tags.highway == 'primary' then + tables.major_roads:insert({ + tags = object.tags, + geom = object:as_linestring(), + sgeom = object:as_linestring():simplify(100), + }) + end + + -- A closed way that also has the right tags for an area is a polygon. + if object.is_closed and has_area_tags(object.tags) then + local g = object:as_polygon() + local a = g:area() + if a < 0.0000001 then + tables.polygons:insert({ + tags = object.tags, + geom = g, + area4326 = a, + area3857 = g:transform(3857):area() + }) + end + else + tables.ways:insert({ + tags = object.tags, + geom = object:as_linestring(), + poly = object:as_polygon(), + geom3857 = object:as_linestring():transform(3857), -- project geometry into target srid 3857 + geomautoproj = object:as_linestring() -- automatically projected into projection of target column + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + local relation_type = object:grab_tag('type') + + if relation_type == 'multipolygon' then + local g = object:as_multipolygon() + local a = g:area() + if a < 0.0000001 then + tables.polygons:insert({ + tags = object.tags, + geom = g, + area4326 = a, + area3857 = g:transform(3857):area() + }) + end + return + end + + if relation_type == 'route' then + local route_geom = object:as_multilinestring() + if #route_geom > 0 then -- check that this is not a null geometry +-- print(object.id, tostring(route_geom), route_geom:srid(), #route_geom) + tables.routes:insert({ + tags = object.tags, + geom = route_geom + }) + for n, line in pairs(route_geom:split_multi()) do + tables.route_parts:insert({ + tags = object.tags, + geom = line + }) + end + end + end +end + diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 9b02b59ff..2893882c3 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -77,6 +77,9 @@ flex_table_column_t &flex_table_t::add_column(std::string const &name, auto &column = m_columns.back(); if (column.is_geometry_column()) { + if (m_geom_column != std::numeric_limits::max()) { + m_has_multiple_geom_columns = true; + } m_geom_column = m_columns.size() - 1; column.set_not_null(); } diff --git a/src/flex-table.hpp b/src/flex-table.hpp index 357e8b386..14b36cc09 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -191,6 +191,11 @@ class flex_table_t std::string full_name() const; std::string full_tmp_name() const; + bool has_multiple_geom_columns() const noexcept + { + return m_has_multiple_geom_columns; + } + private: /// The name of the table std::string m_name; @@ -225,6 +230,9 @@ class flex_table_t /// Cluster the table by geometry. bool m_cluster_by_geom = true; + /// Does this table have more than one geometry column? + bool m_has_multiple_geom_columns = false; + }; // class flex_table_t class table_connection_t diff --git a/src/middle-ram.cpp b/src/middle-ram.cpp index 486fe7a5c..87f59e095 100644 --- a/src/middle-ram.cpp +++ b/src/middle-ram.cpp @@ -264,6 +264,13 @@ middle_ram_t::rel_members_get(osmium::Relation const &rel, buffer->commit(); ++count; } + } else { + { + osmium::builder::NodeBuilder builder{*buffer}; + builder.set_id(member.ref()); + } + buffer->commit(); + ++count; } break; case osmium::item_type::way: diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 54bdcdafc..d4cd528b3 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -10,6 +10,7 @@ #include "db-copy.hpp" #include "expire-tiles.hpp" #include "format.hpp" +#include "geom-from-osm.hpp" #include "geom-functions.hpp" #include "geom-transform.hpp" #include "logging.hpp" @@ -70,18 +71,37 @@ static std::mutex lua_mutex; } \ } +TRAMPOLINE(app_geometry_gc, geometry_gc) TRAMPOLINE(app_define_table, define_table) TRAMPOLINE(app_get_bbox, get_bbox) +TRAMPOLINE(app_as_linestring, as_linestring) +TRAMPOLINE(app_as_point, as_point) +TRAMPOLINE(app_as_polygon, as_polygon) +TRAMPOLINE(app_as_multilinestring, as_multilinestring) +TRAMPOLINE(app_as_multipolygon, as_multipolygon) +TRAMPOLINE(app_as_geometrycollection, as_geometrycollection) +TRAMPOLINE(geom_len, __len) +TRAMPOLINE(geom_tostring, __tostring) +TRAMPOLINE(geom_geometrytype, geometrytype) +TRAMPOLINE(geom_is_null, is_null) +TRAMPOLINE(geom_srid, srid) +TRAMPOLINE(geom_area, area) +TRAMPOLINE(geom_centroid, centroid) +TRAMPOLINE(geom_transform, transform) +TRAMPOLINE(geom_split_multi, split_multi) +TRAMPOLINE(geom_simplify, simplify) TRAMPOLINE(table_name, name) TRAMPOLINE(table_schema, schema) TRAMPOLINE(table_cluster, cluster) TRAMPOLINE(table_add_row, add_row) +TRAMPOLINE(table_insert, insert) TRAMPOLINE(table_columns, columns) TRAMPOLINE(table_tostring, __tostring) static char const *const osm2pgsql_table_name = "osm2pgsql.Table"; static char const *const osm2pgsql_object_metatable = "osm2pgsql.object_metatable"; +static char const *const osm2pgsql_geometry_name = "osm2pgsql.Geometry"; prepared_lua_function_t::prepared_lua_function_t(lua_State *lua_state, calling_context context, @@ -235,13 +255,28 @@ static int sgn(double val) noexcept return 0; } +class not_null_exception : public std::runtime_error +{ +public: + not_null_exception(std::string const &message, + flex_table_column_t const *column) + : std::runtime_error(message), m_column(column) + {} + + flex_table_column_t const &column() const noexcept { return *m_column; } + +private: + flex_table_column_t const *m_column; +}; // class not_null_exception + static void write_null(db_copy_mgr_t *copy_mgr, flex_table_column_t const &column) { if (column.not_null()) { - throw std::runtime_error{ + throw not_null_exception{ "Can not add NULL to column '{}' declared NOT NULL."_format( - column.name())}; + column.name()), + &column}; } copy_mgr->add_null_column(); } @@ -468,6 +503,22 @@ static void write_json(json_writer_type *writer, lua_State *lua_state, } } +geom::geometry_t *unpack_geometry(lua_State *lua_state) { + void *user_data = lua_touserdata(lua_state, -1); + + if (user_data == nullptr || !lua_getmetatable(lua_state, -1)) { + throw std::runtime_error{"First parameter must be a geometry."}; + } + + luaL_getmetatable(lua_state, osm2pgsql_geometry_name); + if (!lua_rawequal(lua_state, -1, -2)) { + throw std::runtime_error{"First parameter must be a geometry."}; + } + lua_pop(lua_state, 2); + + return static_cast(user_data); +} + void output_flex_t::write_column( db_copy_mgr_t *copy_mgr, flex_table_column_t const &column) @@ -484,10 +535,9 @@ void output_flex_t::write_column( int const ltype = lua_type(lua_state(), -1); // Certain Lua types can never be added to the database - if (ltype == LUA_TFUNCTION || ltype == LUA_TUSERDATA || - ltype == LUA_TTHREAD) { + if (ltype == LUA_TFUNCTION || ltype == LUA_TTHREAD) { throw std::runtime_error{ - "Can not add Lua objects of type function, userdata, or thread."}; + "Can not add Lua objects of type function or thread."}; } // A Lua nil value is always translated to a database NULL @@ -640,9 +690,44 @@ void output_flex_t::write_column( "Invalid type '{}' for direction column."_format( lua_typename(lua_state(), ltype))}; } + } else if (column.is_geometry_column()) { + // If this is a geometry column, the Lua function 'insert()' was + // called, because for 'add_row()' geometry columns are handled + // earlier and 'write_column()' is not called. + if (ltype == LUA_TUSERDATA) { + auto const *const geom = unpack_geometry(lua_state()); + if (geom && !geom->is_null()) { + if (geom->srid() == column.srid()) { + // OSM id not available here, so use dummy 0, it is used + // for debug messages only anyway. + m_expire.from_geometry(*geom, 0); + copy_mgr->add_hex_geom(geom_to_ewkb(*geom)); + } else { + auto const proj = + reprojection::create_projection(column.srid()); + auto const tgeom = geom::transform(*geom, *proj); + // OSM id not available here, so use dummy 0, it is used + // for debug messages only anyway. + m_expire.from_geometry(tgeom, 0); + copy_mgr->add_hex_geom(geom_to_ewkb(tgeom)); + } + } else { + write_null(copy_mgr, column); + } + } else { + throw std::runtime_error{ + "Need geometry data for geometry column '{}'."_format( + column.name())}; + } + } else if (column.type() == table_column_type::area) { + // If this is an area column, the Lua function 'insert()' was + // called, because for 'add_row()' area columns are handled + // earlier and 'write_column()' is not called. + throw std::runtime_error{"Column type 'area' not allowed with " + "'insert()'. Maybe use 'real'?"}; } else { - throw std::runtime_error{ - "Column type {} not implemented."_format(static_cast(column.type()))}; + throw std::runtime_error{"Column type {} not implemented."_format( + static_cast(column.type()))}; } lua_pop(lua_state(), 1); @@ -786,6 +871,253 @@ int output_flex_t::app_get_bbox() return 0; } +template +void create_lua_geometry_object(lua_State *lua_state, CREATE &&create) +{ + void *ptr = lua_newuserdata(lua_state, sizeof(geom::geometry_t)); + new (ptr) geom::geometry_t{}; + std::forward(create)(static_cast(ptr)); + + // Set the metatable of this object + luaL_getmetatable(lua_state, osm2pgsql_geometry_name); + lua_setmetatable(lua_state, -2); +} + +int output_flex_t::app_as_point() +{ + check_context_and_state("as_point", "process_node() function", + m_calling_context != calling_context::process_node); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_point(geom, *m_context_node); + }); + + return 1; +} + +int output_flex_t::app_as_linestring() +{ + check_context_and_state("as_linestring", "process_way() function", + m_calling_context != calling_context::process_way); + + m_way_cache.add_nodes(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_linestring(geom, m_way_cache.get()); + }); + + return 1; +} + +int output_flex_t::app_as_polygon() +{ + check_context_and_state("as_polygon", "process_way() function", + m_calling_context != calling_context::process_way); + + m_way_cache.add_nodes(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_polygon(geom, m_way_cache.get()); + }); + + return 1; +} + +int output_flex_t::app_as_multilinestring() +{ + check_context_and_state("as_multilinestring", "process_relation() function", + m_calling_context != + calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_multilinestring(geom, m_relation_cache.members_buffer()); + }); + + return 1; +} + +int output_flex_t::app_as_multipolygon() +{ + check_context_and_state("as_multipolygon", "process_relation() function", + m_calling_context != + calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_multipolygon(geom, m_relation_cache.get(), + m_relation_cache.members_buffer()); + }); + + return 1; +} + +int output_flex_t::app_as_geometrycollection() +{ + check_context_and_state( + "as_geometrycollection", "process_relation() function", + m_calling_context != calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_collection(geom, m_relation_cache.members_buffer()); + }); + + return 1; +} + +// XXX Implementation for Lua __tostring function on geometries. Currently +// just returns the type as string. This could be improved, for instance by +// showing a WKT representation of the geometry. +int output_flex_t::geom_tostring() +{ + return geom_geometrytype(); +} + +int output_flex_t::geom_geometrytype() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + auto const type = geometry_type(*input_geometry); + lua_pushlstring(lua_state(), type.data(), type.size()); + return 1; +} + +int output_flex_t::geom_len() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushinteger(lua_state(), + static_cast(num_geometries(*input_geometry))); + return 1; +} + +int output_flex_t::geom_is_null() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushboolean(lua_state(), input_geometry->is_null()); + return 1; +} + +int output_flex_t::geom_srid() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushinteger(lua_state(), + static_cast(input_geometry->srid())); + return 1; +} + +int output_flex_t::geom_area() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for area()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + double const area = geom::area(*input_geometry); + lua_pushnumber(lua_state(), area); + + return 1; +} + +int output_flex_t::geom_centroid() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for centroid()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = geom::centroid(*input_geometry); + }); + + return 1; +} + +int output_flex_t::geom_transform() +{ + if (lua_gettop(lua_state()) != 2) { + throw std::runtime_error{ + "One parameter needed for transform(), the target srid."}; + } + +#if LUA_VERSION_NUM >= 503 + if (!lua_isinteger(lua_state(), -1)) { +#else + if (!lua_isnumber(lua_state(), -1)) { +#endif + throw std::runtime_error{ + "transform() srid parameter has to be an integer."}; + } + + int const srid = lua_tointeger(lua_state(), -1); + lua_pop(lua_state(), 1); // srid parameter + + auto const *const input_geometry = unpack_geometry(lua_state()); + + if (input_geometry->srid() != 4326) { + throw std::runtime_error{ + "Can not transform already transformed geometry."}; + } + auto const proj = reprojection::create_projection(srid); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::transform(geom, *input_geometry, *proj); + }); + + return 1; +} + +// XXX this is inefficient, because we are copying the whole geometry and +// creating a large Lua array. Better implement this with an iterator? +int output_flex_t::geom_split_multi() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for split_multi()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + auto const geoms = geom::split_multi(*input_geometry, true); + + lua_createtable(lua_state(), (int)geoms.size(), 0); + int n = 0; + for (auto const &g : geoms) { + lua_pushinteger(lua_state(), ++n); + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = std::move(g); + }); + lua_rawset(lua_state(), -3); + } + + return 1; +} + +int output_flex_t::geom_simplify() +{ + if (lua_gettop(lua_state()) != 2) { + throw std::runtime_error{ + "simplify() function needs one parameter: tolerance."}; + } + + if (!lua_isnumber(lua_state(), -1)) { + throw std::runtime_error{ + "simplify() tolerance parameters has to be a number."}; + } + + double const tolerance = lua_tonumber(lua_state(), -1); + lua_pop(lua_state(), 1); // tolerance parameter + + auto const *const input_geometry = unpack_geometry(lua_state()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = geom::simplify(*input_geometry, tolerance); + }); + + return 1; +} + static void check_name(std::string const &name, char const *in) { auto const pos = name.find_first_of("\"',.;$%&/()<>{}=?^*#"); @@ -981,6 +1313,18 @@ void output_flex_t::setup_flex_table_columns(flex_table_t *table) } } +int output_flex_t::app_geometry_gc() +{ + void *geom = lua_touserdata(lua_state(), 1); + if (geom) { + static_cast(geom)->~geometry_t(); + } + + lua_pop(lua_state(), 1); // pop parameter from stack + + return 0; +} + int output_flex_t::app_define_table() { if (m_calling_context != calling_context::main) { @@ -1099,7 +1443,7 @@ void output_flex_t::relation_cache_t::init(osmium::Relation const &relation) bool output_flex_t::relation_cache_t::add_members(middle_query_t const &middle) { - if (m_members_buffer.committed() == 0) { + if (members_buffer().committed() == 0) { auto const num_members = middle.rel_members_get( *m_relation, &m_members_buffer, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way); @@ -1108,6 +1452,12 @@ bool output_flex_t::relation_cache_t::add_members(middle_query_t const &middle) return false; } + for (auto &node : m_members_buffer.select()) { + if (!node.location().valid()) { + node.set_location(middle.get_node_location(node.id())); + } + } + for (auto &nodes : m_members_buffer.select()) { middle.nodes_get_list(&nodes); } @@ -1175,6 +1525,99 @@ int output_flex_t::table_add_row() return 0; } +osmium::OSMObject const & +output_flex_t::check_and_get_context_object(flex_table_t const &table) +{ + if (m_calling_context == calling_context::process_node) { + if (!table.matches_type(osmium::item_type::node)) { + throw std::runtime_error{ + "Trying to add node to table '{}'."_format(table.name())}; + } + return *m_context_node; + } + + if (m_calling_context == calling_context::process_way) { + if (!table.matches_type(osmium::item_type::way)) { + throw std::runtime_error{ + "Trying to add way to table '{}'."_format(table.name())}; + } + return m_way_cache.get(); + } + + assert(m_calling_context == calling_context::process_relation); + + if (!table.matches_type(osmium::item_type::relation)) { + throw std::runtime_error{ + "Trying to add relation to table '{}'."_format(table.name())}; + } + return m_relation_cache.get(); +} + +int output_flex_t::table_insert() +{ + if (m_disable_add_row) { + return 0; + } + + if (m_calling_context != calling_context::process_node && + m_calling_context != calling_context::process_way && + m_calling_context != calling_context::process_relation) { + throw std::runtime_error{ + "The function insert() can only be called from the " + "process_node/way/relation() functions."}; + } + + auto const num_params = lua_gettop(lua_state()); + if (num_params != 2) { + throw std::runtime_error{ + "Need two parameters: The osm2pgsql.table and the row data."}; + } + + // The first parameter is the table object. + auto &table_connection = + m_table_connections.at(table_idx_from_param(lua_state())); + + // The second parameter must be a Lua table with the contents for the + // fields. + luaL_checktype(lua_state(), 2, LUA_TTABLE); + lua_remove(lua_state(), 1); + + auto const &table = table_connection.table(); + auto const &object = check_and_get_context_object(table); + osmid_t const id = table.map_id(object.type(), object.id()); + + table_connection.new_line(); + auto *copy_mgr = table_connection.copy_mgr(); + + try { + for (auto const &column : table_connection.table()) { + if (column.create_only()) { + continue; + } + if (column.type() == table_column_type::id_type) { + copy_mgr->add_column(type_to_char(object.type())); + } else if (column.type() == table_column_type::id_num) { + copy_mgr->add_column(id); + } else { + write_column(copy_mgr, column); + } + } + } catch (not_null_exception const &e) { + copy_mgr->rollback_line(); + lua_pushboolean(lua_state(), false); + lua_pushstring(lua_state(), "null value in not null column."); + lua_pushstring(lua_state(), e.column().name().c_str()); + push_osm_object_to_lua_stack(lua_state(), object, + get_options()->extra_attributes); + return 4; + } + + copy_mgr->finish_line(); + + lua_pushboolean(lua_state(), true); + return 1; +} + int output_flex_t::table_columns() { auto const &table = get_table_from_param(); @@ -1231,11 +1674,21 @@ get_transform(lua_State *lua_state, flex_table_column_t const &column) lua_getfield(lua_state, -1, column.name().c_str()); int const ltype = lua_type(lua_state, -1); - if (ltype != LUA_TTABLE) { + + // Field not set, return null transform + if (ltype == LUA_TNIL) { lua_pop(lua_state, 1); // geom field return transform; } + // Field set to anything but a Lua table is not allowed + if (ltype != LUA_TTABLE) { + lua_pop(lua_state, 1); // geom field + throw std::runtime_error{ + "Invalid geometry transformation for column '{}'."_format( + column.name())}; + } + lua_getfield(lua_state, -1, "create"); char const *create_type = lua_tostring(lua_state, -1); if (create_type == nullptr) { @@ -1326,6 +1779,13 @@ void output_flex_t::add_row(table_connection_t *table_connection, assert(table_connection); auto const &table = table_connection->table(); + if (table.has_multiple_geom_columns()) { + throw std::runtime_error{ + "Table '{}' has more than one geometry column." + " This is not allowed with 'add_row()'." + " Maybe use 'insert()' instead?"_format(table.name())}; + } + osmid_t const id = table.map_id(object.type(), object.id()); if (!table.has_geom_column()) { @@ -1743,12 +2203,40 @@ static void init_table_class(lua_State *lua_state) lua_setfield(lua_state, -2, "__index"); luaX_add_table_func(lua_state, "__tostring", lua_trampoline_table_tostring); luaX_add_table_func(lua_state, "add_row", lua_trampoline_table_add_row); + luaX_add_table_func(lua_state, "insert", lua_trampoline_table_insert); luaX_add_table_func(lua_state, "name", lua_trampoline_table_name); luaX_add_table_func(lua_state, "schema", lua_trampoline_table_schema); luaX_add_table_func(lua_state, "cluster", lua_trampoline_table_cluster); luaX_add_table_func(lua_state, "columns", lua_trampoline_table_columns); } +/** + * Define the osm2pgsql.Geometry class/metatable. + */ +static void init_geometry_class(lua_State *lua_state) +{ + if (luaL_newmetatable(lua_state, osm2pgsql_geometry_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + + luaX_add_table_func(lua_state, "__gc", lua_trampoline_app_geometry_gc); + luaX_add_table_func(lua_state, "__len", lua_trampoline_geom_len); + luaX_add_table_func(lua_state, "__tostring", lua_trampoline_geom_tostring); + lua_pushvalue(lua_state, -1); + lua_setfield(lua_state, -2, "__index"); + luaX_add_table_func(lua_state, "is_null", lua_trampoline_geom_is_null); + luaX_add_table_func(lua_state, "srid", lua_trampoline_geom_srid); + luaX_add_table_func(lua_state, "area", lua_trampoline_geom_area); + luaX_add_table_func(lua_state, "geometrytype", + lua_trampoline_geom_geometrytype); + luaX_add_table_func(lua_state, "centroid", lua_trampoline_geom_centroid); + luaX_add_table_func(lua_state, "transform", lua_trampoline_geom_transform); + luaX_add_table_func(lua_state, "split_multi", + lua_trampoline_geom_split_multi); + luaX_add_table_func(lua_state, "simplify", lua_trampoline_geom_simplify); + lua_pop(lua_state, 1); // __index +} + void output_flex_t::init_lua(std::string const &filename) { m_lua_state.reset(luaL_newstate(), @@ -1788,10 +2276,22 @@ void output_flex_t::init_lua(std::string const &filename) lua_tostring(lua_state(), -1))}; } - // Store the "get_bbox" in the "object_metatable". + // Store the methods on OSM objects in its metatable. lua_getglobal(lua_state(), "object_metatable"); lua_getfield(lua_state(), -1, "__index"); luaX_add_table_func(lua_state(), "get_bbox", lua_trampoline_app_get_bbox); + luaX_add_table_func(lua_state(), "as_linestring", + lua_trampoline_app_as_linestring); + luaX_add_table_func(lua_state(), "as_point", + lua_trampoline_app_as_point); + luaX_add_table_func(lua_state(), "as_polygon", + lua_trampoline_app_as_polygon); + luaX_add_table_func(lua_state(), "as_multilinestring", + lua_trampoline_app_as_multilinestring); + luaX_add_table_func(lua_state(), "as_multipolygon", + lua_trampoline_app_as_multipolygon); + luaX_add_table_func(lua_state(), "as_geometrycollection", + lua_trampoline_app_as_geometrycollection); lua_settop(lua_state(), 0); // Store the global object "object_metatable" defined in the init.lua @@ -1803,6 +2303,12 @@ void output_flex_t::init_lua(std::string const &filename) lua_pushnil(lua_state()); lua_setglobal(lua_state(), "object_metatable"); + assert(lua_gettop(lua_state()) == 0); + + init_geometry_class(lua_state()); + + assert(lua_gettop(lua_state()) == 0); + // Load user config file luaX_set_context(lua_state(), this); if (luaL_dofile(lua_state(), filename.c_str())) { diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 4a463dd44..9849e1ec3 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -153,12 +153,32 @@ class output_flex_t : public output_t void merge_expire_trees(output_t *other) override; + int app_as_point(); + int app_as_linestring(); + int app_as_polygon(); + int app_as_multilinestring(); + int app_as_multipolygon(); + int app_as_geometrycollection(); + int app_define_table(); - int app_mark_way(); + int app_geometry_gc(); int app_get_bbox(); + int app_mark_way(); + + int geom_area(); + int geom_centroid(); + int geom_geometrytype(); + int geom_is_null(); + int geom_len(); + int geom_simplify(); + int geom_split_multi(); + int geom_srid(); + int geom_tostring(); + int geom_transform(); int table_tostring(); int table_add_row(); + int table_insert(); int table_name(); int table_schema(); int table_cluster(); @@ -192,10 +212,14 @@ class output_flex_t : public output_t void write_column(db_copy_mgr_t *copy_mgr, flex_table_column_t const &column); + void write_row(table_connection_t *table_connection, osmium::item_type id_type, osmid_t id, geom::geometry_t const &geom, int srid); + osmium::OSMObject const & + check_and_get_context_object(flex_table_t const &table); + geom::geometry_t run_transform(reprojection const &proj, geom_transform_t const *transform, osmium::Node const &node); diff --git a/tests/bdd/flex/collection.feature b/tests/bdd/flex/collection.feature new file mode 100644 index 000000000..43609eddf --- /dev/null +++ b/tests/bdd/flex/collection.feature @@ -0,0 +1,69 @@ +Feature: Create geometry collections from relations + + Background: + Given the lua style + """ + local dtable = osm2pgsql.define_table{ + name = 'osm2pgsql_test_collection', + ids = { type = 'relation', id_column = 'osm_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 } + } + } + + function osm2pgsql.process_relation(object) + dtable:insert({ + name = object.tags.name, + geom = object.as_geometrycollection() + }) + end + """ + + Scenario: Create geometry collection from different relations + Given the 1.0 grid + | 13 | 12 | 17 | | 16 | + | 10 | 11 | | 14 | 15 | + And the OSM data + """ + w20 Nn10,n11,n12,n13,n10 + w21 Nn14,n15,n16 + r30 Tname=single Mw20@ + r31 Tname=multi Mw20@,w21@ + r32 Tname=mixed Mn17@,w21@ + r33 Tname=node Mn17@ + """ + When running osm2pgsql flex + + Then table osm2pgsql_test_collection contains exactly + | osm_id | name | ST_GeometryType(geom) | ST_NumGeometries(geom) | ST_GeometryType(ST_GeometryN(geom, 1)) | + | 30 | single | ST_GeometryCollection | 1 | ST_LineString | + | 31 | multi | ST_GeometryCollection | 2 | ST_LineString | + | 32 | mixed | ST_GeometryCollection | 2 | ST_Point | + | 33 | node | ST_GeometryCollection | 1 | ST_Point | + + And table osm2pgsql_test_collection contains exactly + | osm_id | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | + | 30 | 10, 11, 12, 13, 10 | NULL | + | 31 | 10, 11, 12, 13, 10 | 14, 15, 16 | + | 32 | 17 | 14, 15, 16 | + | 33 | 17 | NULL | + + Scenario: NULL entry generated for broken geometries + Given the grid + | 10 | + And the OSM data + """ + w20 Nn10 + r30 Tname=foo Mn11@ + r31 Tname=bar Mw20@ + r32 Tname=baz Mw21@ + """ + When running osm2pgsql flex + + Then table osm2pgsql_test_collection contains exactly + | osm_id | name | ST_GeometryType(geom) | + | 30 | foo | NULL | + | 31 | bar | NULL | + | 32 | baz | NULL | + diff --git a/tests/bdd/flex/geometry-processing.feature b/tests/bdd/flex/geometry-processing.feature new file mode 100644 index 000000000..14c51a022 --- /dev/null +++ b/tests/bdd/flex/geometry-processing.feature @@ -0,0 +1,97 @@ +Feature: Tests for Lua geometry processing functions + + Scenario: + Given the OSM data + """ + n1 Tamenity=restaurant,name=point x1.1 y1.2 + """ + And the lua style + """ + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'name', type = 'text' }, + { column = 'geom4326', type = 'point', projection = 4326 }, + { column = 'geom3857', type = 'point', projection = 3857 }, + { column = 'geomauto', type = 'point', projection = 3857 }, + }) + + function osm2pgsql.process_node(object) + points:insert({ + name = object.tags.name, + geom4326 = object:as_point(), + geom3857 = object:as_point():transform(3857), + geomauto = object:as_point() + }) + end + + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains + | node_id | name | ST_AsText(geom4326) | geom3857 = geomauto | + | 1 | point | 1.1 1.2 | True | + + Scenario: + Given the 0.1 grid with origin 9.0 50.3 + | | | 7 | | | 8 | + | | | | 11 | 12 | | + | 3 | 4 | | 9 | 10 | | + | 1 | 2 | 5 | | | 6 | + And the OSM data + """ + w1 Tnatural=water,name=poly Nn1,n2,n4,n3,n1 + w2 Nn5,n6,n8,n7,n5 + w3 Nn9,n10,n12,n11,n9 + r1 Tnatural=water,name=multi Mw2@,w3@ + """ + And the lua style + """ + local tables = {} + + tables.ways = osm2pgsql.define_way_table('osm2pgsql_test_ways', { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'linestring', projection = 4326 }, + { column = 'geomsimple', type = 'linestring', projection = 4326 }, + }) + + tables.polygons = osm2pgsql.define_area_table('osm2pgsql_test_polygons', { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 }, + { column = 'center', type = 'point', projection = 4326 }, + }) + + function is_empty(some_table) + return next(some_table) == nil + end + + function osm2pgsql.process_way(object) + if is_empty(object.tags) then + return + end + + tables.ways:insert({ + name = object.tags.name, + geom = object:as_linestring(), + geomsimple = object:as_linestring():simplify(0.1) + }) + tables.polygons:insert({ + name = object.tags.name, + geom = object:as_polygon(), + center = object:as_polygon():centroid() + }) + end + + function osm2pgsql.process_relation(object) + tables.polygons:insert({ + name = object.tags.name, + geom = object:as_multipolygon() + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_ways contains + | way_id | name | ST_AsText(geom) | ST_AsText(geomsimple) | + | 1 | poly | 1, 2, 4, 3, 1 | 1, 4, 1 | + And table osm2pgsql_test_polygons contains + | area_id | name | ST_AsText(geom) | ST_AsText(center) | + | 1 | poly | (1, 2, 4, 3, 1) | 9.05 50.05 | + | -1 | multi | (5, 6, 8, 7, 5),(9, 11, 12, 10, 9) | NULL | + diff --git a/tests/bdd/flex/geometry-split-multi.feature b/tests/bdd/flex/geometry-split-multi.feature new file mode 100644 index 000000000..cd58f867b --- /dev/null +++ b/tests/bdd/flex/geometry-split-multi.feature @@ -0,0 +1,42 @@ +Feature: Tests for geometry split_multi function + + Scenario: + Given the grid + | 1 | 2 | | + | 4 | | 3 | + | | 5 | 6 | + And the OSM data + """ + w20 Thighway=motorway Nn1,n2,n3 + w21 Thighway=motorway Nn4,n5,n6 + r30 Ttype=route,route=road Mw20@,w21@ + r31 Ttype=route,route=road Mw20@ + r32 Ttype=something Mw20@ + r33 Ttype=route,route=road Mn1@ + """ + And the lua style + """ + local routes = osm2pgsql.define_relation_table('osm2pgsql_test_routes', { + { column = 'geom', type = 'linestring', projection = 4326 }, + }) + + function osm2pgsql.process_relation(object) + if object.tags.type == 'route' then + local g = object:as_multilinestring() + if not g:is_null() then + for n, line in pairs(g:split_multi()) do + routes:insert({ + geom = line + }) + end + end + end + end + + """ + When running osm2pgsql flex + Then table osm2pgsql_test_routes contains exactly + | relation_id | ST_AsText(geom) | + | 30 | 1, 2, 3 | + | 30 | 4, 5, 6 | + | 31 | 1, 2, 3 | diff --git a/tests/bdd/flex/insert-area.feature b/tests/bdd/flex/insert-area.feature new file mode 100644 index 000000000..ba8f5a040 --- /dev/null +++ b/tests/bdd/flex/insert-area.feature @@ -0,0 +1,67 @@ +Feature: Tests for area() function + + Scenario Outline: + Given the 0.1 grid with origin 9.0 50.3 + | | | 7 | | | 8 | + | | | | 11 | 12 | | + | 3 | 4 | | 9 | 10 | | + | 1 | 2 | 5 | | | 6 | + And the OSM data + """ + w1 Tnatural=water,name=poly Nn1,n2,n4,n3,n1 + w2 Nn5,n6,n8,n7,n5 + w3 Nn9,n10,n12,n11,n9 + r1 Tnatural=water,name=multi Mw2@,w3@ + """ + And the lua style + """ + local polygons = osm2pgsql.define_table{ + name = 'osm2pgsql_test_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = }, + { column = 'area', type = 'real' }, + } + } + + function osm2pgsql.process_way(object) + local polygon = object:as_polygon() + polygons:insert({ + name = object.tags.name, + geom = polygon, + area = polygon:transform():area() + }) + end + + function osm2pgsql.process_relation(object) + local polygon = object:as_multipolygon() + polygons:insert({ + name = object.tags.name, + geom = polygon, + area = polygon:transform():area() + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_polygon contains + | name | ST_Area(geom) | area | ST_Area(ST_Transform(geom, 4326)) | + | poly | | | 0.01 | + | multi | | | 0.08 | + + Examples: + | geom proj | area proj | st_area poly | area poly | st_area multi | area multi | + | 4326 | 4326 | 0.01 | 0.01 | 0.08 | 0.08 | + | 4326 | 3857 | 0.01 | 192987010.0 | 0.08 | 1547130000.0 | + | 3857 | 4326 | 192987010.0 | 0.01 | 1547130000.0 | 0.08 | + | 3857 | 3857 | 192987010.0 | 192987010.0 | 1547130000.0 | 1547130000.0 | + + @config.have_proj + Examples: Generic projection + | geom proj | area proj | st_area poly | area poly | st_area multi | area multi | + | 4326 | 25832 | 0.01 | 79600737.537 | 0.08 | 635499542.954 | + | 3857 | 25832 | 192987010.0 | 79600737.537 | 1547130000.0 | 635499542.954 | + | 25832 | 4326 | 79600737.537 | 0.01 | 635499542.954 | 0.08 | + | 25832 | 3857 | 79600737.537 | 192987010.0 | 635499542.954 | 1547130000.0 | + | 25832 | 25832 | 79600737.537 | 79600737.537 | 635499542.954 | 635499542.954 | + diff --git a/tests/bdd/flex/invalid-lua.feature b/tests/bdd/flex/invalid-lua.feature new file mode 100644 index 000000000..36309b172 --- /dev/null +++ b/tests/bdd/flex/invalid-lua.feature @@ -0,0 +1,12 @@ +Feature: Tests for basic Lua functions + + Scenario: + Given the OSM data + """ + n1 Tamenity=restaurant x10 y10 + """ + And the lua style + """ + this-is-not-valid-lua + """ + Then running osm2pgsql flex fails diff --git a/tests/bdd/flex/null-geom.feature b/tests/bdd/flex/null-geom.feature new file mode 100644 index 000000000..922c0cbd8 --- /dev/null +++ b/tests/bdd/flex/null-geom.feature @@ -0,0 +1,85 @@ +Feature: Null geometry handling + + Scenario: Invalid geometries show up as NULL + Given the lua style + """ + local tables = {} + + tables.null = osm2pgsql.define_table{ + name = 'osm2pgsql_test_null', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 } + } + } + + tables.not_null = osm2pgsql.define_table{ + name = 'osm2pgsql_test_not_null', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326, not_null = true } + } + } + + function osm2pgsql.process_node(object) + local g = object.as_point() + tables.null:insert({ + name = object.tags.name, + geom = g + }) + print("GEOM=", g) + if g then + tables.not_null:insert({ + name = object.tags.name, + geom = g + }) + end + end + + function osm2pgsql.process_way(object) + local g = object.as_linestring() + tables.null:insert({ + name = object.tags.name, + geom = g + }) + print("GEOM=", g) + if not g:is_null() then + tables.not_null:insert({ + name = object.tags.name, + geom = g + }) + end + end + """ + + And the grid + | 10 | | 11 | + + And the OSM data + """ + n12 x3.4 y5.6 Tname=valid + n13 x42 y42 + n14 x42 y42 + w20 Tname=valid Nn10,n11 + w21 Tname=invalid Nn10 + w22 Tname=invalid Nn13,n13 + w23 Tname=invalid Nn13,n14 + """ + + When running osm2pgsql flex + + Then table osm2pgsql_test_null contains exactly + | osm_type | osm_id | ST_AsText(geom) | + | N | 12 | 3.4 5.6 | + | W | 20 | 10, 11 | + | W | 21 | NULL | + | W | 22 | NULL | + | W | 23 | NULL | + + And table osm2pgsql_test_not_null contains exactly + | osm_type | osm_id | ST_AsText(geom) | + | N | 12 | 3.4 5.6 | + | W | 20 | 10, 11 | + From 1927422ce8a0256c0594f8a62e7413f2518bbecc Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Mon, 1 Aug 2022 08:48:05 +0200 Subject: [PATCH 2/2] Try with something actually movable --- src/output-flex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/output-flex.cpp b/src/output-flex.cpp index d4cd528b3..445385c71 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -1083,7 +1083,7 @@ int output_flex_t::geom_split_multi() lua_createtable(lua_state(), (int)geoms.size(), 0); int n = 0; - for (auto const &g : geoms) { + for (auto&& g : geoms) { lua_pushinteger(lua_state(), ++n); create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { *geom = std::move(g);