diff --git a/flex-config/new-insert-syntax.lua b/flex-config/new-insert-syntax.lua new file mode 100644 index 000000000..3eef0ebaa --- /dev/null +++ b/flex-config/new-insert-syntax.lua @@ -0,0 +1,215 @@ +-- This config example file is released into the Public Domain. + +-- This is Lua config showing the handling of custom geometries. +-- +-- This is "work in progress", changes are possible. +-- +-- Note that the insert() function is used instead of the old add_row()! + +-- The following magic can be used to wrap the internal "insert" function +local orig_insert = osm2pgsql.Table.insert +local function my_insert(table, data) + local json = require 'json' + inserted, message, column, object = orig_insert(table, data) + if not inserted then + for key, value in pairs(data) do + if type(value) == 'userdata' then + data[key] = tostring(value) + end + end + print("insert() failed: " .. message .. " column='" .. column .. "' object='" .. json.encode(object) .. "' data='" .. json.encode(data) .. "'") + end +end +osm2pgsql.Table.insert = my_insert + + +local tables = {} + +-- This table will get all nodes and areas with an "addr:street" tag, for +-- areas we'll calculate the centroid. +tables.addr = osm2pgsql.define_table({ + name = 'addr', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', projection = 4326 }, + } +}) + +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', projection = 4326 }, + { column = 'poly', type = 'polygon', projection = 4326 }, + { column = 'geom3857', type = 'linestring', projection = 3857 }, + { column = 'geomautoproj', type = 'linestring', projection = 3857 }, +}) + +tables.major_roads = osm2pgsql.define_way_table('major_roads', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring' }, + { column = 'sgeom', type = 'linestring' } +}) + +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'geometry', not_null = true }, + { column = 'area4326', type = 'real' }, + { column = 'area3857', type = 'real' } +}) + +tables.routes = osm2pgsql.define_relation_table('routes', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'multilinestring', projection = 4326 }, +}) + +tables.route_parts = osm2pgsql.define_relation_table('route_parts', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', projection = 4326 }, +}) + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Helper function that looks at the tags and decides if this is possibly +-- an area. +function has_area_tags(tags) + if tags.area == 'yes' then + return true + end + if tags.area == 'no' then + return false + end + + return tags.aeroway + or tags.amenity + or tags.building + or tags.harbour + or tags.historic + or tags.landuse + or tags.leisure + or tags.man_made + or tags.military + or tags.natural + or tags.office + or tags.place + or tags.power + or tags.public_transport + or tags.shop + or tags.sport + or tags.tourism + or tags.water + or tags.waterway + or tags.wetland + or tags['abandoned:aeroway'] + or tags['abandoned:amenity'] + or tags['abandoned:building'] + or tags['abandoned:landuse'] + or tags['abandoned:power'] + or tags['area:highway'] +end + +function osm2pgsql.process_node(object) + if clean_tags(object.tags) then + return + end + + if object.tags['addr:street'] then + tables.addr:insert({ + tags = object.tags, + geom = object:as_point() + }) + end +end + +function osm2pgsql.process_way(object) + if clean_tags(object.tags) then + return + end + + if object.is_closed and object.tags['addr:street'] then + tables.addr:insert({ + tags = object.tags, + geom = object:as_polygon():centroid() + }) + end + + if object.tags.highway == 'motorway' or + object.tags.highway == 'trunk' or + object.tags.highway == 'primary' then + tables.major_roads:insert({ + tags = object.tags, + geom = object:as_linestring(), + sgeom = object:as_linestring():simplify(100), + }) + end + + -- A closed way that also has the right tags for an area is a polygon. + if object.is_closed and has_area_tags(object.tags) then + local g = object:as_polygon() + local a = g:area() + if a < 0.0000001 then + tables.polygons:insert({ + tags = object.tags, + geom = g, + area4326 = a, + area3857 = g:transform(3857):area() + }) + end + else + tables.ways:insert({ + tags = object.tags, + geom = object:as_linestring(), + poly = object:as_polygon(), + geom3857 = object:as_linestring():transform(3857), -- project geometry into target srid 3857 + geomautoproj = object:as_linestring() -- automatically projected into projection of target column + }) + end +end + +function osm2pgsql.process_relation(object) + if clean_tags(object.tags) then + return + end + + local relation_type = object:grab_tag('type') + + if relation_type == 'multipolygon' then + local g = object:as_multipolygon() + local a = g:area() + if a < 0.0000001 then + tables.polygons:insert({ + tags = object.tags, + geom = g, + area4326 = a, + area3857 = g:transform(3857):area() + }) + end + return + end + + if relation_type == 'route' then + local route_geom = object:as_multilinestring() + if #route_geom > 0 then -- check that this is not a null geometry +-- print(object.id, tostring(route_geom), route_geom:srid(), #route_geom) + tables.routes:insert({ + tags = object.tags, + geom = route_geom + }) + for n, line in pairs(route_geom:split_multi()) do + tables.route_parts:insert({ + tags = object.tags, + geom = line + }) + end + end + end +end + diff --git a/src/flex-table.cpp b/src/flex-table.cpp index 9b02b59ff..2893882c3 100644 --- a/src/flex-table.cpp +++ b/src/flex-table.cpp @@ -77,6 +77,9 @@ flex_table_column_t &flex_table_t::add_column(std::string const &name, auto &column = m_columns.back(); if (column.is_geometry_column()) { + if (m_geom_column != std::numeric_limits::max()) { + m_has_multiple_geom_columns = true; + } m_geom_column = m_columns.size() - 1; column.set_not_null(); } diff --git a/src/flex-table.hpp b/src/flex-table.hpp index 357e8b386..14b36cc09 100644 --- a/src/flex-table.hpp +++ b/src/flex-table.hpp @@ -191,6 +191,11 @@ class flex_table_t std::string full_name() const; std::string full_tmp_name() const; + bool has_multiple_geom_columns() const noexcept + { + return m_has_multiple_geom_columns; + } + private: /// The name of the table std::string m_name; @@ -225,6 +230,9 @@ class flex_table_t /// Cluster the table by geometry. bool m_cluster_by_geom = true; + /// Does this table have more than one geometry column? + bool m_has_multiple_geom_columns = false; + }; // class flex_table_t class table_connection_t diff --git a/src/middle-ram.cpp b/src/middle-ram.cpp index 486fe7a5c..87f59e095 100644 --- a/src/middle-ram.cpp +++ b/src/middle-ram.cpp @@ -264,6 +264,13 @@ middle_ram_t::rel_members_get(osmium::Relation const &rel, buffer->commit(); ++count; } + } else { + { + osmium::builder::NodeBuilder builder{*buffer}; + builder.set_id(member.ref()); + } + buffer->commit(); + ++count; } break; case osmium::item_type::way: diff --git a/src/output-flex.cpp b/src/output-flex.cpp index 54bdcdafc..445385c71 100644 --- a/src/output-flex.cpp +++ b/src/output-flex.cpp @@ -10,6 +10,7 @@ #include "db-copy.hpp" #include "expire-tiles.hpp" #include "format.hpp" +#include "geom-from-osm.hpp" #include "geom-functions.hpp" #include "geom-transform.hpp" #include "logging.hpp" @@ -70,18 +71,37 @@ static std::mutex lua_mutex; } \ } +TRAMPOLINE(app_geometry_gc, geometry_gc) TRAMPOLINE(app_define_table, define_table) TRAMPOLINE(app_get_bbox, get_bbox) +TRAMPOLINE(app_as_linestring, as_linestring) +TRAMPOLINE(app_as_point, as_point) +TRAMPOLINE(app_as_polygon, as_polygon) +TRAMPOLINE(app_as_multilinestring, as_multilinestring) +TRAMPOLINE(app_as_multipolygon, as_multipolygon) +TRAMPOLINE(app_as_geometrycollection, as_geometrycollection) +TRAMPOLINE(geom_len, __len) +TRAMPOLINE(geom_tostring, __tostring) +TRAMPOLINE(geom_geometrytype, geometrytype) +TRAMPOLINE(geom_is_null, is_null) +TRAMPOLINE(geom_srid, srid) +TRAMPOLINE(geom_area, area) +TRAMPOLINE(geom_centroid, centroid) +TRAMPOLINE(geom_transform, transform) +TRAMPOLINE(geom_split_multi, split_multi) +TRAMPOLINE(geom_simplify, simplify) TRAMPOLINE(table_name, name) TRAMPOLINE(table_schema, schema) TRAMPOLINE(table_cluster, cluster) TRAMPOLINE(table_add_row, add_row) +TRAMPOLINE(table_insert, insert) TRAMPOLINE(table_columns, columns) TRAMPOLINE(table_tostring, __tostring) static char const *const osm2pgsql_table_name = "osm2pgsql.Table"; static char const *const osm2pgsql_object_metatable = "osm2pgsql.object_metatable"; +static char const *const osm2pgsql_geometry_name = "osm2pgsql.Geometry"; prepared_lua_function_t::prepared_lua_function_t(lua_State *lua_state, calling_context context, @@ -235,13 +255,28 @@ static int sgn(double val) noexcept return 0; } +class not_null_exception : public std::runtime_error +{ +public: + not_null_exception(std::string const &message, + flex_table_column_t const *column) + : std::runtime_error(message), m_column(column) + {} + + flex_table_column_t const &column() const noexcept { return *m_column; } + +private: + flex_table_column_t const *m_column; +}; // class not_null_exception + static void write_null(db_copy_mgr_t *copy_mgr, flex_table_column_t const &column) { if (column.not_null()) { - throw std::runtime_error{ + throw not_null_exception{ "Can not add NULL to column '{}' declared NOT NULL."_format( - column.name())}; + column.name()), + &column}; } copy_mgr->add_null_column(); } @@ -468,6 +503,22 @@ static void write_json(json_writer_type *writer, lua_State *lua_state, } } +geom::geometry_t *unpack_geometry(lua_State *lua_state) { + void *user_data = lua_touserdata(lua_state, -1); + + if (user_data == nullptr || !lua_getmetatable(lua_state, -1)) { + throw std::runtime_error{"First parameter must be a geometry."}; + } + + luaL_getmetatable(lua_state, osm2pgsql_geometry_name); + if (!lua_rawequal(lua_state, -1, -2)) { + throw std::runtime_error{"First parameter must be a geometry."}; + } + lua_pop(lua_state, 2); + + return static_cast(user_data); +} + void output_flex_t::write_column( db_copy_mgr_t *copy_mgr, flex_table_column_t const &column) @@ -484,10 +535,9 @@ void output_flex_t::write_column( int const ltype = lua_type(lua_state(), -1); // Certain Lua types can never be added to the database - if (ltype == LUA_TFUNCTION || ltype == LUA_TUSERDATA || - ltype == LUA_TTHREAD) { + if (ltype == LUA_TFUNCTION || ltype == LUA_TTHREAD) { throw std::runtime_error{ - "Can not add Lua objects of type function, userdata, or thread."}; + "Can not add Lua objects of type function or thread."}; } // A Lua nil value is always translated to a database NULL @@ -640,9 +690,44 @@ void output_flex_t::write_column( "Invalid type '{}' for direction column."_format( lua_typename(lua_state(), ltype))}; } + } else if (column.is_geometry_column()) { + // If this is a geometry column, the Lua function 'insert()' was + // called, because for 'add_row()' geometry columns are handled + // earlier and 'write_column()' is not called. + if (ltype == LUA_TUSERDATA) { + auto const *const geom = unpack_geometry(lua_state()); + if (geom && !geom->is_null()) { + if (geom->srid() == column.srid()) { + // OSM id not available here, so use dummy 0, it is used + // for debug messages only anyway. + m_expire.from_geometry(*geom, 0); + copy_mgr->add_hex_geom(geom_to_ewkb(*geom)); + } else { + auto const proj = + reprojection::create_projection(column.srid()); + auto const tgeom = geom::transform(*geom, *proj); + // OSM id not available here, so use dummy 0, it is used + // for debug messages only anyway. + m_expire.from_geometry(tgeom, 0); + copy_mgr->add_hex_geom(geom_to_ewkb(tgeom)); + } + } else { + write_null(copy_mgr, column); + } + } else { + throw std::runtime_error{ + "Need geometry data for geometry column '{}'."_format( + column.name())}; + } + } else if (column.type() == table_column_type::area) { + // If this is an area column, the Lua function 'insert()' was + // called, because for 'add_row()' area columns are handled + // earlier and 'write_column()' is not called. + throw std::runtime_error{"Column type 'area' not allowed with " + "'insert()'. Maybe use 'real'?"}; } else { - throw std::runtime_error{ - "Column type {} not implemented."_format(static_cast(column.type()))}; + throw std::runtime_error{"Column type {} not implemented."_format( + static_cast(column.type()))}; } lua_pop(lua_state(), 1); @@ -786,6 +871,253 @@ int output_flex_t::app_get_bbox() return 0; } +template +void create_lua_geometry_object(lua_State *lua_state, CREATE &&create) +{ + void *ptr = lua_newuserdata(lua_state, sizeof(geom::geometry_t)); + new (ptr) geom::geometry_t{}; + std::forward(create)(static_cast(ptr)); + + // Set the metatable of this object + luaL_getmetatable(lua_state, osm2pgsql_geometry_name); + lua_setmetatable(lua_state, -2); +} + +int output_flex_t::app_as_point() +{ + check_context_and_state("as_point", "process_node() function", + m_calling_context != calling_context::process_node); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_point(geom, *m_context_node); + }); + + return 1; +} + +int output_flex_t::app_as_linestring() +{ + check_context_and_state("as_linestring", "process_way() function", + m_calling_context != calling_context::process_way); + + m_way_cache.add_nodes(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_linestring(geom, m_way_cache.get()); + }); + + return 1; +} + +int output_flex_t::app_as_polygon() +{ + check_context_and_state("as_polygon", "process_way() function", + m_calling_context != calling_context::process_way); + + m_way_cache.add_nodes(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_polygon(geom, m_way_cache.get()); + }); + + return 1; +} + +int output_flex_t::app_as_multilinestring() +{ + check_context_and_state("as_multilinestring", "process_relation() function", + m_calling_context != + calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_multilinestring(geom, m_relation_cache.members_buffer()); + }); + + return 1; +} + +int output_flex_t::app_as_multipolygon() +{ + check_context_and_state("as_multipolygon", "process_relation() function", + m_calling_context != + calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_multipolygon(geom, m_relation_cache.get(), + m_relation_cache.members_buffer()); + }); + + return 1; +} + +int output_flex_t::app_as_geometrycollection() +{ + check_context_and_state( + "as_geometrycollection", "process_relation() function", + m_calling_context != calling_context::process_relation); + + m_relation_cache.add_members(middle()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::create_collection(geom, m_relation_cache.members_buffer()); + }); + + return 1; +} + +// XXX Implementation for Lua __tostring function on geometries. Currently +// just returns the type as string. This could be improved, for instance by +// showing a WKT representation of the geometry. +int output_flex_t::geom_tostring() +{ + return geom_geometrytype(); +} + +int output_flex_t::geom_geometrytype() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + auto const type = geometry_type(*input_geometry); + lua_pushlstring(lua_state(), type.data(), type.size()); + return 1; +} + +int output_flex_t::geom_len() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushinteger(lua_state(), + static_cast(num_geometries(*input_geometry))); + return 1; +} + +int output_flex_t::geom_is_null() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushboolean(lua_state(), input_geometry->is_null()); + return 1; +} + +int output_flex_t::geom_srid() +{ + auto const *const input_geometry = unpack_geometry(lua_state()); + lua_pushinteger(lua_state(), + static_cast(input_geometry->srid())); + return 1; +} + +int output_flex_t::geom_area() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for area()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + double const area = geom::area(*input_geometry); + lua_pushnumber(lua_state(), area); + + return 1; +} + +int output_flex_t::geom_centroid() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for centroid()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = geom::centroid(*input_geometry); + }); + + return 1; +} + +int output_flex_t::geom_transform() +{ + if (lua_gettop(lua_state()) != 2) { + throw std::runtime_error{ + "One parameter needed for transform(), the target srid."}; + } + +#if LUA_VERSION_NUM >= 503 + if (!lua_isinteger(lua_state(), -1)) { +#else + if (!lua_isnumber(lua_state(), -1)) { +#endif + throw std::runtime_error{ + "transform() srid parameter has to be an integer."}; + } + + int const srid = lua_tointeger(lua_state(), -1); + lua_pop(lua_state(), 1); // srid parameter + + auto const *const input_geometry = unpack_geometry(lua_state()); + + if (input_geometry->srid() != 4326) { + throw std::runtime_error{ + "Can not transform already transformed geometry."}; + } + auto const proj = reprojection::create_projection(srid); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + geom::transform(geom, *input_geometry, *proj); + }); + + return 1; +} + +// XXX this is inefficient, because we are copying the whole geometry and +// creating a large Lua array. Better implement this with an iterator? +int output_flex_t::geom_split_multi() +{ + if (lua_gettop(lua_state()) > 1) { + throw std::runtime_error{"No parameter(s) needed for split_multi()."}; + } + + auto const *const input_geometry = unpack_geometry(lua_state()); + auto const geoms = geom::split_multi(*input_geometry, true); + + lua_createtable(lua_state(), (int)geoms.size(), 0); + int n = 0; + for (auto&& g : geoms) { + lua_pushinteger(lua_state(), ++n); + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = std::move(g); + }); + lua_rawset(lua_state(), -3); + } + + return 1; +} + +int output_flex_t::geom_simplify() +{ + if (lua_gettop(lua_state()) != 2) { + throw std::runtime_error{ + "simplify() function needs one parameter: tolerance."}; + } + + if (!lua_isnumber(lua_state(), -1)) { + throw std::runtime_error{ + "simplify() tolerance parameters has to be a number."}; + } + + double const tolerance = lua_tonumber(lua_state(), -1); + lua_pop(lua_state(), 1); // tolerance parameter + + auto const *const input_geometry = unpack_geometry(lua_state()); + + create_lua_geometry_object(lua_state(), [&](geom::geometry_t *geom) { + *geom = geom::simplify(*input_geometry, tolerance); + }); + + return 1; +} + static void check_name(std::string const &name, char const *in) { auto const pos = name.find_first_of("\"',.;$%&/()<>{}=?^*#"); @@ -981,6 +1313,18 @@ void output_flex_t::setup_flex_table_columns(flex_table_t *table) } } +int output_flex_t::app_geometry_gc() +{ + void *geom = lua_touserdata(lua_state(), 1); + if (geom) { + static_cast(geom)->~geometry_t(); + } + + lua_pop(lua_state(), 1); // pop parameter from stack + + return 0; +} + int output_flex_t::app_define_table() { if (m_calling_context != calling_context::main) { @@ -1099,7 +1443,7 @@ void output_flex_t::relation_cache_t::init(osmium::Relation const &relation) bool output_flex_t::relation_cache_t::add_members(middle_query_t const &middle) { - if (m_members_buffer.committed() == 0) { + if (members_buffer().committed() == 0) { auto const num_members = middle.rel_members_get( *m_relation, &m_members_buffer, osmium::osm_entity_bits::node | osmium::osm_entity_bits::way); @@ -1108,6 +1452,12 @@ bool output_flex_t::relation_cache_t::add_members(middle_query_t const &middle) return false; } + for (auto &node : m_members_buffer.select()) { + if (!node.location().valid()) { + node.set_location(middle.get_node_location(node.id())); + } + } + for (auto &nodes : m_members_buffer.select()) { middle.nodes_get_list(&nodes); } @@ -1175,6 +1525,99 @@ int output_flex_t::table_add_row() return 0; } +osmium::OSMObject const & +output_flex_t::check_and_get_context_object(flex_table_t const &table) +{ + if (m_calling_context == calling_context::process_node) { + if (!table.matches_type(osmium::item_type::node)) { + throw std::runtime_error{ + "Trying to add node to table '{}'."_format(table.name())}; + } + return *m_context_node; + } + + if (m_calling_context == calling_context::process_way) { + if (!table.matches_type(osmium::item_type::way)) { + throw std::runtime_error{ + "Trying to add way to table '{}'."_format(table.name())}; + } + return m_way_cache.get(); + } + + assert(m_calling_context == calling_context::process_relation); + + if (!table.matches_type(osmium::item_type::relation)) { + throw std::runtime_error{ + "Trying to add relation to table '{}'."_format(table.name())}; + } + return m_relation_cache.get(); +} + +int output_flex_t::table_insert() +{ + if (m_disable_add_row) { + return 0; + } + + if (m_calling_context != calling_context::process_node && + m_calling_context != calling_context::process_way && + m_calling_context != calling_context::process_relation) { + throw std::runtime_error{ + "The function insert() can only be called from the " + "process_node/way/relation() functions."}; + } + + auto const num_params = lua_gettop(lua_state()); + if (num_params != 2) { + throw std::runtime_error{ + "Need two parameters: The osm2pgsql.table and the row data."}; + } + + // The first parameter is the table object. + auto &table_connection = + m_table_connections.at(table_idx_from_param(lua_state())); + + // The second parameter must be a Lua table with the contents for the + // fields. + luaL_checktype(lua_state(), 2, LUA_TTABLE); + lua_remove(lua_state(), 1); + + auto const &table = table_connection.table(); + auto const &object = check_and_get_context_object(table); + osmid_t const id = table.map_id(object.type(), object.id()); + + table_connection.new_line(); + auto *copy_mgr = table_connection.copy_mgr(); + + try { + for (auto const &column : table_connection.table()) { + if (column.create_only()) { + continue; + } + if (column.type() == table_column_type::id_type) { + copy_mgr->add_column(type_to_char(object.type())); + } else if (column.type() == table_column_type::id_num) { + copy_mgr->add_column(id); + } else { + write_column(copy_mgr, column); + } + } + } catch (not_null_exception const &e) { + copy_mgr->rollback_line(); + lua_pushboolean(lua_state(), false); + lua_pushstring(lua_state(), "null value in not null column."); + lua_pushstring(lua_state(), e.column().name().c_str()); + push_osm_object_to_lua_stack(lua_state(), object, + get_options()->extra_attributes); + return 4; + } + + copy_mgr->finish_line(); + + lua_pushboolean(lua_state(), true); + return 1; +} + int output_flex_t::table_columns() { auto const &table = get_table_from_param(); @@ -1231,11 +1674,21 @@ get_transform(lua_State *lua_state, flex_table_column_t const &column) lua_getfield(lua_state, -1, column.name().c_str()); int const ltype = lua_type(lua_state, -1); - if (ltype != LUA_TTABLE) { + + // Field not set, return null transform + if (ltype == LUA_TNIL) { lua_pop(lua_state, 1); // geom field return transform; } + // Field set to anything but a Lua table is not allowed + if (ltype != LUA_TTABLE) { + lua_pop(lua_state, 1); // geom field + throw std::runtime_error{ + "Invalid geometry transformation for column '{}'."_format( + column.name())}; + } + lua_getfield(lua_state, -1, "create"); char const *create_type = lua_tostring(lua_state, -1); if (create_type == nullptr) { @@ -1326,6 +1779,13 @@ void output_flex_t::add_row(table_connection_t *table_connection, assert(table_connection); auto const &table = table_connection->table(); + if (table.has_multiple_geom_columns()) { + throw std::runtime_error{ + "Table '{}' has more than one geometry column." + " This is not allowed with 'add_row()'." + " Maybe use 'insert()' instead?"_format(table.name())}; + } + osmid_t const id = table.map_id(object.type(), object.id()); if (!table.has_geom_column()) { @@ -1743,12 +2203,40 @@ static void init_table_class(lua_State *lua_state) lua_setfield(lua_state, -2, "__index"); luaX_add_table_func(lua_state, "__tostring", lua_trampoline_table_tostring); luaX_add_table_func(lua_state, "add_row", lua_trampoline_table_add_row); + luaX_add_table_func(lua_state, "insert", lua_trampoline_table_insert); luaX_add_table_func(lua_state, "name", lua_trampoline_table_name); luaX_add_table_func(lua_state, "schema", lua_trampoline_table_schema); luaX_add_table_func(lua_state, "cluster", lua_trampoline_table_cluster); luaX_add_table_func(lua_state, "columns", lua_trampoline_table_columns); } +/** + * Define the osm2pgsql.Geometry class/metatable. + */ +static void init_geometry_class(lua_State *lua_state) +{ + if (luaL_newmetatable(lua_state, osm2pgsql_geometry_name) != 1) { + throw std::runtime_error{"Internal error: Lua newmetatable failed."}; + } + + luaX_add_table_func(lua_state, "__gc", lua_trampoline_app_geometry_gc); + luaX_add_table_func(lua_state, "__len", lua_trampoline_geom_len); + luaX_add_table_func(lua_state, "__tostring", lua_trampoline_geom_tostring); + lua_pushvalue(lua_state, -1); + lua_setfield(lua_state, -2, "__index"); + luaX_add_table_func(lua_state, "is_null", lua_trampoline_geom_is_null); + luaX_add_table_func(lua_state, "srid", lua_trampoline_geom_srid); + luaX_add_table_func(lua_state, "area", lua_trampoline_geom_area); + luaX_add_table_func(lua_state, "geometrytype", + lua_trampoline_geom_geometrytype); + luaX_add_table_func(lua_state, "centroid", lua_trampoline_geom_centroid); + luaX_add_table_func(lua_state, "transform", lua_trampoline_geom_transform); + luaX_add_table_func(lua_state, "split_multi", + lua_trampoline_geom_split_multi); + luaX_add_table_func(lua_state, "simplify", lua_trampoline_geom_simplify); + lua_pop(lua_state, 1); // __index +} + void output_flex_t::init_lua(std::string const &filename) { m_lua_state.reset(luaL_newstate(), @@ -1788,10 +2276,22 @@ void output_flex_t::init_lua(std::string const &filename) lua_tostring(lua_state(), -1))}; } - // Store the "get_bbox" in the "object_metatable". + // Store the methods on OSM objects in its metatable. lua_getglobal(lua_state(), "object_metatable"); lua_getfield(lua_state(), -1, "__index"); luaX_add_table_func(lua_state(), "get_bbox", lua_trampoline_app_get_bbox); + luaX_add_table_func(lua_state(), "as_linestring", + lua_trampoline_app_as_linestring); + luaX_add_table_func(lua_state(), "as_point", + lua_trampoline_app_as_point); + luaX_add_table_func(lua_state(), "as_polygon", + lua_trampoline_app_as_polygon); + luaX_add_table_func(lua_state(), "as_multilinestring", + lua_trampoline_app_as_multilinestring); + luaX_add_table_func(lua_state(), "as_multipolygon", + lua_trampoline_app_as_multipolygon); + luaX_add_table_func(lua_state(), "as_geometrycollection", + lua_trampoline_app_as_geometrycollection); lua_settop(lua_state(), 0); // Store the global object "object_metatable" defined in the init.lua @@ -1803,6 +2303,12 @@ void output_flex_t::init_lua(std::string const &filename) lua_pushnil(lua_state()); lua_setglobal(lua_state(), "object_metatable"); + assert(lua_gettop(lua_state()) == 0); + + init_geometry_class(lua_state()); + + assert(lua_gettop(lua_state()) == 0); + // Load user config file luaX_set_context(lua_state(), this); if (luaL_dofile(lua_state(), filename.c_str())) { diff --git a/src/output-flex.hpp b/src/output-flex.hpp index 4a463dd44..9849e1ec3 100644 --- a/src/output-flex.hpp +++ b/src/output-flex.hpp @@ -153,12 +153,32 @@ class output_flex_t : public output_t void merge_expire_trees(output_t *other) override; + int app_as_point(); + int app_as_linestring(); + int app_as_polygon(); + int app_as_multilinestring(); + int app_as_multipolygon(); + int app_as_geometrycollection(); + int app_define_table(); - int app_mark_way(); + int app_geometry_gc(); int app_get_bbox(); + int app_mark_way(); + + int geom_area(); + int geom_centroid(); + int geom_geometrytype(); + int geom_is_null(); + int geom_len(); + int geom_simplify(); + int geom_split_multi(); + int geom_srid(); + int geom_tostring(); + int geom_transform(); int table_tostring(); int table_add_row(); + int table_insert(); int table_name(); int table_schema(); int table_cluster(); @@ -192,10 +212,14 @@ class output_flex_t : public output_t void write_column(db_copy_mgr_t *copy_mgr, flex_table_column_t const &column); + void write_row(table_connection_t *table_connection, osmium::item_type id_type, osmid_t id, geom::geometry_t const &geom, int srid); + osmium::OSMObject const & + check_and_get_context_object(flex_table_t const &table); + geom::geometry_t run_transform(reprojection const &proj, geom_transform_t const *transform, osmium::Node const &node); diff --git a/tests/bdd/flex/collection.feature b/tests/bdd/flex/collection.feature new file mode 100644 index 000000000..43609eddf --- /dev/null +++ b/tests/bdd/flex/collection.feature @@ -0,0 +1,69 @@ +Feature: Create geometry collections from relations + + Background: + Given the lua style + """ + local dtable = osm2pgsql.define_table{ + name = 'osm2pgsql_test_collection', + ids = { type = 'relation', id_column = 'osm_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 } + } + } + + function osm2pgsql.process_relation(object) + dtable:insert({ + name = object.tags.name, + geom = object.as_geometrycollection() + }) + end + """ + + Scenario: Create geometry collection from different relations + Given the 1.0 grid + | 13 | 12 | 17 | | 16 | + | 10 | 11 | | 14 | 15 | + And the OSM data + """ + w20 Nn10,n11,n12,n13,n10 + w21 Nn14,n15,n16 + r30 Tname=single Mw20@ + r31 Tname=multi Mw20@,w21@ + r32 Tname=mixed Mn17@,w21@ + r33 Tname=node Mn17@ + """ + When running osm2pgsql flex + + Then table osm2pgsql_test_collection contains exactly + | osm_id | name | ST_GeometryType(geom) | ST_NumGeometries(geom) | ST_GeometryType(ST_GeometryN(geom, 1)) | + | 30 | single | ST_GeometryCollection | 1 | ST_LineString | + | 31 | multi | ST_GeometryCollection | 2 | ST_LineString | + | 32 | mixed | ST_GeometryCollection | 2 | ST_Point | + | 33 | node | ST_GeometryCollection | 1 | ST_Point | + + And table osm2pgsql_test_collection contains exactly + | osm_id | ST_AsText(ST_GeometryN(geom, 1)) | ST_AsText(ST_GeometryN(geom, 2)) | + | 30 | 10, 11, 12, 13, 10 | NULL | + | 31 | 10, 11, 12, 13, 10 | 14, 15, 16 | + | 32 | 17 | 14, 15, 16 | + | 33 | 17 | NULL | + + Scenario: NULL entry generated for broken geometries + Given the grid + | 10 | + And the OSM data + """ + w20 Nn10 + r30 Tname=foo Mn11@ + r31 Tname=bar Mw20@ + r32 Tname=baz Mw21@ + """ + When running osm2pgsql flex + + Then table osm2pgsql_test_collection contains exactly + | osm_id | name | ST_GeometryType(geom) | + | 30 | foo | NULL | + | 31 | bar | NULL | + | 32 | baz | NULL | + diff --git a/tests/bdd/flex/geometry-processing.feature b/tests/bdd/flex/geometry-processing.feature new file mode 100644 index 000000000..14c51a022 --- /dev/null +++ b/tests/bdd/flex/geometry-processing.feature @@ -0,0 +1,97 @@ +Feature: Tests for Lua geometry processing functions + + Scenario: + Given the OSM data + """ + n1 Tamenity=restaurant,name=point x1.1 y1.2 + """ + And the lua style + """ + local points = osm2pgsql.define_node_table('osm2pgsql_test_points', { + { column = 'name', type = 'text' }, + { column = 'geom4326', type = 'point', projection = 4326 }, + { column = 'geom3857', type = 'point', projection = 3857 }, + { column = 'geomauto', type = 'point', projection = 3857 }, + }) + + function osm2pgsql.process_node(object) + points:insert({ + name = object.tags.name, + geom4326 = object:as_point(), + geom3857 = object:as_point():transform(3857), + geomauto = object:as_point() + }) + end + + """ + When running osm2pgsql flex + Then table osm2pgsql_test_points contains + | node_id | name | ST_AsText(geom4326) | geom3857 = geomauto | + | 1 | point | 1.1 1.2 | True | + + Scenario: + Given the 0.1 grid with origin 9.0 50.3 + | | | 7 | | | 8 | + | | | | 11 | 12 | | + | 3 | 4 | | 9 | 10 | | + | 1 | 2 | 5 | | | 6 | + And the OSM data + """ + w1 Tnatural=water,name=poly Nn1,n2,n4,n3,n1 + w2 Nn5,n6,n8,n7,n5 + w3 Nn9,n10,n12,n11,n9 + r1 Tnatural=water,name=multi Mw2@,w3@ + """ + And the lua style + """ + local tables = {} + + tables.ways = osm2pgsql.define_way_table('osm2pgsql_test_ways', { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'linestring', projection = 4326 }, + { column = 'geomsimple', type = 'linestring', projection = 4326 }, + }) + + tables.polygons = osm2pgsql.define_area_table('osm2pgsql_test_polygons', { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 }, + { column = 'center', type = 'point', projection = 4326 }, + }) + + function is_empty(some_table) + return next(some_table) == nil + end + + function osm2pgsql.process_way(object) + if is_empty(object.tags) then + return + end + + tables.ways:insert({ + name = object.tags.name, + geom = object:as_linestring(), + geomsimple = object:as_linestring():simplify(0.1) + }) + tables.polygons:insert({ + name = object.tags.name, + geom = object:as_polygon(), + center = object:as_polygon():centroid() + }) + end + + function osm2pgsql.process_relation(object) + tables.polygons:insert({ + name = object.tags.name, + geom = object:as_multipolygon() + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_ways contains + | way_id | name | ST_AsText(geom) | ST_AsText(geomsimple) | + | 1 | poly | 1, 2, 4, 3, 1 | 1, 4, 1 | + And table osm2pgsql_test_polygons contains + | area_id | name | ST_AsText(geom) | ST_AsText(center) | + | 1 | poly | (1, 2, 4, 3, 1) | 9.05 50.05 | + | -1 | multi | (5, 6, 8, 7, 5),(9, 11, 12, 10, 9) | NULL | + diff --git a/tests/bdd/flex/geometry-split-multi.feature b/tests/bdd/flex/geometry-split-multi.feature new file mode 100644 index 000000000..cd58f867b --- /dev/null +++ b/tests/bdd/flex/geometry-split-multi.feature @@ -0,0 +1,42 @@ +Feature: Tests for geometry split_multi function + + Scenario: + Given the grid + | 1 | 2 | | + | 4 | | 3 | + | | 5 | 6 | + And the OSM data + """ + w20 Thighway=motorway Nn1,n2,n3 + w21 Thighway=motorway Nn4,n5,n6 + r30 Ttype=route,route=road Mw20@,w21@ + r31 Ttype=route,route=road Mw20@ + r32 Ttype=something Mw20@ + r33 Ttype=route,route=road Mn1@ + """ + And the lua style + """ + local routes = osm2pgsql.define_relation_table('osm2pgsql_test_routes', { + { column = 'geom', type = 'linestring', projection = 4326 }, + }) + + function osm2pgsql.process_relation(object) + if object.tags.type == 'route' then + local g = object:as_multilinestring() + if not g:is_null() then + for n, line in pairs(g:split_multi()) do + routes:insert({ + geom = line + }) + end + end + end + end + + """ + When running osm2pgsql flex + Then table osm2pgsql_test_routes contains exactly + | relation_id | ST_AsText(geom) | + | 30 | 1, 2, 3 | + | 30 | 4, 5, 6 | + | 31 | 1, 2, 3 | diff --git a/tests/bdd/flex/insert-area.feature b/tests/bdd/flex/insert-area.feature new file mode 100644 index 000000000..ba8f5a040 --- /dev/null +++ b/tests/bdd/flex/insert-area.feature @@ -0,0 +1,67 @@ +Feature: Tests for area() function + + Scenario Outline: + Given the 0.1 grid with origin 9.0 50.3 + | | | 7 | | | 8 | + | | | | 11 | 12 | | + | 3 | 4 | | 9 | 10 | | + | 1 | 2 | 5 | | | 6 | + And the OSM data + """ + w1 Tnatural=water,name=poly Nn1,n2,n4,n3,n1 + w2 Nn5,n6,n8,n7,n5 + w3 Nn9,n10,n12,n11,n9 + r1 Tnatural=water,name=multi Mw2@,w3@ + """ + And the lua style + """ + local polygons = osm2pgsql.define_table{ + name = 'osm2pgsql_test_polygon', + ids = { type = 'area', id_column = 'osm_id' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = }, + { column = 'area', type = 'real' }, + } + } + + function osm2pgsql.process_way(object) + local polygon = object:as_polygon() + polygons:insert({ + name = object.tags.name, + geom = polygon, + area = polygon:transform():area() + }) + end + + function osm2pgsql.process_relation(object) + local polygon = object:as_multipolygon() + polygons:insert({ + name = object.tags.name, + geom = polygon, + area = polygon:transform():area() + }) + end + """ + When running osm2pgsql flex + Then table osm2pgsql_test_polygon contains + | name | ST_Area(geom) | area | ST_Area(ST_Transform(geom, 4326)) | + | poly | | | 0.01 | + | multi | | | 0.08 | + + Examples: + | geom proj | area proj | st_area poly | area poly | st_area multi | area multi | + | 4326 | 4326 | 0.01 | 0.01 | 0.08 | 0.08 | + | 4326 | 3857 | 0.01 | 192987010.0 | 0.08 | 1547130000.0 | + | 3857 | 4326 | 192987010.0 | 0.01 | 1547130000.0 | 0.08 | + | 3857 | 3857 | 192987010.0 | 192987010.0 | 1547130000.0 | 1547130000.0 | + + @config.have_proj + Examples: Generic projection + | geom proj | area proj | st_area poly | area poly | st_area multi | area multi | + | 4326 | 25832 | 0.01 | 79600737.537 | 0.08 | 635499542.954 | + | 3857 | 25832 | 192987010.0 | 79600737.537 | 1547130000.0 | 635499542.954 | + | 25832 | 4326 | 79600737.537 | 0.01 | 635499542.954 | 0.08 | + | 25832 | 3857 | 79600737.537 | 192987010.0 | 635499542.954 | 1547130000.0 | + | 25832 | 25832 | 79600737.537 | 79600737.537 | 635499542.954 | 635499542.954 | + diff --git a/tests/bdd/flex/invalid-lua.feature b/tests/bdd/flex/invalid-lua.feature new file mode 100644 index 000000000..36309b172 --- /dev/null +++ b/tests/bdd/flex/invalid-lua.feature @@ -0,0 +1,12 @@ +Feature: Tests for basic Lua functions + + Scenario: + Given the OSM data + """ + n1 Tamenity=restaurant x10 y10 + """ + And the lua style + """ + this-is-not-valid-lua + """ + Then running osm2pgsql flex fails diff --git a/tests/bdd/flex/null-geom.feature b/tests/bdd/flex/null-geom.feature new file mode 100644 index 000000000..922c0cbd8 --- /dev/null +++ b/tests/bdd/flex/null-geom.feature @@ -0,0 +1,85 @@ +Feature: Null geometry handling + + Scenario: Invalid geometries show up as NULL + Given the lua style + """ + local tables = {} + + tables.null = osm2pgsql.define_table{ + name = 'osm2pgsql_test_null', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326 } + } + } + + tables.not_null = osm2pgsql.define_table{ + name = 'osm2pgsql_test_not_null', + ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, + columns = { + { column = 'name', type = 'text' }, + { column = 'geom', type = 'geometry', projection = 4326, not_null = true } + } + } + + function osm2pgsql.process_node(object) + local g = object.as_point() + tables.null:insert({ + name = object.tags.name, + geom = g + }) + print("GEOM=", g) + if g then + tables.not_null:insert({ + name = object.tags.name, + geom = g + }) + end + end + + function osm2pgsql.process_way(object) + local g = object.as_linestring() + tables.null:insert({ + name = object.tags.name, + geom = g + }) + print("GEOM=", g) + if not g:is_null() then + tables.not_null:insert({ + name = object.tags.name, + geom = g + }) + end + end + """ + + And the grid + | 10 | | 11 | + + And the OSM data + """ + n12 x3.4 y5.6 Tname=valid + n13 x42 y42 + n14 x42 y42 + w20 Tname=valid Nn10,n11 + w21 Tname=invalid Nn10 + w22 Tname=invalid Nn13,n13 + w23 Tname=invalid Nn13,n14 + """ + + When running osm2pgsql flex + + Then table osm2pgsql_test_null contains exactly + | osm_type | osm_id | ST_AsText(geom) | + | N | 12 | 3.4 5.6 | + | W | 20 | 10, 11 | + | W | 21 | NULL | + | W | 22 | NULL | + | W | 23 | NULL | + + And table osm2pgsql_test_not_null contains exactly + | osm_type | osm_id | ST_AsText(geom) | + | N | 12 | 3.4 5.6 | + | W | 20 | 10, 11 | +