From 2c52327ace099eac3d903166b1f6408221d83ded Mon Sep 17 00:00:00 2001 From: systemed Date: Wed, 16 Apr 2025 16:00:03 +0100 Subject: [PATCH] Simple key/value store --- docs/CONFIGURATION.md | 12 ++++++++++-- include/osm_lua_processing.h | 7 ++++++- resources/process-openmaptiles.lua | 2 +- src/osm_lua_processing.cpp | 18 +++++++++++++++--- src/tilemaker.cpp | 5 +++-- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 49ab1ad4..8533183b 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -124,7 +124,7 @@ Your Lua file can supply these functions for tilemaker to call: 2. (optional) `way_keys`, a list of those OSM tags which indicate that a way should be processed 3. `node_function()`, a function to process an OSM node and add it to layers 4. `way_function()`, a function to process an OSM way and add it to layers -5. (optional) `init_function(name)`, a function to initialize Lua logic +5. (optional) `init_function(name, is_first)`, a function to initialize Lua logic 6. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics) 7. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation 8. (optional) `relation_function`, a function to process an OSM relation and add it to layers @@ -184,7 +184,7 @@ If your Lua file causes an error due to mistaken syntax, you can test it at the `way_keys` is similar, but for ways. For ways, you may also wish to express the filter in terms of the tag value, or as an inversion. For example, to exclude buildings: `way_keys = {"~building"}`. To build a map only of major roads: `way_keys = {"highway=motorway", "highway=trunk", "highway=primary", "highway=secondary"}` -`init_function(name)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. +`init_function(name, is_first)` and `exit_function` are called at the start and end of processing (once per thread). You can use this to output statistics or even to read a small amount of external data. `is_first` will be true only the first time `init_function` is called. Other functions are described below and in RELATIONS.md. @@ -243,3 +243,11 @@ To enable these functions, set `index` to true in your shapefile layer definitio `CoveredBy` and `FindCovering` work similarly but check if the object is covered by a shapefile layer object. `AreaIntersecting` returns the area of the current way's intersection with the shapefile layer. You can use this to find whether a water body is already represented in a shapefile ocean layer. + +### Lua key/value store + +tilemaker has a simple key/value store accessible from Lua which you can use to bring in external data. The same store is used across all processing threads. + +Read your data from file, using [Lua's I/O functions](https://www.lua.org/pil/21.1.html), in `init_function` (checking that `is_first` is set for the first run only). Set a key/value pair with `SetData(key,value)` - for example `SetData("name","Bill")`. Both key and value should be strings. + +You can then retrieve the value within `way_function` or similar with `GetData(key)`. If no value was found, the empty string is returned. \ No newline at end of file diff --git a/include/osm_lua_processing.h b/include/osm_lua_processing.h index 86cd680b..72100635 100644 --- a/include/osm_lua_processing.h +++ b/include/osm_lua_processing.h @@ -6,6 +6,7 @@ #include #include #include +#include #include "geom.h" #include "osm_store.h" #include "shared_data.h" @@ -56,7 +57,8 @@ class OsmLuaProcessing { const class ShpMemTiles &shpMemTiles, class OsmMemTiles &osmMemTiles, AttributeStore &attributeStore, - bool materializeGeometries + bool materializeGeometries, + bool isFirst ); ~OsmLuaProcessing(); @@ -239,6 +241,9 @@ class OsmLuaProcessing { const TagMap* currentTags; bool isPostScanRelation; // processing a relation in postScanRelation + static std::unordered_map dataStore; + static std::mutex dataStoreMutex; + private: /// Internal: clear current cached state inline void reset() { diff --git a/resources/process-openmaptiles.lua b/resources/process-openmaptiles.lua index 2b1c6bd3..30aefcae 100644 --- a/resources/process-openmaptiles.lua +++ b/resources/process-openmaptiles.lua @@ -17,7 +17,7 @@ additional_languages = { } -------- -- Enter/exit Tilemaker -function init_function() +function init_function(name,is_first) end function exit_function() end diff --git a/src/osm_lua_processing.cpp b/src/osm_lua_processing.cpp index 2200794f..cba5948c 100644 --- a/src/osm_lua_processing.cpp +++ b/src/osm_lua_processing.cpp @@ -18,6 +18,8 @@ thread_local kaguya::State *g_luaState = nullptr; thread_local OsmLuaProcessing* osmLuaProcessing = nullptr; std::mutex vectorLayerMetadataMutex; +std::unordered_map OsmLuaProcessing::dataStore; +std::mutex OsmLuaProcessing::dataStoreMutex; void handleOsmLuaProcessingUserSignal(int signum) { osmLuaProcessing->handleUserSignal(signum); @@ -195,6 +197,14 @@ std::string rawFindInRelation(const std::string& key) { return osmLuaProcessing- void rawAccept() { return osmLuaProcessing->Accept(); } double rawAreaIntersecting(const std::string& layerName) { return osmLuaProcessing->AreaIntersecting(layerName); } +void rawSetData(const std::string &key, const std::string &value) { + std::lock_guard lock(osmLuaProcessing->dataStoreMutex); + osmLuaProcessing->dataStore[key] = value; +} +std::string rawGetData(const std::string &key) { + auto r = osmLuaProcessing->dataStore.find(key); + return r==osmLuaProcessing->dataStore.end() ? "" : r->second; +} bool supportsRemappingShapefiles = false; @@ -217,7 +227,8 @@ OsmLuaProcessing::OsmLuaProcessing( const class ShpMemTiles &shpMemTiles, class OsmMemTiles &osmMemTiles, AttributeStore &attributeStore, - bool materializeGeometries): + bool materializeGeometries, + bool isFirst) : osmStore(osmStore), shpMemTiles(shpMemTiles), osmMemTiles(osmMemTiles), @@ -273,6 +284,8 @@ OsmLuaProcessing::OsmLuaProcessing( luaState["NextRelation"] = &rawNextRelation; luaState["RestartRelations"] = &rawRestartRelations; luaState["FindInRelation"] = &rawFindInRelation; + luaState["SetData"] = &rawSetData; + luaState["GetData"] = &rawGetData; supportsRemappingShapefiles = !!luaState["attribute_function"]; supportsReadingRelations = !!luaState["relation_scan_function"]; supportsPostScanRelations = !!luaState["relation_postscan_function"]; @@ -283,7 +296,7 @@ OsmLuaProcessing::OsmLuaProcessing( // ---- Call init_function of Lua logic if (!!luaState["init_function"]) { - luaState["init_function"](this->config.projectName); + luaState["init_function"](this->config.projectName, isFirst); } } @@ -1182,4 +1195,3 @@ std::vector OsmLuaProcessing::finalizeOutputs() { } return list; } - diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp index 5220933a..7dcd5ad4 100644 --- a/src/tilemaker.cpp +++ b/src/tilemaker.cpp @@ -253,7 +253,7 @@ int main(const int argc, const char* argv[]) { shpMemTiles.open(); OsmLuaProcessing osmLuaProcessing(osmStore, config, layers, options.luaFile, - shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries); + shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries, true); // ---- Load external sources (shp/geojson) @@ -311,7 +311,7 @@ int main(const int argc, const char* argv[]) { [&]() { thread_local std::pair> osmLuaProcessing; if (osmLuaProcessing.first != inputFile) { - osmLuaProcessing = std::make_pair(inputFile, std::make_shared(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries)); + osmLuaProcessing = std::make_pair(inputFile, std::make_shared(osmStore, config, layers, options.luaFile, shpMemTiles, osmMemTiles, attributeStore, options.osm.materializeGeometries, false)); } return osmLuaProcessing.second; }, @@ -323,6 +323,7 @@ int main(const int argc, const char* argv[]) { attributeStore.finalize(); osmMemTiles.reportSize(); attributeStore.reportSize(); + osmLuaProcessing.dataStore.clear(); // no longer needed // ---- Initialise SharedData