From 3635d1aa23b27cd40d17a9b96a3156f520f996ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beat=20K=C3=BCng?= Date: Wed, 29 Oct 2025 13:31:10 +0100 Subject: [PATCH 1/3] messages: resolve the field definition for basic types For MessageInfo, Parameter, ParameterDefault This allows for streaming parsers to access the data immediately. --- ulog_cpp/messages.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ulog_cpp/messages.cpp b/ulog_cpp/messages.cpp index 91aedef..0a3ea8d 100644 --- a/ulog_cpp/messages.cpp +++ b/ulog_cpp/messages.cpp @@ -68,6 +68,10 @@ MessageInfo::MessageInfo(const uint8_t* msg, bool is_multi) : _is_multi(is_multi _field = Field(info->key_value_str, info->key_len); initValues(info->key_value_str + info->key_len, info->msg_size - info->key_len - 1); } + if (_field.type().type != Field::BasicType::NESTED) { + // For basic types we can resolve the definition immediately + _field.resolveDefinition(0); + } } void MessageInfo::initValues(const char* values, int len) { @@ -468,6 +472,10 @@ ParameterDefault::ParameterDefault(const uint8_t* msg) _field = Field(param_default->key_value_str, param_default->key_len); initValues(param_default->key_value_str + param_default->key_len, param_default->msg_size - param_default->key_len - 2); + if (_field.type().type != Field::BasicType::NESTED) { + // For basic types we can resolve the definition immediately + _field.resolveDefinition(0); + } } void ParameterDefault::initValues(const char* values, int len) { From 1889772eaec5b68aa39104ffd19a444d32b36ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beat=20K=C3=BCng?= Date: Wed, 29 Oct 2025 13:32:25 +0100 Subject: [PATCH 2/3] examples: add streamed parsing example Example for parsing and processing an ULog file in real-time, without keeping the whole file in memory. --- examples/CMakeLists.txt | 5 + examples/ulog_data.cpp | 2 + examples/ulog_info.cpp | 2 + examples/ulog_streamed_parsing.cpp | 186 +++++++++++++++++++++++++++++ examples/ulog_writer.cpp | 2 + 5 files changed, 197 insertions(+) create mode 100644 examples/ulog_streamed_parsing.cpp diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 484a21c..1b6d0ca 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -14,3 +14,8 @@ add_executable(ulog_writer ulog_writer.cpp) target_link_libraries(ulog_writer PUBLIC ulog_cpp::ulog_cpp ) + +add_executable(ulog_streamed_parsing ulog_streamed_parsing.cpp) +target_link_libraries(ulog_streamed_parsing PUBLIC + ulog_cpp::ulog_cpp +) diff --git a/examples/ulog_data.cpp b/examples/ulog_data.cpp index 3564d29..9e402ff 100644 --- a/examples/ulog_data.cpp +++ b/examples/ulog_data.cpp @@ -9,6 +9,8 @@ #include #include +// Example of how to use the typed data API for accessing topic data + int main(int argc, char** argv) { if (argc < 2) { diff --git a/examples/ulog_info.cpp b/examples/ulog_info.cpp index f338a9b..fc871fc 100644 --- a/examples/ulog_info.cpp +++ b/examples/ulog_info.cpp @@ -13,6 +13,8 @@ #include #include +// Example of how to access the different data messages (info, logging, parameters, subscriptions) + int main(int argc, char** argv) { if (argc < 2) { diff --git a/examples/ulog_streamed_parsing.cpp b/examples/ulog_streamed_parsing.cpp new file mode 100644 index 0000000..7e931df --- /dev/null +++ b/examples/ulog_streamed_parsing.cpp @@ -0,0 +1,186 @@ +/**************************************************************************** + * Copyright (c) 2025 PX4 Development Team. + * SPDX-License-Identifier: BSD-3-Clause + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include + +// Example for parsing and processing an ULog file in real-time, without keeping the whole file +// in memory + +class TopicSubscription { + public: + virtual ~TopicSubscription() = default; + + virtual void handleData(const ulog_cpp::TypedDataView& data) = 0; +}; + +// A topic we are interested in +class VehicleStatus : public TopicSubscription { + public: + explicit VehicleStatus(const std::shared_ptr& subscription) + { + _timestamp_field = subscription->field("timestamp"); + _nav_state_field = subscription->field("nav_state"); + // Optional field (e.g. when a message changes) + if (subscription->fieldMap().find("armed_state") != subscription->fieldMap().end()) { + _armed_state_field = subscription->field("armed_state"); + } + } + + void handleData(const ulog_cpp::TypedDataView& data) override + { + const auto timestamp = data[*_timestamp_field].as(); + const auto nav_state = data[*_nav_state_field].as(); + uint8_t armed_state = 0; + if (_armed_state_field) { + armed_state = data[*_nav_state_field].as(); + } + printf("vehicle_status: t: %" PRId64 ": nav_state: %" PRIu32 ", armed_state: %" PRId8 "\n", + timestamp, nav_state, armed_state); + } + + private: + std::shared_ptr _timestamp_field; + std::shared_ptr _nav_state_field; + std::shared_ptr _armed_state_field; +}; + +class ULogDataHandler : public ulog_cpp::DataContainer { + public: + ULogDataHandler() : ulog_cpp::DataContainer(ulog_cpp::DataContainer::StorageConfig::Header) {} + + void error(const std::string& msg, bool is_recoverable) override + { + printf("Parsing error: %s\n", msg.c_str()); + } + + void headerComplete() override { ulog_cpp::DataContainer::headerComplete(); } + + void messageInfo(const ulog_cpp::MessageInfo& message_info) override + { + DataContainer::messageInfo(message_info); + if (message_info.isMulti()) { + // Multi messages might be continued, but we only know with the next message, so we keep it + // stored and append if needed. We assume that continued multi messages are not interleaved + // with other messages. + if (message_info.isContinued()) { + if (_current_multi_message.field().name() == message_info.field().name()) { + // Append to previous + _current_multi_message.valueRaw().insert(_current_multi_message.valueRaw().end(), + message_info.valueRaw().begin(), + message_info.valueRaw().end()); + } + } else { + finishCurrentMultiMessage(); + _current_multi_message = message_info; + } + } else { + finishCurrentMultiMessage(); + messageInfoComplete(message_info); + } + } + void parameter(const ulog_cpp::Parameter& parameter) override + { + finishCurrentMultiMessage(); + DataContainer::parameter(parameter); + } + void addLoggedMessage(const ulog_cpp::AddLoggedMessage& add_logged_message) override + { + finishCurrentMultiMessage(); + DataContainer::addLoggedMessage(add_logged_message); + if (_subscriptions_by_message_id.find(add_logged_message.msgId()) != + _subscriptions_by_message_id.end()) { + throw ulog_cpp::ParsingException("Duplicate AddLoggedMessage message ID"); + } + + auto format_iter = messageFormats().find(add_logged_message.messageName()); + if (format_iter == messageFormats().end()) { + throw ulog_cpp::ParsingException("AddLoggedMessage message format not found"); + } + + auto ulog_subscription = std::make_shared( + add_logged_message, std::vector{}, format_iter->second); + + if (add_logged_message.messageName() == "vehicle_status" && add_logged_message.multiId() == 0) { + auto subscription = std::make_shared(ulog_subscription); + _subscriptions_by_message_id.insert( + {add_logged_message.msgId(), SubscriptionData{ulog_subscription, subscription}}); + } + } + void logging(const ulog_cpp::Logging& logging) override + { + finishCurrentMultiMessage(); + DataContainer::logging(logging); + } + void data(const ulog_cpp::Data& data) override + { + finishCurrentMultiMessage(); + const auto iter = _subscriptions_by_message_id.find(data.msgId()); + if (iter != _subscriptions_by_message_id.end()) { + const ulog_cpp::TypedDataView data_view(data, *iter->second.ulog_subscription->format()); + iter->second.subscription->handleData(data_view); + } + } + + private: + struct SubscriptionData { + std::shared_ptr ulog_subscription; + std::shared_ptr subscription; + }; + + void finishCurrentMultiMessage() + { + if (!_current_multi_message.field().name().empty()) { + messageInfoComplete(_current_multi_message); + _current_multi_message.field() = {}; + } + } + void messageInfoComplete(const ulog_cpp::MessageInfo& message_info) + { + if (message_info.field().definitionResolved()) { + printf("Info message: %s\n", message_info.field().name().c_str()); + } + } + + std::map _subscriptions_by_message_id; + ulog_cpp::MessageInfo _current_multi_message{"", + ""}; ///< Keep this stored for continued messages +}; + +int main(int argc, char** argv) +{ + if (argc < 2) { + printf("Usage: %s \n", argv[0]); + return -1; + } + FILE* file = fopen(argv[1], "rb"); + if (!file) { + printf("opening file failed\n"); + return -1; + } + uint8_t buffer[4096]; + int bytes_read; + const auto data_container = std::make_shared(); + ulog_cpp::Reader reader{data_container}; + while ((bytes_read = fread(buffer, 1, sizeof(buffer), file)) > 0) { + try { + reader.readChunk(buffer, bytes_read); + } catch (const ulog_cpp::ExceptionBase& exception) { + printf("Failed to parse ulog file: %s\n", exception.what()); + return -1; + } + if (data_container->hadFatalError()) { + printf("Failed to parse ulog file\n"); + return -1; + } + } + fclose(file); + + return 0; +} diff --git a/examples/ulog_writer.cpp b/examples/ulog_writer.cpp index 57a3f89..2b629d8 100644 --- a/examples/ulog_writer.cpp +++ b/examples/ulog_writer.cpp @@ -8,6 +8,8 @@ #include #include +// Example of how to create an ULog file with timeseries, printf messages and parameters + using namespace std::chrono_literals; namespace { From ad6c8fc14ef68b55f75f6b72974540dc884d5f02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beat=20K=C3=BCng?= Date: Wed, 29 Oct 2025 16:38:34 +0100 Subject: [PATCH 3/3] README: document using cmake FetchContent --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 584b344..ce925c8 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,25 @@ Streamed C++ [ULog](https://docs.px4.io/main/en/dev_log/ulog_file_format.html) r Check the [examples](examples) subdirectory. ## Include in a project +### Using cmake FetchContent +To use the library with `FetchContent` in cmake, add the following to the `CMakeLists.txt`: +```cmake +include(FetchContent) +FetchContent_Declare( + ulog_cpp + GIT_REPOSITORY https://github.com/PX4/ulog_cpp.git + GIT_TAG v1.0.0 + SYSTEM + EXCLUDE_FROM_ALL) + +FetchContent_MakeAvailable(ulog_cpp) + +target_link_libraries(YOUR_PROJECT PUBLIC + ulog_cpp::ulog_cpp +) +``` + +### As a submodule To add the library as a submodule with cmake, use the following steps: ```shell git submodule add https://github.com/PX4/ulog_cpp.git ulog_cpp