From 15bc0168ceda2ccc51f04c3f7419a7caf9a3396f Mon Sep 17 00:00:00 2001 From: Philip Windecker <95633467+philipwindecker@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:11:57 +0100 Subject: [PATCH 01/33] 832 consolidate hosted osi documentation sources (#837) * Issue #832: Removed obsolete steps from workflow "protobuf" * Issue #832: Replaced DCO reference in pull request template * Issue #832: Removed outdated reference to Asciidoc output. --------- Signed-off-by: Philip WINDECKER (AVENYR GmbH) --- .github/pull_request_template.md | 4 ++-- .github/workflows/protobuf.yml | 17 +---------------- osi_version.proto.in | 5 +---- 3 files changed, 4 insertions(+), 22 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b18379317..f1cd74669 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -13,8 +13,8 @@ How has it been tested? #### Take this checklist as orientation for yourself, if this PR is ready for the Change Control Board: - [ ] My suggestion follows the [style and contributors guidelines](https://opensimulationinterface.github.io/osi-antora-generator/asamosi/latest/specification/contributing/start_contributing.html). - [ ] I have taken care about the [message documentation](https://opensimulationinterface.github.io/osi-antora-generator/asamosi/latest/specification/contributing/commenting_messages.html) and the [fields and enums documentation](https://opensimulationinterface.github.io/osi-antora-generator/asamosi/latest/specification/contributing/commenting_fields_enums.html). -- [ ] I have done the [DCO signoff](https://opensimulationinterface.github.io/osi-documentation/open-simulation-interface/doc/howtocontribute.html#developer-certification-of-origin-dco). -- [ ] My changes generate no errors when passing CI tests. +- [ ] I have done the [DCO signoff](https://opensimulationinterface.github.io/osi-antora-generator/asamosi/latest/specification/contributing/dco.html). +- [ ] My changes generate no errors when passing CI tests. - [ ] I have successfully implemented and tested my fix/feature locally. - [ ] Appropriate reviewer(s) are assigned. diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 19eba40c5..050c9a212 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -97,7 +97,7 @@ jobs: - name: Get git Version id: get_version - run: echo "VERSION=$(git describe --tags --always)" >> $GITHUB_OUTPUT + run: echo "VERSION=$(git describe --tags --always)" >> $GITHUB_OUTPUT - name: Prepare Documentation Build run: | @@ -122,14 +122,6 @@ jobs: - name: Run Python Tests run: python -m unittest discover tests - - name: Archive Documentation - if: ${{ github.event_name == 'pull_request' }} - uses: actions/upload-artifact@v4 - with: - name: linux64-doc - path: doc/html - if-no-files-found: error - - name: Upload Python Distribution if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && ( github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') ) ) }} uses: actions/upload-artifact@v4 @@ -137,13 +129,6 @@ jobs: name: python-dist path: dist/ - - name: Deploy to gh-pages if push to master branch - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./doc/html - build-proto3-linux64: name: Build Proto3 Linux 64 diff --git a/osi_version.proto.in b/osi_version.proto.in index 1170d1e58..1ff524f29 100644 --- a/osi_version.proto.in +++ b/osi_version.proto.in @@ -25,10 +25,7 @@ package osi3; // properly set the version components prior to sending. // // Increments will happen as part of changes to the whole interface. -// The meaning of different InterfaceVersions is defined in [1]. -// -// \par Reference: -// [1] Open Simulation Interface. (2020, January 25). README. Retrieved January 25, 2020, from https://opensimulationinterface.github.io/osi-documentation/open-simulation-interface/README.html +// The meaning of different InterfaceVersions is defined in the versioning chapter of the official OSI documentation. // message InterfaceVersion { From e78dff7443bb71e563431175fbd09bafba27da2c Mon Sep 17 00:00:00 2001 From: Philip Windecker <95633467+philipwindecker@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:46:18 +0100 Subject: [PATCH 02/33] Issue #836: Added missing pages and linked them in navigation. (#838) * Issue #836: Added links to interface specification pages. ---------- Signed-off-by: Philip WINDECKER (AVENYR GmbH) Co-authored-by: Carlo van Driesten --- doc/architecture/architecture_overview.adoc | 18 ++++++++++++++---- doc/architecture/ground_truth.adoc | 4 +++- doc/architecture/host_vehicle_data.adoc | 12 ++++++++++++ doc/architecture/motion_request.adoc | 4 +++- doc/architecture/sensor_data.adoc | 2 ++ doc/architecture/sensor_view.adoc | 4 +++- .../sensor_view_configuration.adoc | 4 +++- doc/architecture/streaming_update.adoc | 4 +++- doc/architecture/traffic_command.adoc | 4 +++- doc/architecture/traffic_command_update.adoc | 9 +++++++++ doc/architecture/traffic_update.adoc | 4 +++- doc/open-simulation-interface_user_guide.adoc | 4 ++++ 12 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 doc/architecture/host_vehicle_data.adoc create mode 100644 doc/architecture/traffic_command_update.adoc diff --git a/doc/architecture/architecture_overview.adoc b/doc/architecture/architecture_overview.adoc index 31f28c154..35959b97b 100644 --- a/doc/architecture/architecture_overview.adoc +++ b/doc/architecture/architecture_overview.adoc @@ -4,11 +4,21 @@ include::{root-path}_config.adoc[] endif::[] = Overview of OSI architecture -OSI contains an object-based environment description that uses the message format of the https://github.com/protocolbuffers/protobuf/wiki[Protocol Buffer] library. -Google developed and maintains the Protocol Buffer library. +OSI contains an object-based environment description that uses the message format of the https://github.com/protocolbuffers/protobuf/wiki[Protocol Buffer^] library developed by Google. OSI defines top-level messages that are used to exchange data between separate models. -Top-level messages define the `GroundTruth` interface, the `SensorData` interface, and – since OSI version 3.0.0 – the interfaces `SensorView` and `SensorViewConfiguration`. -Further top-level messages that were added in later versions of OSI are `HostVehicleData`, `TrafficCommand`, `TrafficCommandUpdate`, `TrafficUpdate`, `MotionRequest`, and `StreamingUpdate`. + +Top-level messages define the following interfaces: + +- xref:architecture/ground_truth.adoc[`GroundTruth`] +- xref:architecture/sensor_data.adoc[`SensorData`] +- xref:architecture/sensor_view.adoc[`SensorView`] +- xref:architecture/sensor_view_configuration.adoc[`SensorViewConfiguration`] +- xref:architecture/host_vehicle_data.adoc[`HostVehicleData`] +- xref:architecture/traffic_command.adoc[`TrafficCommand`] +- xref:architecture/traffic_command_update.adoc[`TrafficCommandUpdate`] +- xref:architecture/traffic_update.adoc[`TrafficUpdate`] +- xref:architecture/motion_request.adoc[`MotionRequest`] +- xref:architecture/streaming_update.adoc[`StreamingUpdate`]. The following figure shows the interfaces and models involved in modeling a sensor. diff --git a/doc/architecture/ground_truth.adoc b/doc/architecture/ground_truth.adoc index 512387778..7520b88fd 100644 --- a/doc/architecture/ground_truth.adoc +++ b/doc/architecture/ground_truth.adoc @@ -6,4 +6,6 @@ endif::[] `GroundTruth` messages describe the simulated environment containing all simulated objects in the global coordinate system at consecutive time instances. They are based on data available to the simulation environment. -`GroundTruth` messages are typically contained in `Sensorview` messages. \ No newline at end of file +`GroundTruth` messages are typically contained in `Sensorview` messages. + +For more details, see xref:gen:structosi3_1_1GroundTruth.adoc[]. \ No newline at end of file diff --git a/doc/architecture/host_vehicle_data.adoc b/doc/architecture/host_vehicle_data.adoc new file mode 100644 index 000000000..d9f27ef7c --- /dev/null +++ b/doc/architecture/host_vehicle_data.adoc @@ -0,0 +1,12 @@ +ifndef::include-only-once[] +:root-path: ../ +include::{root-path}_config.adoc[] +endif::[] += Host vehicle data + +The `HostVehicleData` message targets the perception of the vehicle of its own internal states. + +It captures the knowledge the vehicle has internally, which can differ from the actual or global truth for various reasons. +This message can also be understood as an interface container for the signals of a rest bus simulation. + +For more details, see xref:gen:structosi3_1_1HostVehicleData.adoc[]. \ No newline at end of file diff --git a/doc/architecture/motion_request.adoc b/doc/architecture/motion_request.adoc index e24a43608..49085d9ff 100644 --- a/doc/architecture/motion_request.adoc +++ b/doc/architecture/motion_request.adoc @@ -5,4 +5,6 @@ endif::[] = Motion Request `MotionRequest` messages are traffic participant internal messages. -They function as a interface between a motion/behavior planning model and a dynamics model including, for example, controllers and vehicle kinematics. \ No newline at end of file +They function as a interface between a motion/behavior planning model and a dynamics model including, for example, controllers and vehicle kinematics. + +For more details, see xref:gen:structosi3_1_1MotionRequest.adoc[]. \ No newline at end of file diff --git a/doc/architecture/sensor_data.adoc b/doc/architecture/sensor_data.adoc index b44f0eeaa..8d29abd64 100644 --- a/doc/architecture/sensor_data.adoc +++ b/doc/architecture/sensor_data.adoc @@ -13,3 +13,5 @@ Sensor data can be used as input for an automated driving function, a sensor mod `SensorData` messages include `FeatureData` messages which contain detected features in the reference frame of a sensor. `FeatureData` messages are generated from `GroundTruth` messages. They serve, for example, as an input to sensor models simulating object detection or feature fusion models. + +For more details, see xref:gen:structosi3_1_1SensorData.adoc[]. \ No newline at end of file diff --git a/doc/architecture/sensor_view.adoc b/doc/architecture/sensor_view.adoc index adeb6254e..d2cf7550a 100644 --- a/doc/architecture/sensor_view.adoc +++ b/doc/architecture/sensor_view.adoc @@ -10,4 +10,6 @@ All information regarding the environment is given with respect to the virtual s * Physical technology-specific data, given with respect to the physical sensor coordinate system specified in the corresponding physical sensor's mounting position. One example of technology-specific data is: xref:gen:structosi3_1_1CameraSensorView.adoc#ac58456a34babf78792ea2608eb963f36[`image_data` of `osi3::CameraSensorView`] -* Ground truth given in the global coordinate system. \ No newline at end of file +* Ground truth given in the global coordinate system. + +For more details, see xref:gen:structosi3_1_1SensorView.adoc[]. \ No newline at end of file diff --git a/doc/architecture/sensor_view_configuration.adoc b/doc/architecture/sensor_view_configuration.adoc index 09d0d5fb7..5ec06a97f 100644 --- a/doc/architecture/sensor_view_configuration.adoc +++ b/doc/architecture/sensor_view_configuration.adoc @@ -35,4 +35,6 @@ In response to this difference, the sensor model can either accept this differen The packaging layer defines the specifics of this auto-negotiation mechanism. -After the initialization phase, the environment simulation provides the actual sensor-view configuration as part of each `SensorView` message. \ No newline at end of file +After the initialization phase, the environment simulation provides the actual sensor-view configuration as part of each `SensorView` message. + +For more details, see xref:gen:structosi3_1_1SensorViewConfiguration.adoc[]. \ No newline at end of file diff --git a/doc/architecture/streaming_update.adoc b/doc/architecture/streaming_update.adoc index 1bf661948..85378e47e 100644 --- a/doc/architecture/streaming_update.adoc +++ b/doc/architecture/streaming_update.adoc @@ -7,4 +7,6 @@ endif::[] The `StreamingUpdate` message provides an interface to transmit a subset of ground truth and/or vehicle internal data. This interface mainly addresses applications with low latency requirements and no need for highly consistent and complete data, e.g. visualization applications. Static and/or non-relevant objects can be omitted as required for the specific use case. -Note that the receiver of partial updates can only rely on the most up-to-date information at the corresponding timestamp. E.g. omitting objects does not indicate static behaviour but it may be sufficient for the use case to update certain objects at a later point in time. \ No newline at end of file +Note that the receiver of partial updates can only rely on the most up-to-date information at the corresponding timestamp. E.g. omitting objects does not indicate static behavior but it may be sufficient for the use case to update certain objects at a later point in time. + +For more details, see xref:gen:structosi3_1_1StreamingUpdate.adoc[]. \ No newline at end of file diff --git a/doc/architecture/traffic_command.adoc b/doc/architecture/traffic_command.adoc index f8cd90d24..ea34a63f2 100644 --- a/doc/architecture/traffic_command.adoc +++ b/doc/architecture/traffic_command.adoc @@ -4,4 +4,6 @@ include::{root-path}_config.adoc[] endif::[] = Traffic command -`TrafficCommand` messages contain control commands from the scenario engine to traffic participant models. \ No newline at end of file +`TrafficCommand` messages contain control commands from the scenario engine to traffic participant models. + +For more details, see xref:gen:structosi3_1_1TrafficCommand.adoc[]. \ No newline at end of file diff --git a/doc/architecture/traffic_command_update.adoc b/doc/architecture/traffic_command_update.adoc new file mode 100644 index 000000000..afd05cfc4 --- /dev/null +++ b/doc/architecture/traffic_command_update.adoc @@ -0,0 +1,9 @@ +ifndef::include-only-once[] +:root-path: ../ +include::{root-path}_config.adoc[] +endif::[] += Traffic command update + +`TrafficCommandUpdate` messages enable the traffic participant model to send updates to the scenario engine about the execution of its received xref:architecture/traffic_command.adoc[TrafficCommand] input. + +For more details, see xref:gen:structosi3_1_1TrafficCommandUpdate.adoc[]. \ No newline at end of file diff --git a/doc/architecture/traffic_update.adoc b/doc/architecture/traffic_update.adoc index 33461d23d..0dce99689 100644 --- a/doc/architecture/traffic_update.adoc +++ b/doc/architecture/traffic_update.adoc @@ -5,4 +5,6 @@ endif::[] = Traffic update `TrafficUpdate` messages are provided by traffic participants. -They provide updates on the position, state, and future trajectory of a traffic participant back to the simulation environment. \ No newline at end of file +They provide updates on the position, state, and future trajectory of a traffic participant back to the simulation environment. + +For more details, see xref:gen:structosi3_1_1TrafficUpdate.adoc[]. \ No newline at end of file diff --git a/doc/open-simulation-interface_user_guide.adoc b/doc/open-simulation-interface_user_guide.adoc index 69296d267..394b81448 100644 --- a/doc/open-simulation-interface_user_guide.adoc +++ b/doc/open-simulation-interface_user_guide.adoc @@ -21,8 +21,12 @@ include::./architecture/sensor_view_configuration.adoc[leveloffset=+3] include::./architecture/sensor_data.adoc[leveloffset=+3] +include::./architecture/host_vehicle_data.adoc[leveloffset=+3] + include::./architecture/traffic_command.adoc[leveloffset=+3] +include::./architecture/traffic_command_update.adoc[leveloffset=+3] + include::./architecture/motion_request.adoc[leveloffset=+3] include::./architecture/traffic_update.adoc[leveloffset=+3] From 08f544b4d0685fe7b5b8d540890ea990b43ee9e8 Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Thu, 28 Nov 2024 13:55:59 +0100 Subject: [PATCH 03/33] Update osi_object.proto * resolves #844 Signed-off-by: Carlo van Driesten --- osi_object.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/osi_object.proto b/osi_object.proto index fb0c518b3..f4a326e59 100644 --- a/osi_object.proto +++ b/osi_object.proto @@ -954,6 +954,10 @@ message MovingObject // optional GenericLightState reversing_light = 6; + // State of the tail light (rear position light). + // + optional GenericLightState tail_light = 11; + // State of the brake lights. // optional BrakeLightState brake_light_state = 7; From d969c39d30a5958610984fe78ada46bf7a9d9ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Ulus?= Date: Wed, 5 Feb 2025 22:42:48 +0300 Subject: [PATCH 04/33] Fix open_simulation_interface-config.cmake.in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This filename must be the same with [this filename](https://github.com/OpenSimulationInterface/open-simulation-interface/blob/e78dff7443bb71e563431175fbd09bafba27da2c/CMakeLists.txt#L230) Signed-off-by: Doğan Ulus --- open_simulation_interface-config.cmake.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open_simulation_interface-config.cmake.in b/open_simulation_interface-config.cmake.in index 615ec5cd0..a285246e2 100644 --- a/open_simulation_interface-config.cmake.in +++ b/open_simulation_interface-config.cmake.in @@ -6,5 +6,5 @@ find_dependency(Protobuf) if(NOT TARGET @PROJECT_NAME@ AND NOT @PROJECT_NAME@_BINARY_DIR) set_and_check(OPEN_SIMULATION_INTERFACE_INCLUDE_DIRS "@PACKAGE_OSI_INSTALL_INCLUDE_DIR@") set(OPEN_SIMULATION_INTERFACE_LIBRARIES "@PROJECT_NAME@") - include("${CMAKE_CURRENT_LIST_DIR}/open_simulation_interface_targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/open_simulation_interface-targets.cmake") endif() From 1e538e1b3d4acf73c5949172807f035ad1a55b3a Mon Sep 17 00:00:00 2001 From: ClemensLinnhoff Date: Thu, 6 Jun 2024 14:16:16 +0200 Subject: [PATCH 05/33] Add pyi_out option to protoc command in the setup.py Signed-off-by: ClemensLinnhoff --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a310a8b3..2e87680a3 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ def generate(self): for source in self.osi_files: sys.stdout.write("Protobuf-compiling " + source + "\n") source_path = os.path.join(package_name, source) - subprocess.check_call([self.find_protoc(), "--python_out=.", source_path]) + subprocess.check_call([self.find_protoc(), "--python_out=.", "--pyi_out=.", source_path]) def maybe_generate(self): if os.path.exists("osi_version.proto.in"): From 985d3260ed01c2ba9caa6363ca9a662fc4cf080b Mon Sep 17 00:00:00 2001 From: ClemensLinnhoff Date: Thu, 6 Jun 2024 14:20:14 +0200 Subject: [PATCH 06/33] Formatting Signed-off-by: ClemensLinnhoff --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2e87680a3..ea0e93eba 100644 --- a/setup.py +++ b/setup.py @@ -116,7 +116,9 @@ def generate(self): for source in self.osi_files: sys.stdout.write("Protobuf-compiling " + source + "\n") source_path = os.path.join(package_name, source) - subprocess.check_call([self.find_protoc(), "--python_out=.", "--pyi_out=.", source_path]) + subprocess.check_call( + [self.find_protoc(), "--python_out=.", "--pyi_out=.", source_path] + ) def maybe_generate(self): if os.path.exists("osi_version.proto.in"): From ab9b6cbb1eed77385c51ab80268b15156804c6cd Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Sun, 24 Nov 2024 17:31:23 +0100 Subject: [PATCH 07/33] Initial multi channel trace file format specification Based on the initial specification work with refactoring for better integration into OSI and with clearer separation of concerns. Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 185 ++++++++++++++++++++++- doc/architecture/trace_file_naming.adoc | 12 ++ 2 files changed, 195 insertions(+), 2 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 60f2d2a4b..b10d03d00 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -5,7 +5,10 @@ endif::[] [#top-osi_trace_file_formats] = OSI trace file formats -There are two formats for storing multiple serialized OSI messages in one trace file. +== Single channel trace file formats +There are two formats for storing and reading multiple serialized OSI messages of the same type in one trace file. +These formats are very bare bones and do not contain any additional information, like meta-data, schemas, nor do they allow for random access. +For more advanced use-cases, consider using the multi channel trace file format. *.osi:: Binary trace file. @@ -16,9 +19,187 @@ The length does not include the integer itself. *.txth:: Human-readable plain-text trace file. Messages are separated by newlines. +Each message is a serialized OSI message in JSON format. + +NOTE: The `.txth` format is intended for human consumption (e.g. for debugging and manual checks). +It is currently not supported for reading by the OSI API, as it is not unambiguously deserializable. NOTE: Previous releases of OSI also supported a so-called plain-text trace file format, with file extension `.txt`. This legacy format did not contain plain-text, but rather binary protobuf messages separated by a special separator. For obvious reasons the format was deprecated and fully replaced with the `.osi` binary file format. This release no longer contains any support for the legacy `.txt` file format. -These files may be used for manual checks. + +== Multi channel trace file format + +=== Overview + +The OSI multi channel trace file format is a binary file format that allows for storing multiple serialized OSI message streams of the same or different types in one trace file, along with additional meta-data, and other related data streams. +Due to the nature of the format, it allows for random access to the data streams, and is suitable for more advanced use-cases. + +The OSI multi channel trace file format is based on the MCAP file format, which is a generic multi channel trace file format. +The OSI multi channel trace file format is a specialization of the MCAP file format, with additional constraints and requirements specific to OSI. +Hence, any valid OSI multi channel trace file is also a valid MCAP file, but not the other way around. + +The following rules apply to OSI multi channel trace files: + +- The file extension to be used is `.mcap`. +- The file must be a valid MCAP file according to the https://mcap.dev/spec[MCAP format specification] version `0x30`. +- The file must be an indexed MCAP file, including chunk index records in the summary section, with all message records written into chunk records. +- Chunk records must either be uncompressed or compressed using either the `zstd` or `lz4` compression algorithms. +- The file must contain a meta-data record with the OSI trace file meta-data defined in section <>. + This meta-data record identifies the file as an OSI multi channel trace file. +- The file must contain a schema record in the summary section for each top-level message type that is used in one or more OSI channels with the OSI message schema as defined in section <>. +- The file must contain at least one OSI message stream in a channel as defined in section <>. +- The file may contain additional non-OSI message streams in other channels. + +NOTE: All OSI and non-OSI message streams stored in the same file share a common time base. +Storing of unrelated message streams in one trace file is therefore not generally useful and should be avoided. + +[#sec-osi-trace-file-multi-global-meta-data] +=== Multi channel trace file global meta-data + +The file must contain a meta-data record with the `name` field being `net.asam.osi.trace` and the following mandatory `metadata` entries: + +`version`:: +The semantic version number of the OSI release that this OSI trace file conforms to. +This is a string in the format `major.minor.patch`, for example `3.0.0`. +Note that this version number is not necessarily the same as the OSI schema version(s) used in the trace file. +Rather, it indicates the version of the OSI trace file format itself. + +`min_osi_version`:: +The minimum version of the OSI schema used in the trace file OSI channels. +This is a string in the format `major.minor.patch`, for example `3.0.0`. + +`max_osi_version`:: +The maximum version of the OSI schema used in the trace file OSI channels. +This is a string in the format `major.minor.patch`, for example `3.0.0`. + +`min_protobuf_version`:: +The version of the protobuf implementation used in the trace file. +This is a string in the format `major.minor.patch`, for example `3.17.3`. + +`max_protobuf_version`:: +The version of the protobuf implementation used in the trace file. +This is a string in the format `major.minor.patch`, for example `3.17.3`. + +`zero_time`:: +The point in time corresponding to time 0 in all timestamps in the trace file. +Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. +Values must match the following regular expression: +`-?([1-9][0-9]{3,}|0[0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T(([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?|(24:00:00(\.0+)?))(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))` ++ +NOTE: that even in pure simulation use cases there is usually a relationship to real time, as on-board components and environment simulation have necessary relationships to real time (for example, the embedded `HostVehicleData` will carry relevant real time information, for localisation and other purposes). + +`creation_time`:: +The point in time when the trace file was created. +Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. + +The `net.asam.osi.trace` meta-data record may also contain the following recommended `metadata` entries: + +`description`:: +A human-readable description of the data contained in the multi channel trace file. + +`creator`:: +A list separated by commas of entities (not tools) involved in the creation of the data contained in the file. + +`license`:: +If the file is licensed under any SPDX registered license, this entry should contain the SPDX identifier for the license. + +`data_sources`:: +A list separated by commas of data sources used in the creation of the data contained in the file. + +The file may contain arbitrary additional meta-data records, however meta-data records with names starting with `net.asam.osi` are reserved for future use by {THIS_STANDARD}. +It is strongly recommended to follow reverse domain name notation for custom meta-data record names to avoid conflicts. + +[#sec-osi-trace-file-multi-schema-record] +=== OSI message schema + +For each OSI top-level message type that is used in one or more OSI channels, the OSI multi channel trace file must contain a corresponding schema record in the summary section. +Note that if multiple versions of the OSI schema are used in the same trace file, a schema record must be included for each version, with different schema IDs. + +The schema record must contain the following fields: + +`id`:: +A file-wide unique non-zero identifier for the schema record. + +`name`:: +The fully qualified name to the message within the OSI descriptor set. For example, for the `SensorView` message type, this would be `osi3.SensorView`. + +`encoding`:: +The value `protobuf`. + +`data`:: +A binary FileDescriptorSet as produced by `protoc --include_imports --descriptor_set_out`. + +The schema record must be stored in the summary section of the trace file, and must be referenced by the OSI channels that use the schema. + +[#sec-osi-trace-file-multi-channel] +=== OSI channel + +An OSI channel is a data stream within the OSI multi channel trace file that contains serialized OSI messages of the same type. +Each OSI channel must be described by a channel record in the summary section of the trace file with the following fields: + +`id`:: +A file-wide unique identifier for the channel. + +`schema_id`:: +The ID of the schema record that describes the message type of the channel. + +`topic`:: +A unique name for the channel within the trace file. +When recording OSI traces for a model packaged according to the OSI Sensor Model Packaging (OSMP) layer, using the naming conventions defined in the OSMP specification for variables as topics is recommended. +For example, for a sensor model with two SensorView inputs and one SensorData output, the topic names would be `OSMPSensorViewIn[1]`, `OSMPSensorViewIn[2]`, and `OSMPSensorDataOut`, accordingly. +In other cases, the topic name should be chosen to reflect the purpose of the channel, and should include some indication of the message type. + +`message_encoding`:: +The value `protobuf`. + +`metadata`:: +A map of additional meta-data for the channel. +This map may contain arbitrary key-value pairs, however keys starting with `net.asam.osi` are reserved for use by {THIS_STANDARD}. +It is strongly recommended to follow reverse domain name notation for custom meta-data keys to avoid conflicts. +The following mandatory entries are defined by {THIS_STANDARD}: + +`net.asam.osi.trace.channel.osi_version`::: +The version of the OSI schema used in creating the data of this OSI channel. +This is a string in the format `major.minor.patch`, for example `3.0.0`. + +`net.asam.osi.trace.channel.protobuf_version`::: +The version of the protobuf implementation used in creating the data of this OSI channel. +This is a string in the format `major.minor.patch`, for example `3.17.3`. + +The following recommended entries are defined by {THIS_STANDARD}: + +`net.asam.osi.trace.channel.description`::: +A human-readable description of the channel. + +The channel record must be stored in the summary section of the trace file, and must be referenced by the OSI message records that are part of the channel. + +All messages in an OSI channel must be stored in chunk records in the data section of the trace file. + +Each message record in a chunk record must contain the following fields: + +`channel_id`:: +The ID of the channel that the message belongs to. + +`publish_time`:: +The point in time when the message was published. + +`sequence`:: +Optional message counter to detect message gaps. +If the relevant packaging layer or other source of messages provides a sequence number this can be used. +Otherwise this should be set to zero to indicate that no reliable sequence number is available. + +`log_time`:: +This field is in nanoseconds and uses the same epoch as the `publish_time` field. +It is used to determine the order of messages in the trace file, and provides for time-based random access to the data streams. +Unless there is a specific reason to set this field to a different value, it should be set to the same value as `publish_time`, as this reflects the time flow as reflected in the OSI message stream. +Only if recreation of the message stream with actual message transmission times is required - for example for asynchronous packaging layers, this field should be set to the time when the message was enqueued for addition to the trace file. + +`publish_time`:: +The timestamp taken from the timestamp field of the stored OSI top-level message. +The field is in nanoseconds, with the epoch being the epoch of the OSI Timestamp data type. +If top-level messages that do not contain a timestamp field are stored in the trace file, the `publish_time` field must be set to the time when the message was enqueued for addition to the trace file. + +`data`:: +The serialized OSI message data. diff --git a/doc/architecture/trace_file_naming.adoc b/doc/architecture/trace_file_naming.adoc index 1c73ed90c..6d1bbe051 100644 --- a/doc/architecture/trace_file_naming.adoc +++ b/doc/architecture/trace_file_naming.adoc @@ -44,6 +44,12 @@ Trace file contains `MotionRequest` messages. `su`:: Trace file contains `StreamingUpdate` messages. +`multi`:: +Trace file contains multiple types of messages for use with multi channel trace file format. +In this case the number-of-frames field should be the largest number of frames across all channels. +The OSI version field should be based on the `version` field in the file meta-data of the multi-channel trace file. +The protobuf version field should be based on the `min_protobuf_version` field in the file meta-data of the multi-channel trace file. + **Example** Given an OSI trace file with the following information: @@ -76,3 +82,9 @@ The recommended file name is: ---- 20210818T150542Z_sv_312_300_1523_highway.osi ---- + +For a corresponding multi channel trace file containing `SensorView` and `GroundTruth` messages, the recommended file name is: + +---- +20210818T150542Z_multi_312_300_1523_highway.mcap +---- From 558429600177791af80e705c61e83f930c7378d9 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Mon, 25 Nov 2024 11:56:55 +0100 Subject: [PATCH 08/33] Minor fixes based on review feedback Co-authored-by: Clemens Linnhoff Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index b10d03d00..65552bf22 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -88,7 +88,7 @@ Must be provided as a full ISO8601 formatted date time string, including timezon Values must match the following regular expression: `-?([1-9][0-9]{3,}|0[0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T(([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?|(24:00:00(\.0+)?))(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))` + -NOTE: that even in pure simulation use cases there is usually a relationship to real time, as on-board components and environment simulation have necessary relationships to real time (for example, the embedded `HostVehicleData` will carry relevant real time information, for localisation and other purposes). +NOTE: that even in pure simulation use cases there is usually a relationship to real time, as on-board components and environment simulation have necessary relationships to real time (for example, the embedded `HostVehicleData` will carry relevant real time information, for localization and other purposes). `creation_time`:: The point in time when the trace file was created. @@ -182,9 +182,6 @@ Each message record in a chunk record must contain the following fields: `channel_id`:: The ID of the channel that the message belongs to. -`publish_time`:: -The point in time when the message was published. - `sequence`:: Optional message counter to detect message gaps. If the relevant packaging layer or other source of messages provides a sequence number this can be used. From 77fd4c62b65b18a38b05ddeaf69f850b5a580b48 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Mon, 25 Nov 2024 13:19:21 +0100 Subject: [PATCH 09/33] Further clarifications and corrections from code review Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 65552bf22..b4fd2e2fc 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -11,15 +11,15 @@ These formats are very bare bones and do not contain any additional information, For more advanced use-cases, consider using the multi channel trace file format. *.osi:: -Binary trace file. +Single channel binary trace file. Messages are separated by a length specification before each message. The length is represented by a four-byte, little-endian, unsigned integer. The length does not include the integer itself. *.txth:: -Human-readable plain-text trace file. +Single channel human-readable plain-text trace file. Messages are separated by newlines. -Each message is a serialized OSI message in JSON format. +Each message is a serialized OSI message in protocol buffer text format. NOTE: The `.txth` format is intended for human consumption (e.g. for debugging and manual checks). It is currently not supported for reading by the OSI API, as it is not unambiguously deserializable. @@ -103,7 +103,7 @@ A human-readable description of the data contained in the multi channel trace fi A list separated by commas of entities (not tools) involved in the creation of the data contained in the file. `license`:: -If the file is licensed under any SPDX registered license, this entry should contain the SPDX identifier for the license. +If the contents of the file is licensed under any SPDX registered license, this entry should contain the SPDX identifier for the license. `data_sources`:: A list separated by commas of data sources used in the creation of the data contained in the file. @@ -136,7 +136,9 @@ The schema record must be stored in the summary section of the trace file, and m [#sec-osi-trace-file-multi-channel] === OSI channel -An OSI channel is a data stream within the OSI multi channel trace file that contains serialized OSI messages of the same type. +An OSI channel is a data stream within the OSI multi channel trace file that contains serialized OSI top-level messages of the same type. +Note that non-top-level messages must not be stored directly in OSI channels. + Each OSI channel must be described by a channel record in the summary section of the trace file with the following fields: `id`:: From aa2b48a161b7979627575f91d63aa680d3021017 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Mon, 25 Nov 2024 14:57:30 +0100 Subject: [PATCH 10/33] Clarify extension bit of naming convention Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_naming.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/architecture/trace_file_naming.adoc b/doc/architecture/trace_file_naming.adoc index 6d1bbe051..0c781aaaa 100644 --- a/doc/architecture/trace_file_naming.adoc +++ b/doc/architecture/trace_file_naming.adoc @@ -9,7 +9,7 @@ endif::[] The names of OSI trace files should have the following format: ---- -_____.osi +_____.[osi|txth|mcap] ---- **Types** From 45f2578498f97606c06c667b3e5bf33311564b58 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 26 Nov 2024 12:26:19 +0100 Subject: [PATCH 11/33] Apply suggestions from code review Co-authored-by: Thomas Sedlmayer Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 28 +++++++++++------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index b4fd2e2fc..8e26e5166 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -68,11 +68,11 @@ Rather, it indicates the version of the OSI trace file format itself. `min_osi_version`:: The minimum version of the OSI schema used in the trace file OSI channels. -This is a string in the format `major.minor.patch`, for example `3.0.0`. +This is a string in the format `major.minor.patch`, for example `3.4.0`. `max_osi_version`:: The maximum version of the OSI schema used in the trace file OSI channels. -This is a string in the format `major.minor.patch`, for example `3.0.0`. +This is a string in the format `major.minor.patch`, for example `3.7.0`. `min_protobuf_version`:: The version of the protobuf implementation used in the trace file. @@ -80,18 +80,18 @@ This is a string in the format `major.minor.patch`, for example `3.17.3`. `max_protobuf_version`:: The version of the protobuf implementation used in the trace file. -This is a string in the format `major.minor.patch`, for example `3.17.3`. +This is a string in the format `major.minor.patch`, for example `3.26.0`. `zero_time`:: -The point in time corresponding to time 0 in all timestamps in the trace file. +The point in real or simulated calendar time corresponding to time 0 in all timestamps in the trace file. Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. Values must match the following regular expression: `-?([1-9][0-9]{3,}|0[0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T(([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?|(24:00:00(\.0+)?))(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))` + -NOTE: that even in pure simulation use cases there is usually a relationship to real time, as on-board components and environment simulation have necessary relationships to real time (for example, the embedded `HostVehicleData` will carry relevant real time information, for localization and other purposes). +NOTE: Even in pure simulation use cases there is usually a relationship to a simulated calendar time, as on-board components and environment simulation have necessary relationships to a simulated calendar time (for example, the embedded `HostVehicleData` will carry relevant simulated calendar time information, for localization and other purposes). `creation_time`:: -The point in time when the trace file was created. +The point in real calendar time when the recording of the data in the trace file was started. Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. The `net.asam.osi.trace` meta-data record may also contain the following recommended `metadata` entries: @@ -99,12 +99,9 @@ The `net.asam.osi.trace` meta-data record may also contain the following recomme `description`:: A human-readable description of the data contained in the multi channel trace file. -`creator`:: +`authors`:: A list separated by commas of entities (not tools) involved in the creation of the data contained in the file. -`license`:: -If the contents of the file is licensed under any SPDX registered license, this entry should contain the SPDX identifier for the license. - `data_sources`:: A list separated by commas of data sources used in the creation of the data contained in the file. @@ -123,7 +120,8 @@ The schema record must contain the following fields: A file-wide unique non-zero identifier for the schema record. `name`:: -The fully qualified name to the message within the OSI descriptor set. For example, for the `SensorView` message type, this would be `osi3.SensorView`. +The fully qualified name to the message within the OSI descriptor set. +For example, for the `SensorView` message type, this would be `osi3.SensorView`. `encoding`:: The value `protobuf`. @@ -149,8 +147,8 @@ The ID of the schema record that describes the message type of the channel. `topic`:: A unique name for the channel within the trace file. -When recording OSI traces for a model packaged according to the OSI Sensor Model Packaging (OSMP) layer, using the naming conventions defined in the OSMP specification for variables as topics is recommended. -For example, for a sensor model with two SensorView inputs and one SensorData output, the topic names would be `OSMPSensorViewIn[1]`, `OSMPSensorViewIn[2]`, and `OSMPSensorDataOut`, accordingly. +When recording OSI traces for a model packaged according to the OSI Sensor Model Packaging (OSMP) layer, using the naming conventions defined in the OSMP specification for variables as topics is recommended, potentially with a suitable added prefix for disambiguation. +For example, for a sensor model named RadarSensorFL with two SensorView inputs and one SensorData output, the topic names would be `RadarSensorFL.OSMPSensorViewIn[1]`, `RadarSensorFL.OSMPSensorViewIn[2]`, and `RadarSensorFL.OSMPSensorDataOut`, accordingly. In other cases, the topic name should be chosen to reflect the purpose of the channel, and should include some indication of the message type. `message_encoding`:: @@ -192,8 +190,8 @@ Otherwise this should be set to zero to indicate that no reliable sequence numbe `log_time`:: This field is in nanoseconds and uses the same epoch as the `publish_time` field. It is used to determine the order of messages in the trace file, and provides for time-based random access to the data streams. -Unless there is a specific reason to set this field to a different value, it should be set to the same value as `publish_time`, as this reflects the time flow as reflected in the OSI message stream. -Only if recreation of the message stream with actual message transmission times is required - for example for asynchronous packaging layers, this field should be set to the time when the message was enqueued for addition to the trace file. +Unless there is a specific reason to set this field to a different value, it should be set to the same value as `publish_time`, as this reflects the time flow of the OSI message stream. +Only if recreation of the message stream with actual message transmission times is required, for example for asynchronous packaging layers, this field should be set to the simulation time when the message was enqueued for addition to the trace file. `publish_time`:: The timestamp taken from the timestamp field of the stored OSI top-level message. From f69b481d5a87f38a38e579818e3e34a357aff3ce Mon Sep 17 00:00:00 2001 From: Carlo van Driesten Date: Thu, 28 Nov 2024 14:14:13 +0100 Subject: [PATCH 12/33] Update trace_file_formats.adoc * Make zero_time and creation_time recommended Signed-off-by: Carlo van Driesten --- doc/architecture/trace_file_formats.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 8e26e5166..aa81d7560 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -82,6 +82,8 @@ This is a string in the format `major.minor.patch`, for example `3.17.3`. The version of the protobuf implementation used in the trace file. This is a string in the format `major.minor.patch`, for example `3.26.0`. +The `net.asam.osi.trace` meta-data record may also contain the following recommended `metadata` entries: + `zero_time`:: The point in real or simulated calendar time corresponding to time 0 in all timestamps in the trace file. Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. @@ -94,8 +96,6 @@ NOTE: Even in pure simulation use cases there is usually a relationship to a sim The point in real calendar time when the recording of the data in the trace file was started. Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. -The `net.asam.osi.trace` meta-data record may also contain the following recommended `metadata` entries: - `description`:: A human-readable description of the data contained in the multi channel trace file. From 301790a201f8f3cf78deb12317b1275270e760d9 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Fri, 29 Nov 2024 13:11:29 +0100 Subject: [PATCH 13/33] Clarify that only a single file global meta-data record is allowed Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index aa81d7560..c19662e3b 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -58,7 +58,7 @@ Storing of unrelated message streams in one trace file is therefore not generall [#sec-osi-trace-file-multi-global-meta-data] === Multi channel trace file global meta-data -The file must contain a meta-data record with the `name` field being `net.asam.osi.trace` and the following mandatory `metadata` entries: +The file must contain exactly one meta-data record with the `name` field being `net.asam.osi.trace` and the following mandatory `metadata` entries: `version`:: The semantic version number of the OSI release that this OSI trace file conforms to. From 15b043f207763f22fc89fe14750e6b3511687b36 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 27 May 2025 13:22:37 +0200 Subject: [PATCH 14/33] Provide example for creation_time based on review feedback Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index c19662e3b..8cb08c5e9 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -93,7 +93,7 @@ Values must match the following regular expression: NOTE: Even in pure simulation use cases there is usually a relationship to a simulated calendar time, as on-board components and environment simulation have necessary relationships to a simulated calendar time (for example, the embedded `HostVehicleData` will carry relevant simulated calendar time information, for localization and other purposes). `creation_time`:: -The point in real calendar time when the recording of the data in the trace file was started. +The point in real calendar time when the recording of the data in the trace file was started, e.g. when the recording system began writing the data to a file. Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. `description`:: From 1d96e477d3e5f16a73b0709ea6e8476ee5efe0ae Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 10 Jun 2025 13:26:56 +0200 Subject: [PATCH 15/33] Enhance discussion of non-top-level message types Based on review feedback Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 8cb08c5e9..265a5919f 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -135,7 +135,8 @@ The schema record must be stored in the summary section of the trace file, and m === OSI channel An OSI channel is a data stream within the OSI multi channel trace file that contains serialized OSI top-level messages of the same type. -Note that non-top-level messages must not be stored directly in OSI channels. +Note that this specification does not allow the storage of non-top-level OSI messages in OSI channels, as they cannot be reliably associated with other OSI data streams without additional context (e.g. due to the absence of a timestamp field or lack of attribution). +It is still possible to store them like any other related data in non-OSI channels, however this usage is unlikely to be portable. Each OSI channel must be described by a channel record in the summary section of the trace file with the following fields: From 5ab48f9921302f549e9ba9931381244e16748667 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 1 Jul 2025 13:56:00 +0200 Subject: [PATCH 16/33] Clarify rules on MCAP profile field in header Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 265a5919f..91dfb73b7 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -40,6 +40,9 @@ The OSI multi channel trace file format is based on the MCAP file format, which The OSI multi channel trace file format is a specialization of the MCAP file format, with additional constraints and requirements specific to OSI. Hence, any valid OSI multi channel trace file is also a valid MCAP file, but not the other way around. +The OSI multi channel trace file format does not specify an MCAP profile. +Therefore, it can be used with any or an empty profile field in the MCAP header record, as long as the given profile does not contradict or disallow any of the requirements of this specification. + The following rules apply to OSI multi channel trace files: - The file extension to be used is `.mcap`. From c0bbdebc4cffe7df416d68bba9c455a3112bf342 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Fri, 11 Jul 2025 08:40:57 +0200 Subject: [PATCH 17/33] Adjust wording to current normative modal verbs Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 91dfb73b7..caf7d42ba 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -46,13 +46,13 @@ Therefore, it can be used with any or an empty profile field in the MCAP header The following rules apply to OSI multi channel trace files: - The file extension to be used is `.mcap`. -- The file must be a valid MCAP file according to the https://mcap.dev/spec[MCAP format specification] version `0x30`. -- The file must be an indexed MCAP file, including chunk index records in the summary section, with all message records written into chunk records. -- Chunk records must either be uncompressed or compressed using either the `zstd` or `lz4` compression algorithms. -- The file must contain a meta-data record with the OSI trace file meta-data defined in section <>. +- The file shall be a valid MCAP file according to the https://mcap.dev/spec[MCAP format specification] version `0x30`. +- The file shall be an indexed MCAP file, including chunk index records in the summary section, with all message records written into chunk records. +- Chunk records shall either be uncompressed or compressed using either the `zstd` or `lz4` compression algorithms. +- The file shall contain a meta-data record with the OSI trace file meta-data defined in section <>. This meta-data record identifies the file as an OSI multi channel trace file. -- The file must contain a schema record in the summary section for each top-level message type that is used in one or more OSI channels with the OSI message schema as defined in section <>. -- The file must contain at least one OSI message stream in a channel as defined in section <>. +- The file shall contain a schema record in the summary section for each top-level message type that is used in one or more OSI channels with the OSI message schema as defined in section <>. +- The file shall contain at least one OSI message stream in a channel as defined in section <>. - The file may contain additional non-OSI message streams in other channels. NOTE: All OSI and non-OSI message streams stored in the same file share a common time base. @@ -61,7 +61,7 @@ Storing of unrelated message streams in one trace file is therefore not generall [#sec-osi-trace-file-multi-global-meta-data] === Multi channel trace file global meta-data -The file must contain exactly one meta-data record with the `name` field being `net.asam.osi.trace` and the following mandatory `metadata` entries: +The file shall contain exactly one meta-data record with the `name` field being `net.asam.osi.trace` and the following mandatory `metadata` entries: `version`:: The semantic version number of the OSI release that this OSI trace file conforms to. @@ -89,15 +89,15 @@ The `net.asam.osi.trace` meta-data record may also contain the following recomme `zero_time`:: The point in real or simulated calendar time corresponding to time 0 in all timestamps in the trace file. -Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. -Values must match the following regular expression: +This shall be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. +Values shall match the following regular expression: `-?([1-9][0-9]{3,}|0[0-9]{3})-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T(([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9](\.[0-9]+)?|(24:00:00(\.0+)?))(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))` + NOTE: Even in pure simulation use cases there is usually a relationship to a simulated calendar time, as on-board components and environment simulation have necessary relationships to a simulated calendar time (for example, the embedded `HostVehicleData` will carry relevant simulated calendar time information, for localization and other purposes). `creation_time`:: The point in real calendar time when the recording of the data in the trace file was started, e.g. when the recording system began writing the data to a file. -Must be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. +This shall be provided as a full ISO8601 formatted date time string, including timezone data, conforming to the https://www.w3.org/TR/xmlschema11-2/#dateTimeStamp[XML Schema dateTimeStamp] lexical space. `description`:: A human-readable description of the data contained in the multi channel trace file. @@ -114,10 +114,10 @@ It is strongly recommended to follow reverse domain name notation for custom met [#sec-osi-trace-file-multi-schema-record] === OSI message schema -For each OSI top-level message type that is used in one or more OSI channels, the OSI multi channel trace file must contain a corresponding schema record in the summary section. -Note that if multiple versions of the OSI schema are used in the same trace file, a schema record must be included for each version, with different schema IDs. +For each OSI top-level message type that is used in one or more OSI channels, the OSI multi channel trace file shall contain a corresponding schema record in the summary section. +Note that if multiple versions of the OSI schema are used in the same trace file, a schema record shall be included for each version, with different schema IDs. -The schema record must contain the following fields: +The schema record shall contain the following fields: `id`:: A file-wide unique non-zero identifier for the schema record. @@ -132,7 +132,7 @@ The value `protobuf`. `data`:: A binary FileDescriptorSet as produced by `protoc --include_imports --descriptor_set_out`. -The schema record must be stored in the summary section of the trace file, and must be referenced by the OSI channels that use the schema. +The schema record shall be stored in the summary section of the trace file, and shall be referenced by the OSI channels that use the schema. [#sec-osi-trace-file-multi-channel] === OSI channel @@ -141,7 +141,7 @@ An OSI channel is a data stream within the OSI multi channel trace file that con Note that this specification does not allow the storage of non-top-level OSI messages in OSI channels, as they cannot be reliably associated with other OSI data streams without additional context (e.g. due to the absence of a timestamp field or lack of attribution). It is still possible to store them like any other related data in non-OSI channels, however this usage is unlikely to be portable. -Each OSI channel must be described by a channel record in the summary section of the trace file with the following fields: +Each OSI channel shall be described by a channel record in the summary section of the trace file with the following fields: `id`:: A file-wide unique identifier for the channel. @@ -177,11 +177,11 @@ The following recommended entries are defined by {THIS_STANDARD}: `net.asam.osi.trace.channel.description`::: A human-readable description of the channel. -The channel record must be stored in the summary section of the trace file, and must be referenced by the OSI message records that are part of the channel. +The channel record shall be stored in the summary section of the trace file, and shall be referenced by the OSI message records that are part of the channel. -All messages in an OSI channel must be stored in chunk records in the data section of the trace file. +All messages in an OSI channel shall be stored in chunk records in the data section of the trace file. -Each message record in a chunk record must contain the following fields: +Each message record in a chunk record shall contain the following fields: `channel_id`:: The ID of the channel that the message belongs to. @@ -200,7 +200,7 @@ Only if recreation of the message stream with actual message transmission times `publish_time`:: The timestamp taken from the timestamp field of the stored OSI top-level message. The field is in nanoseconds, with the epoch being the epoch of the OSI Timestamp data type. -If top-level messages that do not contain a timestamp field are stored in the trace file, the `publish_time` field must be set to the time when the message was enqueued for addition to the trace file. +If top-level messages that do not contain a timestamp field are stored in the trace file, the `publish_time` field shall be set to the time when the message was enqueued for addition to the trace file. `data`:: The serialized OSI message data. From d2f71df1dc44438d0193a891ae07640374b2606d Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 28 Jan 2025 12:43:18 +0100 Subject: [PATCH 18/33] Add Tait-Bryan conv. and quat formula Signed-off-by: Thomas Sedlmayer --- doc/usecases/transforming_coordinate_systems.adoc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/doc/usecases/transforming_coordinate_systems.adoc b/doc/usecases/transforming_coordinate_systems.adoc index 541003335..f8a1980e0 100644 --- a/doc/usecases/transforming_coordinate_systems.adoc +++ b/doc/usecases/transforming_coordinate_systems.adoc @@ -71,6 +71,8 @@ Get Tait–Bryan angles from rotation matrix cite:[wiki_euler_angles]: \phi = \arctan2(R_{23}/\cos(\theta),R_{33}/\cos(\theta)) ++++ +Note that OSI uses the following convention on choosing rotation axes for Tait-Bryan angles: *z-y'-x''* intrinsic rotations (equivalent to *x-y-z* extrinsic rotations); see cite:[tait_bryan_convention]. + **Relative orientation**: Object rotation Matrix: latexmath:[\boldsymbol{R}_{object}^{src}] + @@ -80,6 +82,19 @@ Resulting rotation matrix between object and host: latexmath:[\boldsymbol{R}_{ob To transform from world coordinates into host vehicle coordinates and back use the formulas from above with the world coordinates frame latexmath:[w] as source system latexmath:[src] and host vehicle coordinates frame latexmath:[v] as target system latexmath:[trg]. To transform from host vehicle coordinates into sensor coordinates and back use the formulas from above with the host vehicle coordinates frame latexmath:[v] as source system latexmath:[src] and sensor coordinates frame latexmath:[s] as target system latexmath:[trg]. +**Converting orientation to quaternions**: + +To transform OSI's orientation representation from Tait-Bryan angles to quaternions, the following formula can be used [cite:euler_to_quaternion]: + +[latexmath] +++++ +\begin{align} + q_i &= \sin \frac{\phi}{2} \cos \frac{\theta}{2} \cos \frac{\psi}{2} - \cos \frac{\phi}{2} \sin \frac{\theta}{2} \sin \frac{\psi}{2}\\ + q_j &= \cos \frac{\phi}{2} \sin \frac{\theta}{2} \cos \frac{\psi}{2} + \sin \frac{\phi}{2} \cos \frac{\theta}{2} \sin \frac{\psi}{2}\\ + q_k &= \cos \frac{\phi}{2} \cos \frac{\theta}{2} \sin \frac{\psi}{2} - \sin \frac{\phi}{2} \sin \frac{\theta}{2} \cos \frac{\psi}{2}\\ + q_r &= \cos \frac{\phi}{2} \cos \frac{\theta}{2} \cos \frac{\psi}{2} + \sin \frac{\phi}{2} \sin \frac{\theta}{2} \sin \frac{\psi}{2} +\end{align} +++++ **Corresponding messages** From b993800e9d15efb3c0f53f858b80755a4a4e4dfb Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Fri, 11 Apr 2025 15:02:01 +0200 Subject: [PATCH 19/33] Fix doc formatting and reformulate Signed-off-by: Thomas Sedlmayer --- doc/usecases/transforming_coordinate_systems.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/usecases/transforming_coordinate_systems.adoc b/doc/usecases/transforming_coordinate_systems.adoc index f8a1980e0..e3c5d5f07 100644 --- a/doc/usecases/transforming_coordinate_systems.adoc +++ b/doc/usecases/transforming_coordinate_systems.adoc @@ -71,7 +71,7 @@ Get Tait–Bryan angles from rotation matrix cite:[wiki_euler_angles]: \phi = \arctan2(R_{23}/\cos(\theta),R_{33}/\cos(\theta)) ++++ -Note that OSI uses the following convention on choosing rotation axes for Tait-Bryan angles: *z-y'-x''* intrinsic rotations (equivalent to *x-y-z* extrinsic rotations); see cite:[tait_bryan_convention]. +Note that OSI uses the following convention on choosing rotation axes for Tait-Bryan angles: **z-y'-x''** intrinsic rotations (equivalent to **x-y-z** extrinsic rotations); see cite:[tait_bryan_convention]. **Relative orientation**: @@ -84,7 +84,7 @@ To transform from host vehicle coordinates into sensor coordinates and back use **Converting orientation to quaternions**: -To transform OSI's orientation representation from Tait-Bryan angles to quaternions, the following formula can be used [cite:euler_to_quaternion]: +To convert OSI's orientation representation from Tait-Bryan angles to quaternions use the following formula cite:[euler_to_quaternion]. The resulting quaternion is equivalent to yaw (ψ), pitch (θ) and roll (ϕ) angles (**x-y-z** extrinsic rotations) or intrinsic Tait-Bryan angles following the **z-y'-x''** convention. [latexmath] ++++ From a4086ec1b8174f41069d9cf0f9b70b74b6d6a68d Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Fri, 11 Jul 2025 11:49:01 +0200 Subject: [PATCH 20/33] Add bibliography reference for MCAP Signed-off-by: Pierre R. Mai --- doc/architecture/trace_file_formats.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index caf7d42ba..248623ae4 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -36,7 +36,7 @@ This release no longer contains any support for the legacy `.txt` file format. The OSI multi channel trace file format is a binary file format that allows for storing multiple serialized OSI message streams of the same or different types in one trace file, along with additional meta-data, and other related data streams. Due to the nature of the format, it allows for random access to the data streams, and is suitable for more advanced use-cases. -The OSI multi channel trace file format is based on the MCAP file format, which is a generic multi channel trace file format. +The OSI multi channel trace file format is based on the MCAP file format cite:[mcap], which is a generic multi channel trace file format. The OSI multi channel trace file format is a specialization of the MCAP file format, with additional constraints and requirements specific to OSI. Hence, any valid OSI multi channel trace file is also a valid MCAP file, but not the other way around. @@ -46,7 +46,7 @@ Therefore, it can be used with any or an empty profile field in the MCAP header The following rules apply to OSI multi channel trace files: - The file extension to be used is `.mcap`. -- The file shall be a valid MCAP file according to the https://mcap.dev/spec[MCAP format specification] version `0x30`. +- The file shall be a valid MCAP file according to the https://mcap.dev/spec[MCAP format specification] version `0x30` (cite:[mcap]). - The file shall be an indexed MCAP file, including chunk index records in the summary section, with all message records written into chunk records. - Chunk records shall either be uncompressed or compressed using either the `zstd` or `lz4` compression algorithms. - The file shall contain a meta-data record with the OSI trace file meta-data defined in section <>. From 4304139702e783b24732ea2f7619cdbaaddc0950 Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Thu, 25 Sep 2025 11:02:33 +0200 Subject: [PATCH 21/33] Adapt vehicle types from openx harmonization (#863) * Adapt vehicle types from openx harmonization * Update deprecation wording * Update micro-mobility dev., notes and references * Revise vehicle type description * Add note on compliance with ASAM TrafficParticipants Specification * Adapt note on OSI's obsolete type mapping issue (switch to past tense, revise wording) * Remove obsolete note on difficult distinction between OSI's vehicle types * Adapt ASAM style of standard names Co-authored-by: jakobkaths <54845080+jakobkaths@users.noreply.github.com> Signed-off-by: Thomas Sedlmayer --- .github/spelling_custom_words_en_US.txt | 6 + osi_object.proto | 219 ++++++++++++++++++++---- 2 files changed, 192 insertions(+), 33 deletions(-) diff --git a/.github/spelling_custom_words_en_US.txt b/.github/spelling_custom_words_en_US.txt index ee6e0358f..e1dbe552b 100644 --- a/.github/spelling_custom_words_en_US.txt +++ b/.github/spelling_custom_words_en_US.txt @@ -83,6 +83,7 @@ dms doi doppler drivable +DSL dt Duesseldorf Durchgangsverkehr @@ -90,6 +91,7 @@ dvr dürfen easting EBIKES +ECE edn edu egm @@ -161,6 +163,7 @@ Hagen halten hier Hochwasser +hoverboards href hsv htm @@ -173,6 +176,7 @@ illuminance ilv im Immission +implementer's incrementing Industriegebiet innerhalb @@ -342,6 +346,8 @@ uk umich umtri und +UNECE +unece Unfall UNGATED Universitaet diff --git a/osi_object.proto b/osi_object.proto index f4a326e59..d2b14c72a 100644 --- a/osi_object.proto +++ b/osi_object.proto @@ -760,23 +760,29 @@ message MovingObject { // Definition of vehicle types. // - // \note OSI provides a richer set of vehicle types than is supported by some - // other OpenX standards (in particular, OpenScenario 1.x and 2.x, and OpenLabel). - // This is primarily for historical reasons. Where a single type from a - // different standard can map to multiple OSI types it is left up to the - // discretion of the OSI implementer how that mapping is achieved. In previous - // versions, for example, a simulator might have used the dimensions of a provided - // 3d model of a vehicle with type "car" in OpenScenario, to determine whether it - // should be a TYPE_SMALL_CAR or TYPE_MEDIUM_CAR in OSI. As part of the harmonization - // effort, it should now map to TYPE_CAR, which is an alias of the old TYPE_MEDIUM_CAR, - // and all other car type enums have been deprecated in favor of TYPE_CAR. - // - // \note Vehicle type classification is a complex area and there are no - // universally recognized standards. As such, the boundaries between some of the - // OSI vehicle types are not well-defined. It is left to the implementer to - // decide how to distinguish between them and agree that with any applications which - // make use of that specific interface instance. For example, how to distinguish - // between a HEAVY_TRUCK and a DELIVERY_VAN, or a TRAILER and a SEMITRAILER. + // \note OSI defines a set of vehicle types derived directly from the + // vehicle main categories specified in the ASAM TrafficParticipants + // Specification [1]. The type names and definitions reflect an effort + // toward compatibility with multiple ASAM OpenX standards, to support + // consistent cross-standard mapping. As part of ongoing and future + // harmonization efforts, OSI will continue to keep its vehicle type + // definitions synchronized with the ASAM TrafficParticipants + // Specification as permitted by technical and practical constraints. + // + // \note Historically, OSI offered a more granular set of vehicle types + // than some other OpenX standards (e.g., OpenSCENARIO XML/DSL and + // OpenLABEL). This was due to legacy design choices. Mapping a single + // type from another standard to multiple OSI types was left to the + // implementer's discretion. For instance, a simulator might have relied + // on the 3D model's dimensions for an OpenSCENARIO "car" to choose + // between TYPE_SMALL_CAR or TYPE_MEDIUM_CAR in OSI. With the ongoing + // harmonization effort, such ambiguities have been resolved. The + // recommended mapping is now TYPE_CAR, which serves as an alias for the + // former TYPE_MEDIUM_CAR. Previous car-specific enums (e.g., + // TYPE_SMALL_CAR) have been deprecated in favor of TYPE_CAR. + // + // \par References: + // [1] ASAM e.V. (2024). ASAM TrafficParticipants Specification. ASAM OpenX Standards. Retrieved June 12, 2025, from https://publications.pages.asam.net/standards/ASAM_TrafficParticipants_Specification/ASAM_TrafficParticipants_Specification/v1.0.0/specification/index.html // enum Type { @@ -808,11 +814,18 @@ message MovingObject // TYPE_COMPACT_CAR = 3; - // Vehicle is a car. + // A car is a motorized vehicle designed primarily for passenger + // transportation. + // Typically has four wheels. // // This is to be used for all car-like vehicles, without any // further differentiated type available. // + // \note Corresponds roughly with UNECE [1] categories M1, G and L7. + // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // TYPE_CAR = 4; // Vehicle is a medium car. @@ -835,67 +848,207 @@ message MovingObject // // \image html OSI_TYPE_DELIVERY_VAN.svg // + // \note Deprecated differentiation, use TYPE_VAN instead + // TYPE_DELIVERY_VAN = 6; - // Vehicle is a (heavy) truck. + // A van is a motorized vehicle with a larger cargo area than a car, + // used for transporting goods or people. + // + // \note Corresponds roughly with UNECE [1] categories M2 and N1. + // + // \note This category is not intended for mini vans, which shall + // rather be categorized as cars. + // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // + TYPE_VAN = 6; + + // A heavy truck is a large commercial vehicle designed for + // transporting heavy loads. + // The cargo area is rigidly fixed to the vehicle itself. + // + // \note Corresponds roughly with UNECE [1] category "N2". // // \image html OSI_TYPE_HEAVY_TRUCK.svg // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // TYPE_HEAVY_TRUCK = 7; - // Vehicle is a tractor capable of pulling a semi-trailer. + // A semi-tractor is a vehicle designed for towing semi-trailers for + // the transportation of heavy loads. + // + // \note Corresponds roughly with UNECE [1] category N3. // // \image html OSI_TYPE_SEMITRACTOR.svg // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // TYPE_SEMITRACTOR = 16; - // This vehicle is a semi-trailer that can be pulled by a + // A semi-trailer is a vehicle designed for being towed by a + // semi-tractor. + // Characteristics compared to a regular trailer are the large size, + // the fact that a large portion of the weight is supported at the + // hitch, and a large overlap with the towing vehicle, i.e. the // semi-tractor. // - // \note The vehicle can be, but doesn't need to be, - // attached to another vehicle. + // \note Corresponds roughly with UNECE [1] category O4. + // + // \note Main reason for separation from the "Trailer" category is + // the difference in motion behavior (vehicle dynamics) and the + // different sensor perception due to the overlay with the towing + // vehicle. + // + // \note The vehicle can be, but doesn't need to be, attached to + // another vehicle. // // \image html OSI_TYPE_SEMITRAILER.svg // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // TYPE_SEMITRAILER = 8; - // Vehicle is a trailer. + // A trailer is a non-motorized vehicle designed for being towed by + // a motorized vehicle to carry goods, animals, or people. + // + // \note Corresponds roughly with UNECE [1] category O1 to O3. // - // \note The vehicle can be, but doesn't need to be, - // attached to another vehicle. + // \note The vehicle can be, but doesn't need to be, attached to + // another vehicle. // // \image html OSI_TYPE_TRAILER.svg // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // TYPE_TRAILER = 9; // Vehicle is a motorbike or moped. // + // \note Deprecated name, use TYPE_MOTORCYCLE instead + // TYPE_MOTORBIKE = 10; - // Vehicle is a bicycle (without motor and specific lights). + // A motorcycle is a motorized vehicle designed primarily for + // passenger transportation. + // Compared to a car, fewer passive safety features, such as a full + // passenger cell, are typically present. + // This category includes both two-wheeled motorcycles and + // three-wheeled vehicles like motorcycles with side-cars or trikes. + // + // \note Corresponds roughly with UNECE [1] categories L1 to L5. + // + // \note Usually considered as a vulnerable road user. + // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf + // + TYPE_MOTORCYCLE = 10; + + // A bicycle is a human-powered or motor-assisted, pedal-driven + // vehicle. + // This category includes typical two-wheeled bicycles as well as + // cargo-bikes and other pedal-driven vehicles with more than two + // wheels. + // + // \note Usually considered as a vulnerable road user. // TYPE_BICYCLE = 11; - // Vehicle is a bus. + // A bus is a motorized vehicle designed to carry multiple + // passengers. + // + // \note Corresponds roughly with UNECE [1] category M3 classes I, + // II, II. + // + // \par References: + // [1] United Nations Economic and Social Council. (2023). Consolidated Resolution on the Construction of Vehicles (ECE/TRANS/WP.29/78/Rev.7) [PDF]. United Nations. Retrieved May 3, 2024, from https://unece.org/sites/default/files/2023-12/ECE_TRANS_WP.29_78_Rev.7e.pdf // TYPE_BUS = 12; - // Vehicle is a tram. + // A tram is a vehicle designed for using rail infrastructure for + // the transport of passengers on rail infrastructure. + // The rail infrastructure may fully or partially overlap with + // public road infrastructure. + // In contrast to trains, trams do not have exclusive right-of-way. + // A tram often acts as a series of connected vehicles. // TYPE_TRAM = 13; - // Vehicle is a train. + // A train is a vehicle designed for the transport of passengers and + // freight on rail infrastructure. + // The rail infrastructure for trains is mostly grade-separated from + // the public road infrastructure as trains have exclusive + // right-of-way. + // Therefore, in case crossings with the road infrastructure occur, + // the exclusive right-of-way is ensured, e.g. by railway barriers. + // A train often acts as a series of connected vehicles. // TYPE_TRAIN = 14; - // Vehicle is a wheelchair. + // A wheelchair is a manually or electrically powered mobility + // device with a seat mounted on a wheeled frame. + // Manual propulsion may be provided by the seated person or a + // person pushing the wheelchair. + // + // \note Usually considered as a vulnerable road user. // TYPE_WHEELCHAIR = 15; - // Vehicle is a stand-up scooter, including - // motorized versions. + // A stand-up scooter is a compact, typically two-wheeled device. + // It is operated with the rider standing on a deck between the + // wheels. + // It may be propelled by a motor or the rider making a kicking + // movement. + // + // \note Usually considered as a vulnerable road user. // TYPE_STANDUP_SCOOTER = 17; + + // A micro-mobility device is a small, lightweight vehicle for + // short-distance travel, like hoverboards or roller skates. + // While bicycles, stand-up scooters, and wheelchairs may + // technically fall into this category, the respective detailed + // categories shall be used instead. + // + // \note Usually considered as a vulnerable road user. + // + TYPE_MICROMOBILITY_DEVICE = 18; + + // A work machine is a vehicle designed for specific tasks (e.g., + // construction equipment, agricultural tractors, forklifts). + // + TYPE_WORK_MACHINE = 19; + + // A watercraft is a vehicle designed for travel on water (boats, + // ships, etc.). + // + // \note This category is deliberately generic and may be refined in + // future versions as needed. + // + TYPE_WATERCRAFT = 20; + + // An aircraft is a vehicle designed for flight through the air + // (airplanes, helicopters, etc.). + // + // \note This category is deliberately generic and may be refined in + // future versions as needed. + // + TYPE_AIRCRAFT = 21; + + // A land vehicle is a vehicle designed for travel on land. + // + // \note This category is deliberately generic to include land + // vehicles that do not fall into other categories and may be + // refined in future versions as needed. + // + TYPE_LAND_VEHICLE = 22; } // The type of the vehicle. From 25497c1c2524582134e27a8a72a36234b5375559 Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Thu, 25 Sep 2025 11:44:09 +0200 Subject: [PATCH 22/33] Fix base reference frame definitions (#866) - Remove specific reference frame definitions in base stationary and base moving - Add specific reference frame definitions where base objects are used - Mention all messages where the base objects are used in base definition Signed-off-by: Thomas Sedlmayer Co-authored-by: Pierre R. Mai --- osi_common.proto | 20 ++++++++------------ osi_detectedobject.proto | 6 ++++++ osi_detectedroadmarking.proto | 3 +++ osi_detectedtrafficlight.proto | 3 +++ osi_detectedtrafficsign.proto | 6 ++++++ osi_object.proto | 10 ++++++++++ osi_roadmarking.proto | 2 ++ osi_trafficlight.proto | 6 ++++++ osi_trafficsign.proto | 6 ++++++ 9 files changed, 50 insertions(+), 12 deletions(-) diff --git a/osi_common.proto b/osi_common.proto index cd49951a6..4e7d64be5 100644 --- a/osi_common.proto +++ b/osi_common.proto @@ -494,14 +494,12 @@ message BoundingBox // \brief The base attributes of a stationary object or entity. // // This includes the \c StationaryObject , \c TrafficSign , -// \c TrafficLight , \c RoadMarking messages. +// \c TrafficLight , \c RoadMarking messages and the corresponding detected +// object messages ( \c DetectedStationaryObject , \c DetectedTrafficSign , +// \c DetectedTrafficLight , and \c DetectedRoadMarking ). // -// \image html OSI_BaseStationary.svg -// -// All coordinates and orientations from ground truth objects are relative to -// the global ground truth frame (see image). (All coordinates and orientations -// from detected objects are relative to the host vehicle frame (see: -// \c Vehicle vehicle reference point).) +// The reference frame of \c BaseStationary depends on the context in which it +// is used and is defined in the message where the \c BaseStationary is used. // message BaseStationary { @@ -575,7 +573,7 @@ message BaseStationary // // \brief The base attributes of an object that is moving. // -// This includes the \c MovingObject messages. +// This includes the \c MovingObject and \c DetectedMovingObject messages. // // \image html OSI_BaseMoving.svg // @@ -583,10 +581,8 @@ message BaseStationary // // E.g. a vehicle is a base moving object. // -// All coordinates and orientations from ground truth objects are relative to -// the global ground truth frame. All coordinates and orientations -// from detected objects are relative to the host vehicle frame -// (see: \c MovingObject vehicle reference point). +// The reference frame of \c BaseMoving depends on the context in which it is +// used and is defined in the message where the \c BaseMoving is used. // message BaseMoving { diff --git a/osi_detectedobject.proto b/osi_detectedobject.proto index 3ea53c4e0..238a4af5f 100644 --- a/osi_detectedobject.proto +++ b/osi_detectedobject.proto @@ -114,6 +114,9 @@ message DetectedStationaryObject // The base parameters of the stationary object. // + // The parent frame of \c DetectedStationaryObject::base is the virtual + // sensor coordinate system. + // optional BaseStationary base = 2; // The root mean squared error of the base parameters of the detected @@ -205,6 +208,9 @@ message DetectedMovingObject // The base parameters of the moving object. // + // The parent frame of \c DetectedMovingObject::base is the virtual sensor + // coordinate system. + // // \note The bounding box does NOT include mirrors for vehicles. // \note The height includes the ground_clearance. It always goes from the // top to the ground. diff --git a/osi_detectedroadmarking.proto b/osi_detectedroadmarking.proto index 481e33d9c..3075e8e47 100644 --- a/osi_detectedroadmarking.proto +++ b/osi_detectedroadmarking.proto @@ -37,6 +37,9 @@ message DetectedRoadMarking // The base parameters of the road marking. // + // The parent frame of \c RoadMarking::base is the virtual sensor coordinate + // system. + // // The orientation of the bounding box \c #base // \c BaseStationary::orientation is defined as follows: // The z-axis of the \c BaseStationary::orientation is the vector from the diff --git a/osi_detectedtrafficlight.proto b/osi_detectedtrafficlight.proto index 0c7878f17..b481e5490 100644 --- a/osi_detectedtrafficlight.proto +++ b/osi_detectedtrafficlight.proto @@ -34,6 +34,9 @@ message DetectedTrafficLight // The base parameters of the traffic light. // + // The parent frame of \c TrafficLight::base is the virtual sensor + // coordinate system. + // // \c BaseStationary::orientation x-axis is view normal of the traffic // light's icon. // diff --git a/osi_detectedtrafficsign.proto b/osi_detectedtrafficsign.proto index 344c20250..39cad6ac3 100644 --- a/osi_detectedtrafficsign.proto +++ b/osi_detectedtrafficsign.proto @@ -52,6 +52,9 @@ message DetectedTrafficSign // The base parameters of the traffic sign. // + // The parent frame of \c DetectedMainSign::base is the virtual sensor + // coordinate system. + // // The orientation of the bounding box \c #base // ( \c BaseStationary::orientation ) is defined as follows: // @@ -186,6 +189,9 @@ message DetectedTrafficSign // The base parameters of the supplementary traffic sign. // + // The parent frame of \c DetectedSupplementarySign::base is the virtual + // sensor coordinate system. + // // The orientation of the bounding box \c SupplementarySign::base // ( \c BaseStationary::orientation ) is defined as follows: // The z-axis of the given \c BaseStationary::orientation is the vector diff --git a/osi_object.proto b/osi_object.proto index d2b14c72a..8814782cb 100644 --- a/osi_object.proto +++ b/osi_object.proto @@ -11,6 +11,8 @@ package osi3; // \c MovingObject e.g. pedestrian, animal, or vehicle) nor a traffic related // object (\c TrafficLight, \c TrafficSign). // +// The parent frame of a stationary object is the global coordinate system. +// // \image html OSI_BaseStationary.svg // // \c StationaryObject excludes traffic lights, traffic signs and road marking @@ -28,6 +30,9 @@ message StationaryObject // The base parameters of the stationary object. // + // The parent frame of \c StationaryObject::base is the global coordinate + // system. + // optional BaseStationary base = 2; // The classification of the stationary object. @@ -395,6 +400,8 @@ message StationaryObject // moving object (animal, pedestrian, etc), but not a stationary // object (\c TrafficLight, \c TrafficSign, or \c StationaryObject). // +// The parent frame of a moving object is the global coordinate system. +// // \image html OSI_MovingObject.svg // // \image html OSI_HostVehicle.svg @@ -416,6 +423,9 @@ message MovingObject // The base parameters of the vehicle. // + // The parent frame of \c MovingObject::base is the global coordinate + // system. + // // \note The bounding box does NOT include side mirrors for vehicles. // \note The height includes the ground_clearance. It always goes from the // top to the ground. diff --git a/osi_roadmarking.proto b/osi_roadmarking.proto index ea1b85912..29d27d6ef 100644 --- a/osi_roadmarking.proto +++ b/osi_roadmarking.proto @@ -37,6 +37,8 @@ message RoadMarking // The base parameters of the road marking. // + // The parent frame of \c RoadMarking::base is the global coordinate system. + // // The orientation of the bounding box \c #base // \c BaseStationary::orientation is defined as follows: // The z-axis of the \c BaseStationary::orientation is the vector from the diff --git a/osi_trafficlight.proto b/osi_trafficlight.proto index 6dfba2929..a8b77f22f 100644 --- a/osi_trafficlight.proto +++ b/osi_trafficlight.proto @@ -9,6 +9,9 @@ package osi3; // // \brief A traffic light. // +// All coordinates and orientations are relative to the global ground truth +// coordinate system. +// // \image html OSI_TrafficLight.svg // // One traffic light message defines a single 'bulb' and not a box of several @@ -27,6 +30,9 @@ message TrafficLight // The base parameters of the traffic light. // + // The parent frame of \c TrafficLight::base is the global coordinate + // system. + // // \c BaseStationary::orientation x-axis is view normal of the traffic // light's icon. // diff --git a/osi_trafficsign.proto b/osi_trafficsign.proto index fa3a0e4d3..5b15bbcee 100644 --- a/osi_trafficsign.proto +++ b/osi_trafficsign.proto @@ -222,6 +222,9 @@ message TrafficSign { // The base parameters of the traffic sign. // + // The parent frame of \c MainSign::base is the global coordinate + // system. + // // The orientation of the bounding box \c #base // ( \c BaseStationary::orientation ) is defined as follows: // The z-axis of the given \c BaseStationary::orientation is the vector @@ -5585,6 +5588,9 @@ message TrafficSign { // The base parameters of the supplementary traffic sign. // + // The parent frame of \c SupplementarySign::base is the global + // coordinate system. + // // The orientation of the bounding box \c SupplementarySign::base // ( \c BaseStationary::orientation ) is defined as follows: // The z-axis of the given \c BaseStationary::orientation is the vector From 2cde78a5430df183e6598143e5e1076c4e95ffd5 Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 22 Jul 2025 12:38:54 +0200 Subject: [PATCH 23/33] Fix logical lane rule Signed-off-by: Thomas Sedlmayer --- osi_logicallane.proto | 4 ---- 1 file changed, 4 deletions(-) diff --git a/osi_logicallane.proto b/osi_logicallane.proto index fc5d9f86a..0e86d728e 100644 --- a/osi_logicallane.proto +++ b/osi_logicallane.proto @@ -407,10 +407,6 @@ message LogicalLane // For LogicalLanes without a correspondence to a Lane.Classification.Subtype // (i.e. TYPE_MEDIAN, TYPE_CURB, TYPE_TRAM, TYPE_RAIL) this field has no value. // - // \rules - // refers_to: Lane - // \endrules - // repeated PhysicalLaneReference physical_lane_reference = 4; // The \link ReferenceLine reference line\endlink for this logical lane From 8c587a216dc0c4e1728f3f91115f315f0bea88cb Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 22 Jul 2025 13:31:50 +0200 Subject: [PATCH 24/33] Remove incorrect rules Signed-off-by: Thomas Sedlmayer --- osi_lane.proto | 7 ------- 1 file changed, 7 deletions(-) diff --git a/osi_lane.proto b/osi_lane.proto index 2ac6b0094..3f4269764 100644 --- a/osi_lane.proto +++ b/osi_lane.proto @@ -748,13 +748,6 @@ message LaneBoundary // \attention For \c BoundaryPoint the same rule for the approximation // error applies as for \c Lane::Classification::centerline. // - // \rules - // first_element width is_equal_to 0.13 - // first_element height is_equal_to 0.14 - // last_element width is_equal_to 0.13 - // last_element height is_equal_to 0.13 - // \endrules - // repeated BoundaryPoint boundary_line = 2; // The classification of the lane boundary. From 9aa0da1fdfe58a4c05801dc4f1356e694fd1916a Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 22 Jul 2025 14:12:27 +0200 Subject: [PATCH 25/33] Fix rule syntax in sensor view Signed-off-by: Thomas Sedlmayer --- osi_sensorview.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osi_sensorview.proto b/osi_sensorview.proto index 68973fbf1..072f96ccd 100644 --- a/osi_sensorview.proto +++ b/osi_sensorview.proto @@ -138,7 +138,7 @@ message SensorView // The ID of the host vehicle in the \c #global_ground_truth data. // // \rules - // refers_to: 'MovingObject' + // refers_to: MovingObject // is_set // \endrules // From a62532927e884db15436eba0392a04e552b8ef16 Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 22 Jul 2025 14:14:35 +0200 Subject: [PATCH 26/33] Fix orientation3d formatting bug Signed-off-by: Thomas Sedlmayer --- osi_common.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osi_common.proto b/osi_common.proto index 4e7d64be5..493a13c6a 100644 --- a/osi_common.proto +++ b/osi_common.proto @@ -148,8 +148,8 @@ message Dimension3d // // The rotations are to be performed \b yaw \b first (around the z-axis), // \b pitch \b second (around the new y-axis) and \b roll \b third (around the -// new x-axis) to follow the definition according to [1] (Tait-Bryan / Euler -// convention z-y'-x''). The preferred angular range is [-pi, pi] for roll +// new x-axis) to follow the definition according to [1] using Tait-Bryan / Euler +// convention z-y'-x''. The preferred angular range is [-pi, pi] for roll // and yaw and [-pi/2, pi/2] for pitch. // // Roll/Pitch are 0 if the objects xy-plane is parallel to its parent's From 42c2c2e227d285db06645f6727dfc914f5c3845f Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Thu, 25 Sep 2025 09:50:10 +0200 Subject: [PATCH 27/33] Add new allowable rules minimum/maximum_length These checks are now supported by qc-osi-trace checker, so allow them in the CI tests. Signed-off-by: Pierre R. Mai --- rules.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rules.yml b/rules.yml index 41db521c3..aa4c82e29 100644 --- a/rules.yml +++ b/rules.yml @@ -11,3 +11,5 @@ first_element: '^[ ]\b(first_element)\b' last_element: '^[ ]\b(last_element)\b' check_if: '^[ ](\bcheck_if\b)(.*\belse do_check\b)' is_set: '^[ ]\b(is_set)\b' +minimum_length: '^[ ]\b(minimum_length)\b: ([\s\d]+)$' +maximum_length: '^[ ]\b(maximum_length)\b: ([\s\d]+)$' From d064f8d95acbd00728947d9dff11c69d822abb5a Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Tue, 7 Oct 2025 14:10:54 +0200 Subject: [PATCH 28/33] Remove faulty refer_to rules Signed-off-by: Thomas Sedlmayer --- osi_featuredata.proto | 20 -------------------- osi_logicaldetectiondata.proto | 4 ---- 2 files changed, 24 deletions(-) diff --git a/osi_featuredata.proto b/osi_featuredata.proto index c82c3c727..fd386cdce 100644 --- a/osi_featuredata.proto +++ b/osi_featuredata.proto @@ -273,10 +273,6 @@ message RadarDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // Measured position of the detection given in spherical coordinates in the @@ -380,10 +376,6 @@ message LidarDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // Measured position of the detection given in spherical coordinates in the @@ -565,10 +557,6 @@ message UltrasonicDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // Measured distance (radius) of the detection. @@ -616,10 +604,6 @@ message UltrasonicIndirectDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // First parameter b of an ellipsoid equation. @@ -706,10 +690,6 @@ message CameraDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // Difference to the base timestamp \c diff --git a/osi_logicaldetectiondata.proto b/osi_logicaldetectiondata.proto index fb7d09f11..c9b9b7299 100644 --- a/osi_logicaldetectiondata.proto +++ b/osi_logicaldetectiondata.proto @@ -137,10 +137,6 @@ message LogicalDetection // // \note ID = MAX(uint64) indicates no reference to an object. // - // \rules - // refers_to: DetectedObject - // \endrules - // optional Identifier object_id = 2; // Measured position of the logical detection given in cartesian coordinates From d16a10f36e7faeab81beffdf57fa1441c5c9742b Mon Sep 17 00:00:00 2001 From: Thomas Sedlmayer Date: Thu, 16 Oct 2025 18:48:58 +0200 Subject: [PATCH 29/33] Update refers_to rule regex (#881) Signed-off-by: Thomas Sedlmayer Co-authored-by: Pierre R. Mai --- rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index aa4c82e29..69501f2db 100644 --- a/rules.yml +++ b/rules.yml @@ -5,7 +5,7 @@ is_less_than: '^[ ]\b(is_less_than)\b: ([\s\d]+)$' is_equal_to: '^[ ]\b(is_equal_to)\b: ([\s\d]+)$' is_different_to: '^[ ]\b(is_different_to)\b: ([\s\d]+)$' is_globally_unique: '^[ ]\b(is_globally_unique)\b' -refers_to: '^[ ]\b(refers_to)\b' +refers_to: '^[ ]\b(refers_to)\b: ([^,\[\]\s]+|\[\s*(?:[^,\[\]\s]+\s*,\s*)*[^,\[\]\s]+\s*\])$' is_iso_country_code: '^[ ]\b(is_iso_country_code)\b' first_element: '^[ ]\b(first_element)\b' last_element: '^[ ]\b(last_element)\b' From e2791b65ffc194d6383a140b432c7cb030a4d9bf Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Thu, 23 Oct 2025 11:59:58 +0200 Subject: [PATCH 30/33] Clarify protobuf version descriptions in documentation (#883) Also change example version for trace file osi version to be a reasonable number. Signed-off-by: Pierre R. Mai Co-authored-by: Carlo van Driesten --- doc/architecture/trace_file_formats.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/architecture/trace_file_formats.adoc b/doc/architecture/trace_file_formats.adoc index 248623ae4..cccb13bca 100644 --- a/doc/architecture/trace_file_formats.adoc +++ b/doc/architecture/trace_file_formats.adoc @@ -65,7 +65,7 @@ The file shall contain exactly one meta-data record with the `name` field being `version`:: The semantic version number of the OSI release that this OSI trace file conforms to. -This is a string in the format `major.minor.patch`, for example `3.0.0`. +This is a string in the format `major.minor.patch`, for example `3.8.0`. Note that this version number is not necessarily the same as the OSI schema version(s) used in the trace file. Rather, it indicates the version of the OSI trace file format itself. @@ -78,11 +78,11 @@ The maximum version of the OSI schema used in the trace file OSI channels. This is a string in the format `major.minor.patch`, for example `3.7.0`. `min_protobuf_version`:: -The version of the protobuf implementation used in the trace file. +The minimum version of the protobuf implementation used in the trace file. This is a string in the format `major.minor.patch`, for example `3.17.3`. `max_protobuf_version`:: -The version of the protobuf implementation used in the trace file. +The maximum version of the protobuf implementation used in the trace file. This is a string in the format `major.minor.patch`, for example `3.26.0`. The `net.asam.osi.trace` meta-data record may also contain the following recommended `metadata` entries: From 1fcee6500087ff0aa80f3aefe088cfb1c8e13987 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 22 Jul 2025 12:37:34 +0200 Subject: [PATCH 31/33] Remove python/cpp build infrastructure Signed-off-by: Pierre R. Mai --- CMakeLists.txt | 278 --------- MANIFEST.in | 1 - convert-to-proto3.sh | 19 - ...mulation_interface-config-version.cmake.in | 21 - open_simulation_interface-config.cmake.in | 10 - pyproject.toml | 35 -- setup.py | 165 ----- tests/test_osi_trace.py | 589 ------------------ 8 files changed, 1118 deletions(-) delete mode 100644 CMakeLists.txt delete mode 100644 MANIFEST.in delete mode 100644 convert-to-proto3.sh delete mode 100644 open_simulation_interface-config-version.cmake.in delete mode 100644 open_simulation_interface-config.cmake.in delete mode 100644 pyproject.toml delete mode 100644 setup.py delete mode 100644 tests/test_osi_trace.py diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 6057fddf8..000000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,278 +0,0 @@ -cmake_minimum_required(VERSION 3.5) - -project(open_simulation_interface) - -# Toplevel check -set(OSI_IS_TOP_LEVEL OFF) -if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) - set(OSI_IS_TOP_LEVEL ON) -endif() - -# Set the C++ standard -set(CMAKE_CXX_STANDARD 11 CACHE STRING "C++ standard to be used") -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -# Optional Flatbuffer support -option(OSI_BUILD_FLATBUFFER "Build flatbuffer versions of libraries" OFF) -option(OSI_BUILD_DOCUMENTATION "Build flatbuffer versions of libraries" ${OSI_IS_TOP_LEVEL}) - -# Set a default build type if none was specified -set(default_build_type "Release") -if(EXISTS "${CMAKE_SOURCE_DIR}/.git") - set(default_build_type "Debug") -endif() - -# read the version number from the file "VERSION" -file(STRINGS "VERSION" VERSION_CONTENTS) -foreach(LINE ${VERSION_CONTENTS}) - string(REGEX REPLACE " |\t" "" LINE ${LINE}) - string(REGEX MATCH "^[^=]+" VERSION_NAME ${LINE}) - string(REPLACE "${VERSION_NAME}=" "" VERSION_VALUE ${LINE}) - set(${VERSION_NAME} "${VERSION_VALUE}") -endforeach() -set(OPEN_SIMULATION_INTERFACE_VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}) - -set(${PROJECT_NAME}_SOVERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") -set(${PROJECT_NAME}_LIBVERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") - -# configure build version -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to '${default_build_type}' as none was specified.") - set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE - STRING "Choose the type of build." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS - "Debug" "Release" "MinSizeRel" "RelWithDebInfo") -endif() - -# Offer the user the choice of overriding the installation directories -set(OSI_INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries") -set(OSI_INSTALL_INCLUDE_DIR include CACHE PATH "Installation directory for header files") - -if(WIN32 AND NOT CYGWIN) - set(DEF_INSTALL_CMAKE_DIR CMake/${PROJECT_NAME}-${VERSION_MAJOR}) -else() - set(DEF_INSTALL_CMAKE_DIR lib/cmake/${PROJECT_NAME}-${VERSION_MAJOR}) -endif() -set(OSI_INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH "Installation directory for CMake files") - -set(OSI_INSTALL_LIB_DIR ${OSI_INSTALL_LIB_DIR}/osi${VERSION_MAJOR}) -set(OSI_INSTALL_INCLUDE_DIR ${OSI_INSTALL_INCLUDE_DIR}/osi${VERSION_MAJOR}) - -configure_file(osi_version.proto.in osi_version.proto) - -find_package(Protobuf 2.6.1 REQUIRED) -set(PROTOBUF_IMPORT_DIRS ${PROTOBUF_INCLUDE_DIRS}) - -set(OSI_PROTO_FILES - ${CMAKE_CURRENT_BINARY_DIR}/osi_version.proto - osi_common.proto - osi_datarecording.proto - osi_detectedtrafficsign.proto - osi_detectedtrafficlight.proto - osi_detectedroadmarking.proto - osi_detectedlane.proto - osi_detectedobject.proto - osi_detectedoccupant.proto - osi_environment.proto - osi_groundtruth.proto - osi_hostvehicledata.proto - osi_motionrequest.proto - osi_trafficsign.proto - osi_trafficlight.proto - osi_trafficupdate.proto - osi_trafficcommand.proto - osi_trafficcommandupdate.proto - osi_referenceline.proto - osi_roadmarking.proto - osi_route.proto - osi_lane.proto - osi_logicallane.proto - osi_featuredata.proto - osi_logicaldetectiondata.proto - osi_object.proto - osi_occupant.proto - osi_sensordata.proto - osi_sensorviewconfiguration.proto - osi_sensorspecific.proto - osi_sensorview.proto - osi_streamingupdate.proto -) - -protobuf_generate_cpp(PROTO_SRCS PROTO_HEADERS ${OSI_PROTO_FILES}) -set(FLAT_HEADERS "") -if(OSI_BUILD_FLATBUFFER) - set(FLAT_FBS "") - find_package(flatbuffers REQUIRED) - if(NOT FLATBUFFERS_FLATC_EXECUTABLE) - set(FLATBUFFERS_FLATC_EXECUTABLE ${flatbuffers_DIR}/../../tools/flatbuffers/flatc) - endif() - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/descriptor.fbs" "namespace osi3;") - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/include") - list(APPEND FLAT_FBS "${CMAKE_CURRENT_BINARY_DIR}/descriptor.fbs") - foreach (proto ${OSI_PROTO_FILES}) - get_filename_component(proto_base ${proto} NAME_WE) - set(fbs "${proto_base}.fbs") - add_custom_command( - OUTPUT "${fbs}" - COMMAND ${FLATBUFFERS_FLATC_EXECUTABLE} -I "${PROTOBUF_IMPORT_DIRS}" -I "${CMAKE_CURRENT_BINARY_DIR}" -o "${CMAKE_CURRENT_BINARY_DIR}" --proto "${proto}" - DEPENDS "${proto}" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - COMMENT "Convert ${proto} to ${fbs} using flatc" - ) - list(APPEND FLAT_FBS "${CMAKE_CURRENT_BINARY_DIR}/${fbs}") - endforeach() - - foreach (flat ${FLAT_FBS}) - get_filename_component(flat_base ${flat} NAME_WE) - set(fbs "${flat_base}.fbs") - set(fbh "${flat_base}_generated.h") - add_custom_command( - OUTPUT "include/${fbh}" - COMMAND ${FLATBUFFERS_FLATC_EXECUTABLE} -o "${CMAKE_CURRENT_BINARY_DIR}/include" --cpp --gen-mutable --gen-name-strings --scoped-enums "${fbs}" - DEPENDS "${FLAT_FBS}" - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" - COMMENT "Process ${fbs} to ${fbh} using flatc" - ) - list(APPEND FLAT_HEADERS "${CMAKE_CURRENT_BINARY_DIR}/include/${fbh}") - endforeach() - - add_custom_target(${PROJECT_NAME}_fbs_build ALL DEPENDS "${FLAT_HEADERS}") - add_library(${PROJECT_NAME}_fbs INTERFACE) - add_library(${PROJECT_NAME}::${PROJECT_NAME}_fbs ALIAS ${PROJECT_NAME}_fbs) - target_include_directories(${PROJECT_NAME}_fbs INTERFACE $) - target_include_directories(${PROJECT_NAME}_fbs SYSTEM INTERFACE $/include>) - target_link_libraries(${PROJECT_NAME}_fbs INTERFACE flatbuffers::flatbuffers) -endif() - -add_library(${PROJECT_NAME}_static STATIC ${PROTO_SRCS} ${PROTO_HEADERS}) -add_library(${PROJECT_NAME}::${PROJECT_NAME}_static ALIAS ${PROJECT_NAME}_static) -target_include_directories(${PROJECT_NAME}_static - PUBLIC - ${PROTOBUF_INCLUDE_DIR} - $ - $ -) -target_link_libraries(${PROJECT_NAME}_static PUBLIC ${PROTOBUF_LIBRARY}) -install(TARGETS ${PROJECT_NAME}_static - EXPORT ${PROJECT_NAME}_targets - ARCHIVE DESTINATION "${OSI_INSTALL_LIB_DIR}" COMPONENT lib) - - -add_library(${PROJECT_NAME}_obj OBJECT ${PROTO_SRCS} ${PROTO_HEADERS}) -add_library(${PROJECT_NAME}::${PROJECT_NAME}_obj ALIAS ${PROJECT_NAME}_obj) -target_include_directories(${PROJECT_NAME}_obj - PUBLIC - ${PROTOBUF_INCLUDE_DIR} - $ - $ -) -set_property(TARGET ${PROJECT_NAME}_obj PROPERTY POSITION_INDEPENDENT_CODE ON) - - -add_library(${PROJECT_NAME}_pic STATIC $) -add_library(${PROJECT_NAME}::${PROJECT_NAME}_pic ALIAS ${PROJECT_NAME}_pic) -target_include_directories(${PROJECT_NAME}_pic - PUBLIC - ${PROTOBUF_INCLUDE_DIR} - $ - $ -) -target_link_libraries(${PROJECT_NAME}_pic PUBLIC ${PROTOBUF_LIBRARY}) - -set_property(TARGET ${PROJECT_NAME}_pic PROPERTY POSITION_INDEPENDENT_CODE ON) - -install(TARGETS ${PROJECT_NAME}_pic - EXPORT ${PROJECT_NAME}_targets - ARCHIVE DESTINATION "${OSI_INSTALL_LIB_DIR}" COMPONENT lib) - -add_library(${PROJECT_NAME} SHARED $) -add_library(${PROJECT_NAME}::${PROJECT_NAME} ALIAS ${PROJECT_NAME}) -target_include_directories(${PROJECT_NAME} - PUBLIC - ${PROTOBUF_INCLUDE_DIR} - $ - $ -) - -set_property( - TARGET ${PROJECT_NAME} - PROPERTY SOVERSION ${${PROJECT_NAME}_SOVERSION} -) -set_property( - TARGET ${PROJECT_NAME} - PROPERTY VERSION ${${PROJECT_NAME}_LIBVERSION} -) - -target_link_libraries(${PROJECT_NAME} PUBLIC ${PROTOBUF_LIBRARY}) -install(TARGETS ${PROJECT_NAME} - EXPORT ${PROJECT_NAME}_targets - DESTINATION "${OSI_INSTALL_LIB_DIR}" COMPONENT lib) - -# Copy proto headers to where they are expected by the package config file -add_custom_command( - TARGET ${PROJECT_NAME} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - ${CMAKE_CURRENT_BINARY_DIR}/${OSI_INSTALL_INCLUDE_DIR} - COMMAND ${CMAKE_COMMAND} -E copy - ${PROTO_HEADERS} - ${CMAKE_CURRENT_BINARY_DIR}/${OSI_INSTALL_INCLUDE_DIR}) - -# Create the package config files -include(CMakePackageConfigHelpers) -write_basic_package_version_file( - "${CMAKE_CURRENT_BINARY_DIR}/open_simulation_interface-config-version.cmake" - VERSION ${OPEN_SIMULATION_INTERFACE_VERSION} - COMPATIBILITY SameMajorVersion -) - -export(EXPORT ${PROJECT_NAME}_targets - FILE "${CMAKE_CURRENT_BINARY_DIR}/open_simulation_interface-targets.cmake" - NAMESPACE ${PROJECT_NAME}:: -) - -configure_package_config_file(open_simulation_interface-config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/open_simulation_interface-config.cmake" - INSTALL_DESTINATION ${OSI_INSTALL_CMAKE_DIR} - PATH_VARS OSI_INSTALL_INCLUDE_DIR - NO_CHECK_REQUIRED_COMPONENTS_MACRO -) - -# Install the *cmake files -install(FILES - "${CMAKE_CURRENT_BINARY_DIR}/open_simulation_interface-config.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/open_simulation_interface-config-version.cmake" - DESTINATION "${OSI_INSTALL_CMAKE_DIR}" - COMPONENT dev) - -# Header files -install(FILES ${PROTO_HEADERS} ${FLAT_HEADERS} - DESTINATION "${OSI_INSTALL_INCLUDE_DIR}") - -# Install the export set for use with the install-tree -install(EXPORT ${PROJECT_NAME}_targets - FILE open_simulation_interface-targets.cmake - NAMESPACE ${PROJECT_NAME}:: - DESTINATION "${OSI_INSTALL_CMAKE_DIR}" - COMPONENT dev) - -if(OSI_BUILD_DOCUMENTATION) - # add a target to generate API documentation with Doxygen - # Dependencies: Doxygen and proto2cpp.py - find_package(Doxygen) - if(Doxygen_FOUND) - set(FILTER_PROTO2CPP_PY_PATH CACHE PATH "directory to the filter proto2cpp.py") - if(NOT EXISTS ${FILTER_PROTO2CPP_PY_PATH}/proto2cpp.py) - message(WARNING "${FILTER_PROTO2CPP_PY_PATH}/proto2cpp.py could not be found.") - else() - set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/doxygen_config.cmake.in) - set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) - - configure_file(${doxyfile_in} ${doxyfile} @ONLY) - - add_custom_target(api_doc ALL - COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - endif() -endif() diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ceeea233f..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1 +0,0 @@ -include VERSION diff --git a/convert-to-proto3.sh b/convert-to-proto3.sh deleted file mode 100644 index 6cda582e9..000000000 --- a/convert-to-proto3.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -# This shell script converts all proto files to proto3 syntax, which can -# then be used as a stand in for the proto2 syntax files. The resulting -# on-the-wire format should be compatible with the proto2-based formats -# in all respects. -# -# Call this script prior to building normally if you want/need to use -# proto3 syntax, e.g. for language bindings that do not support proto2, -# like C#. - -test -f osi_version.proto && rm osi_version.proto - -for f in osi_version.proto.in osi_*.proto -do - mv $f $f.pb2 && sed -e 's/syntax *= *"proto2";/syntax = "proto3";/' -e 's/^\([ \t]*\)optional /\1/' $f.pb2 > $f -done - -mv CMakeLists.txt CMakeLists.txt.pb2 && sed -e 's/find_package(Protobuf 2.6.1 REQUIRED)/find_package(Protobuf 3.0.0 REQUIRED)/' CMakeLists.txt.pb2 > CMakeLists.txt diff --git a/open_simulation_interface-config-version.cmake.in b/open_simulation_interface-config-version.cmake.in deleted file mode 100644 index 2f6ae95e7..000000000 --- a/open_simulation_interface-config-version.cmake.in +++ /dev/null @@ -1,21 +0,0 @@ - -set(PACKAGE_VERSION_MAJOR "@VERSION_MAJOR@") - -set(PACKAGE_VERSION "@OPEN_SIMULATION_INTERFACE_VERSION@") -if("${PACKAGE_FIND_VERSION}" VERSION_EQUAL "0") - message(FATAL_ERROR "Please select at least the major version you want to use!") -else() - # Check whether the requested PACKAGE_FIND_VERSION is compatible - if("${PACKAGE_VERSION_MAJOR}" VERSION_EQUAL "${PACKAGE_FIND_VERSION_MAJOR}") - if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") - set(PACKAGE_VERSION_COMPATIBLE FALSE) - else() - set(PACKAGE_VERSION_COMPATIBLE TRUE) - if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") - set(PACKAGE_VERSION_EXACT TRUE) - endif() - endif() - else() - set(PACKAGE_VERSION_COMPATIBLE FALSE) - endif() -endif() diff --git a/open_simulation_interface-config.cmake.in b/open_simulation_interface-config.cmake.in deleted file mode 100644 index a285246e2..000000000 --- a/open_simulation_interface-config.cmake.in +++ /dev/null @@ -1,10 +0,0 @@ -@PACKAGE_INIT@ - -include(CMakeFindDependencyMacro) -find_dependency(Protobuf) - -if(NOT TARGET @PROJECT_NAME@ AND NOT @PROJECT_NAME@_BINARY_DIR) - set_and_check(OPEN_SIMULATION_INTERFACE_INCLUDE_DIRS "@PACKAGE_OSI_INSTALL_INCLUDE_DIR@") - set(OPEN_SIMULATION_INTERFACE_LIBRARIES "@PROJECT_NAME@") - include("${CMAKE_CURRENT_LIST_DIR}/open_simulation_interface-targets.cmake") -endif() diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 6627fa881..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,35 +0,0 @@ -[build-system] -requires = [ - "setuptools", - "wheel", - "protoc-wheel-0==24.4", -] -build-backend = "setuptools.build_meta" - -[project] -name = "open-simulation-interface" -description = "ASAM Open Simulation Interface Python Bindings." -authors = [ - {name = "ASAM Open Simulation Interface Project", email = "osi@asam.net"}, -] -maintainers = [ - {name = "ASAM Open Simulation Interface Project", email = "osi@asam.net"}, -] -dependencies = [ - "protobuf>=4.24.4", -] - -license = {file = "LICENSE"} -readme = "README.md" -classifiers = [ - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", -] -dynamic = ["version"] - -[project.urls] -Homepage = "https://github.com/OpenSimulationInterface/open-simulation-interface" -Repository = "https://github.com/OpenSimulationInterface/open-simulation-interface.git" -"Bug Tracker" = "https://github.com/OpenSimulationInterface/open-simulation-interface/issues" - -[project.scripts] -osi2read = "osi3trace.osi2read:main" diff --git a/setup.py b/setup.py deleted file mode 100644 index ea0e93eba..000000000 --- a/setup.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/local/env python3 -# -*- coding: UTF-8 -*- - -import os -import subprocess -import sys -import re -from distutils.spawn import find_executable - -from setuptools import setup -from setuptools.command.sdist import sdist -from setuptools.command.build_py import build_py - -# protoc -from protoc import PROTOC_EXE - -# configure the version number -VERSION_MAJOR = None -VERSION_MINOR = None -VERSION_PATCH = None -VERSION_SUFFIX = None -with open("VERSION", "rt") as versionin: - for line in versionin: - if line.startswith("VERSION_MAJOR"): - VERSION_MAJOR = int(line.split("=")[1].strip()) - if line.startswith("VERSION_MINOR"): - VERSION_MINOR = int(line.split("=")[1].strip()) - if line.startswith("VERSION_PATCH"): - VERSION_PATCH = int(line.split("=")[1].strip()) - if line.startswith("VERSION_SUFFIX"): - VERSION_SUFFIX = line.split("=")[1].strip() - -package_name = "osi3" -package_path = os.path.join(os.getcwd(), package_name) - - -class ProtobufGenerator: - @staticmethod - def find_protoc(): - """Locates protoc executable""" - - if os.path.exists(PROTOC_EXE): - protoc = PROTOC_EXE - elif "PROTOC" in os.environ and os.path.exists(os.environ["PROTOC"]): - protoc = os.environ["PROTOC"] - else: - protoc = find_executable("protoc") - - if protoc is None: - sys.stderr.write( - "protoc not found. Is protobuf-compiler installed? \n" - "Alternatively, you can point the PROTOC environment variable " - "to a local version." - ) - sys.exit(1) - return protoc - - osi_files = ( - "osi_common.proto", - "osi_datarecording.proto", - "osi_detectedlane.proto", - "osi_detectedobject.proto", - "osi_detectedoccupant.proto", - "osi_detectedroadmarking.proto", - "osi_detectedtrafficlight.proto", - "osi_detectedtrafficsign.proto", - "osi_environment.proto", - "osi_featuredata.proto", - "osi_groundtruth.proto", - "osi_hostvehicledata.proto", - "osi_lane.proto", - "osi_logicaldetectiondata.proto", - "osi_logicallane.proto", - "osi_motionrequest.proto", - "osi_object.proto", - "osi_occupant.proto", - "osi_referenceline.proto", - "osi_roadmarking.proto", - "osi_route.proto", - "osi_sensordata.proto", - "osi_sensorspecific.proto", - "osi_sensorview.proto", - "osi_sensorviewconfiguration.proto", - "osi_streamingupdate.proto", - "osi_trafficcommand.proto", - "osi_trafficcommandupdate.proto", - "osi_trafficlight.proto", - "osi_trafficsign.proto", - "osi_trafficupdate.proto", - "osi_version.proto", - ) - - """ Generate Protobuf Messages """ - - def generate(self): - sys.stdout.write("Generating Protobuf Version Message\n") - with open("osi_version.proto.in", "rt") as fin: - with open("osi_version.proto", "wt") as fout: - for line in fin: - lineConfigured = line.replace("@VERSION_MAJOR@", str(VERSION_MAJOR)) - lineConfigured = lineConfigured.replace( - "@VERSION_MINOR@", str(VERSION_MINOR) - ) - lineConfigured = lineConfigured.replace( - "@VERSION_PATCH@", str(VERSION_PATCH) - ) - fout.write(lineConfigured) - pattern = re.compile('^import "osi_') - for source in self.osi_files: - with open(source) as src_file: - with open(os.path.join(package_path, source), "w") as dst_file: - for line in src_file: - dst_file.write( - pattern.sub('import "' + package_name + "/osi_", line) - ) - for source in self.osi_files: - sys.stdout.write("Protobuf-compiling " + source + "\n") - source_path = os.path.join(package_name, source) - subprocess.check_call( - [self.find_protoc(), "--python_out=.", "--pyi_out=.", source_path] - ) - - def maybe_generate(self): - if os.path.exists("osi_version.proto.in"): - self.generate() - - -class CustomBuildPyCommand(build_py): - def run(self): - ProtobufGenerator().maybe_generate() - build_py.run(self) - - -class CustomSDistCommand(sdist): - def run(self): - ProtobufGenerator().generate() - sdist.run(self) - - -try: - os.mkdir(package_path) -except Exception: - pass - -try: - with open(os.path.join(package_path, "__init__.py"), "wt") as init_file: - init_file.write( - f"__version__ = '{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}{VERSION_SUFFIX or ''}'\n" - ) -except Exception: - pass - -setup( - version=str(VERSION_MAJOR) - + "." - + str(VERSION_MINOR) - + "." - + str(VERSION_PATCH) - + (VERSION_SUFFIX or ""), - packages=[package_name, "osi3trace"], - cmdclass={ - "sdist": CustomSDistCommand, - "build_py": CustomBuildPyCommand, - }, -) diff --git a/tests/test_osi_trace.py b/tests/test_osi_trace.py deleted file mode 100644 index 9d74f5a9d..000000000 --- a/tests/test_osi_trace.py +++ /dev/null @@ -1,589 +0,0 @@ -import os -import tempfile -import unittest - -from osi3trace.osi_trace import OSITrace -from osi3.osi_sensorview_pb2 import SensorView -from osi3.osi_sensorviewconfiguration_pb2 import SensorViewConfiguration -from osi3.osi_groundtruth_pb2 import GroundTruth -from osi3.osi_hostvehicledata_pb2 import HostVehicleData -from osi3.osi_sensordata_pb2 import SensorData -from osi3.osi_trafficcommand_pb2 import TrafficCommand -from osi3.osi_trafficcommandupdate_pb2 import TrafficCommandUpdate -from osi3.osi_trafficupdate_pb2 import TrafficUpdate -from osi3.osi_motionrequest_pb2 import MotionRequest -from osi3.osi_streamingupdate_pb2 import StreamingUpdate - -import struct - - -class TestOSITrace(unittest.TestCase): - def test_osi_trace_sv(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_sv.txth") - path_input = os.path.join(tmpdirname, "input_sv.osi") - create_sample_sv(path_input) - - trace = OSITrace(path_input) - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, SensorView) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_svc(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_svc.txth") - path_input = os.path.join(tmpdirname, "input_svc.osi") - create_sample_svc(path_input) - - trace = OSITrace(path_input, "SensorViewConfiguration") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, SensorViewConfiguration) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 1) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_gt(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_gt.txth") - path_input = os.path.join(tmpdirname, "input_gt.osi") - create_sample_gt(path_input) - - trace = OSITrace(path_input, "GroundTruth") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, GroundTruth) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_hvd(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_hvd.txth") - path_input = os.path.join(tmpdirname, "input_hvd.osi") - create_sample_hvd(path_input) - - trace = OSITrace(path_input, "HostVehicleData") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, HostVehicleData) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_sd(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_sd.txth") - path_input = os.path.join(tmpdirname, "input_sd.osi") - create_sample_sd(path_input) - - trace = OSITrace(path_input, "SensorData") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, SensorData) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_tc(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_tc.txth") - path_input = os.path.join(tmpdirname, "input_tc.osi") - create_sample_tc(path_input) - - trace = OSITrace(path_input, "TrafficCommand") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, TrafficCommand) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_tcu(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_tcu.txth") - path_input = os.path.join(tmpdirname, "input_tcu.osi") - create_sample_tcu(path_input) - - trace = OSITrace(path_input, "TrafficCommandUpdate") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, TrafficCommandUpdate) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_tu(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_tu.txth") - path_input = os.path.join(tmpdirname, "input_tu.osi") - create_sample_tu(path_input) - - trace = OSITrace(path_input, "TrafficUpdate") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, TrafficUpdate) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_mr(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_mr.txth") - path_input = os.path.join(tmpdirname, "input_mr.osi") - create_sample_mr(path_input) - - trace = OSITrace(path_input, "MotionRequest") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, MotionRequest) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_su(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_output = os.path.join(tmpdirname, "output_su.txth") - path_input = os.path.join(tmpdirname, "input_su.osi") - create_sample_su(path_input) - - trace = OSITrace(path_input, "StreamingUpdate") - with open(path_output, "wt") as f: - for message in trace: - self.assertIsInstance(message, StreamingUpdate) - f.write(str(message)) - - self.assertEqual(len(trace.retrieve_offsets()), 10) - trace.close() - - self.assertTrue(os.path.exists(path_output)) - - def test_osi_trace_offsets_robustness(self): - with tempfile.TemporaryDirectory() as tmpdirname: - path_input = os.path.join(tmpdirname, "input_robust.osi") - create_sample_sv(path_input) - - trace = OSITrace(path_input) - # Test whether the function can handle be run multiple times safely - offsets = trace.retrieve_offsets(None) - offsets2 = trace.retrieve_offsets(None) - trace.close() - - self.assertEqual(len(offsets), 10) - self.assertEqual(offsets, offsets2) - - -def create_sample_sv(path): - f = open(path, "ab") - sensorview = SensorView() - - sensorview.version.version_major = 3 - sensorview.version.version_minor = 0 - sensorview.version.version_patch = 0 - - sensorview.timestamp.seconds = 0 - sensorview.timestamp.nanos = 0 - - sensorview.sensor_id.value = 42 - - sv_ground_truth = sensorview.global_ground_truth - sv_ground_truth.version.version_major = 3 - sv_ground_truth.version.version_minor = 0 - sv_ground_truth.version.version_patch = 0 - - sv_ground_truth.timestamp.seconds = 0 - sv_ground_truth.timestamp.nanos = 0 - - moving_object = sv_ground_truth.moving_object.add() - moving_object.id.value = 114 - - # Generate 10 OSI messages for 9 seconds - for i in range(10): - # Increment the time - sensorview.timestamp.seconds += 1 - sensorview.timestamp.nanos += 100000 - - sv_ground_truth.timestamp.seconds += 1 - sv_ground_truth.timestamp.nanos += 100000 - - moving_object.vehicle_classification.type = 2 - - moving_object.base.dimension.length = 5 - moving_object.base.dimension.width = 2 - moving_object.base.dimension.height = 1 - - moving_object.base.position.x = 0.0 + i - moving_object.base.position.y = 0.0 - moving_object.base.position.z = 0.0 - - moving_object.base.orientation.roll = 0.0 - moving_object.base.orientation.pitch = 0.0 - moving_object.base.orientation.yaw = 0.0 - - """Serialize""" - bytes_buffer = sensorview.SerializeToString() - f.write(struct.pack(" Date: Tue, 22 Jul 2025 16:16:16 +0200 Subject: [PATCH 32/33] Rework workflow, remove obsolete python files Signed-off-by: Pierre R. Mai --- .github/workflows/protobuf.yml | 179 ++----------------------------- CMakeLists.txt | 33 ++++++ osi3trace/osi2read.py | 66 ------------ osi3trace/osi_trace.py | 188 --------------------------------- 4 files changed, 42 insertions(+), 424 deletions(-) create mode 100644 CMakeLists.txt delete mode 100644 osi3trace/osi2read.py delete mode 100644 osi3trace/osi_trace.py diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index 050c9a212..b4718a086 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -1,10 +1,5 @@ name: ProtoBuf CI Builds -env: - PROTOBUF_VERSION: 3.20.1 - PROTOBUF_VARIANT: '-all' # Use '-all' prior to 22.0, '' after - ABSEIL_VERSION: 20230802.1 - on: push: pull_request: @@ -21,8 +16,8 @@ jobs: with: config_path: .github/.pyspelling.yml - build-proto2-linux64: - name: Build Proto2 Linux 64 + run-tests: + name: Run tests and build docs runs-on: ubuntu-22.04 @@ -32,11 +27,6 @@ jobs: with: submodules: true - - name: Check Build Setup - run: | - ( result=0 ; for f in *.proto ; do grep -wq "$f" CMakeLists.txt || { echo "Missing $f in CMakeLists.txt" && let "result++"; } ; done ; exit $result ) - ( result=0 ; for f in *.proto ; do grep -q '"'$f'"' setup.py || { echo "Missing $f in setup.py" && let "result++"; } ; done ; exit $result ) - - name: Setup Python uses: actions/setup-python@v5 with: @@ -45,49 +35,15 @@ jobs: - name: Install Python Dependencies run: | python -m pip install --upgrade pip - python -m pip install build python -m pip install -r requirements_tests.txt - - name: Check black format - run: | - black --check --diff . - - name: Install Doxygen run: sudo apt-get install doxygen graphviz - - name: Cache Dependencies - id: cache-depends - uses: actions/cache@v4 - with: - path: protobuf-${{ env.PROTOBUF_VERSION }} - key: ${{ runner.os }}-v2-depends - - - name: Download ProtoBuf ${{ env.PROTOBUF_VERSION }} - if: steps.cache-depends.outputs.cache-hit != 'true' - run: curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${{env.PROTOBUF_VERSION}}/protobuf${{env.PROTOBUF_VARIANT}}-${{env.PROTOBUF_VERSION}}.tar.gz && tar xzvf protobuf${{env.PROTOBUF_VARIANT}}-${{env.PROTOBUF_VERSION}}.tar.gz - - - name: Download Abseil ${{ env.ABSEIL_VERSION }} - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '' - run: curl -OL https://github.com/abseil/abseil-cpp/archive/refs/tags/${{env.ABSEIL_VERSION}}.tar.gz && tar xzvf ${{env.ABSEIL_VERSION}}.tar.gz && rm -rf protobuf-${{env.PROTOBUF_VERSION}}/third_party/abseil-cpp && mv abseil-cpp-${{env.ABSEIL_VERSION}} protobuf-${{env.PROTOBUF_VERSION}}/third_party/abseil-cpp - - - name: Build ProtoBuf ${{ env.PROTOBUF_VERSION }} via autotools - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '-all' - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: ./configure DIST_LANG=cpp --prefix=/usr && make - - - name: Build ProtoBuf ${{ env.PROTOBUF_VERSION }} via cmake - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '' - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: cmake -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_TESTS=OFF . && cmake --build . --config Release -j 4 - - - name: Install ProtoBuf ${{ env.PROTOBUF_VERSION }} - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: sudo make install && sudo ldconfig - - name: Install proto2cpp run: git clone --depth 1 https://github.com/OpenSimulationInterface/proto2cpp.git - - name: Prepare C++ Build + - name: Prepare Build run: mkdir build - name: Add Development Version Suffix @@ -105,136 +61,19 @@ jobs: echo "EXCLUDE_PATTERNS = */osi3/* */protobuf-*/* */proto2cpp/* */flatbuffers/*" >> doxygen_config.cmake.in echo "GENERATE_TREEVIEW = YES" >> doxygen_config.cmake.in - - name: Configure C++ Build + - name: Configure Build working-directory: build - run: cmake -D FILTER_PROTO2CPP_PY_PATH=$GITHUB_WORKSPACE/proto2cpp ${{ env.PROTOBUF_VARIANT =='' && '-DCMAKE_CXX_STANDARD=17' }} .. + run: cmake -D FILTER_PROTO2CPP_PY_PATH=$GITHUB_WORKSPACE/proto2cpp .. - - name: Build C++ + - name: Build working-directory: build run: cmake --build . --config Release -j 4 - - name: Build Python - run: python -m build - - - name: Install Python - run: python -m pip install . - - name: Run Python Tests run: python -m unittest discover tests - - name: Upload Python Distribution - if: ${{ github.event_name == 'pull_request' || ( github.event_name == 'push' && ( github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') ) ) }} + - name: Upload documentation uses: actions/upload-artifact@v4 with: - name: python-dist - path: dist/ - - build-proto3-linux64: - name: Build Proto3 Linux 64 - - runs-on: ubuntu-22.04 - - steps: - - name: Checkout OSI - uses: actions/checkout@v4 - with: - submodules: true - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install build - python -m pip install -r requirements_tests.txt - - - name: Cache Dependencies - id: cache-depends - uses: actions/cache@v4 - with: - path: protobuf-${{ env.PROTOBUF_VERSION }} - key: ${{ runner.os }}-v2-depends - - - name: Download ProtoBuf ${{ env.PROTOBUF_VERSION }} - if: steps.cache-depends.outputs.cache-hit != 'true' - run: curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${{env.PROTOBUF_VERSION}}/protobuf${{env.PROTOBUF_VARIANT}}-${{env.PROTOBUF_VERSION}}.tar.gz && tar xzvf protobuf${{env.PROTOBUF_VARIANT}}-${{env.PROTOBUF_VERSION}}.tar.gz - - - name: Download Abseil ${{ env.ABSEIL_VERSION }} - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '' - run: curl -OL https://github.com/abseil/abseil-cpp/archive/refs/tags/${{env.ABSEIL_VERSION}}.tar.gz && tar xzvf ${{env.ABSEIL_VERSION}}.tar.gz && rm -rf protobuf-${{env.PROTOBUF_VERSION}}/third_party/abseil-cpp && mv abseil-cpp-${{env.ABSEIL_VERSION}} protobuf-${{env.PROTOBUF_VERSION}}/third_party/abseil-cpp - - - name: Build ProtoBuf ${{ env.PROTOBUF_VERSION }} via autotools - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '-all' - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: ./configure DIST_LANG=cpp --prefix=/usr && make - - - name: Build ProtoBuf ${{ env.PROTOBUF_VERSION }} via cmake - if: steps.cache-depends.outputs.cache-hit != 'true' && env.PROTOBUF_VARIANT == '' - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: cmake -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_TESTS=OFF . && cmake --build . --config Release -j 4 - - - name: Install ProtoBuf ${{ env.PROTOBUF_VERSION }} - working-directory: protobuf-${{ env.PROTOBUF_VERSION }} - run: sudo make install && sudo ldconfig - - - name: Prepare C++ Build - run: mkdir build - - - name: Switch to Proto3 Syntax - run: | - bash convert-to-proto3.sh - rm *.pb2 - - - name: Add Development Version Suffix - if: ${{ !startsWith(github.ref, 'refs/tags') }} - run: | - echo "VERSION_SUFFIX = .dev`date -u '+%Y%m%d%H%M%S'`" >> VERSION - - - name: Configure C++ Build - working-directory: build - run: cmake ${{ env.PROTOBUF_VARIANT =='' && '-DCMAKE_CXX_STANDARD=17' }} .. - - - name: Build C++ - working-directory: build - run: cmake --build . --config Release -j 4 - - - name: Build Python - run: python -m build - - - name: Install Python - run: python -m pip install . - - - name: Run Python Tests - run: python -m unittest discover tests - - publish-python-dist: - name: Publish Python Distribution - - runs-on: ubuntu-22.04 - - permissions: - id-token: write - - needs: [build-proto2-linux64, build-proto3-linux64] - - if: ${{ github.event_name == 'push' && ( github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') ) }} - - steps: - - name: Download Distribution - uses: actions/download-artifact@v4 - with: - name: python-dist - path: dist/ - - - name: Publish Snapshot Release on TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - continue-on-error: true - with: - repository-url: https://test.pypi.org/legacy/ - - - name: Publish Full Release on PyPI - if: startsWith(github.ref, 'refs/tags/v') && ( ! contains(github.ref, '-') ) - uses: pypa/gh-action-pypi-publish@release/v1 + name: doxygen-doc + path: doc/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..3cf21e062 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.10) + +project(open_simulation_interface_doc) + +# read the version number from the file "VERSION" +file(STRINGS "VERSION" VERSION_CONTENTS) +foreach(LINE ${VERSION_CONTENTS}) + string(REGEX REPLACE " |\t" "" LINE ${LINE}) + string(REGEX MATCH "^[^=]+" VERSION_NAME ${LINE}) + string(REPLACE "${VERSION_NAME}=" "" VERSION_VALUE ${LINE}) + set(${VERSION_NAME} "${VERSION_VALUE}") +endforeach() + +configure_file(osi_version.proto.in osi_version.proto) + +# add a target to generate API documentation with Doxygen +# Dependencies: Doxygen and proto2cpp.py +find_package(Doxygen) +if(Doxygen_FOUND) + set(FILTER_PROTO2CPP_PY_PATH CACHE PATH "directory to the filter proto2cpp.py") + if(NOT EXISTS ${FILTER_PROTO2CPP_PY_PATH}/proto2cpp.py) + message(WARNING "${FILTER_PROTO2CPP_PY_PATH}/proto2cpp.py could not be found.") + else() + set(doxyfile_in ${CMAKE_CURRENT_SOURCE_DIR}/doxygen_config.cmake.in) + set(doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) + + configure_file(${doxyfile_in} ${doxyfile} @ONLY) + + add_custom_target(api_doc ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endif() +endif() diff --git a/osi3trace/osi2read.py b/osi3trace/osi2read.py deleted file mode 100644 index a75bb9a76..000000000 --- a/osi3trace/osi2read.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -This program converts serialized osi trace files into a human readable txth file. - -Example usage: - python3 osi2read.py -d trace.osi -o myreadableosifile -""" - -from osi3trace.osi_trace import OSITrace -import argparse -import pathlib - - -def command_line_arguments(): - """Define and handle command line interface""" - - parser = argparse.ArgumentParser( - description="Convert a serialized osi trace file to a readable txth output.", - prog="osi2read", - ) - parser.add_argument( - "--data", - "-d", - help="Path to the file with serialized data.", - type=str, - required=True, - ) - parser.add_argument( - "--type", - "-t", - help="Name of the type used to serialize data.", - choices=OSITrace.message_types(), - default="SensorView", - type=str, - required=False, - ) - parser.add_argument( - "--output", - "-o", - help="Output name of the file.", - type=str, - required=False, - ) - - return parser.parse_args() - - -def main(): - # Handling of command line arguments - args = command_line_arguments() - - # Initialize the OSI trace class - trace = OSITrace(args.data, args.type) - - if not args.output: - path = pathlib.Path(args.data).with_suffix(".txth") - args.output = str(path) - - with open(args.output, "wt") as f: - for message in trace: - f.write(str(message)) - - trace.close() - - -if __name__ == "__main__": - main() diff --git a/osi3trace/osi_trace.py b/osi3trace/osi_trace.py deleted file mode 100644 index 3c0577023..000000000 --- a/osi3trace/osi_trace.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -Module to handle and manage OSI trace files. -""" - -import lzma -import struct - -from osi3.osi_sensorview_pb2 import SensorView -from osi3.osi_sensorviewconfiguration_pb2 import SensorViewConfiguration -from osi3.osi_groundtruth_pb2 import GroundTruth -from osi3.osi_hostvehicledata_pb2 import HostVehicleData -from osi3.osi_sensordata_pb2 import SensorData -from osi3.osi_trafficcommand_pb2 import TrafficCommand -from osi3.osi_trafficcommandupdate_pb2 import TrafficCommandUpdate -from osi3.osi_trafficupdate_pb2 import TrafficUpdate -from osi3.osi_motionrequest_pb2 import MotionRequest -from osi3.osi_streamingupdate_pb2 import StreamingUpdate - - -MESSAGES_TYPE = { - "SensorView": SensorView, - "SensorViewConfiguration": SensorViewConfiguration, - "GroundTruth": GroundTruth, - "HostVehicleData": HostVehicleData, - "SensorData": SensorData, - "TrafficCommand": TrafficCommand, - "TrafficCommandUpdate": TrafficCommandUpdate, - "TrafficUpdate": TrafficUpdate, - "MotionRequest": MotionRequest, - "StreamingUpdate": StreamingUpdate, -} - - -class OSITrace: - """This class can import and decode OSI trace files.""" - - @staticmethod - def map_message_type(type_name): - """Map the type name to the protobuf message type.""" - return MESSAGES_TYPE[type_name] - - @staticmethod - def message_types(): - """Message types that OSITrace supports.""" - return list(MESSAGES_TYPE.keys()) - - def __init__(self, path=None, type_name="SensorView", cache_messages=False): - self.type = self.map_message_type(type_name) - self.file = None - self.current_index = None - self.message_offsets = None - self.read_complete = False - self.message_cache = {} if cache_messages else None - self._header_length = 4 - if path: - self.from_file(path, type_name, cache_messages) - - def from_file(self, path, type_name="SensorView", cache_messages=False): - """Import a trace from a file""" - self.type = self.map_message_type(type_name) - - if path.lower().endswith((".lzma", ".xz")): - self.file = lzma.open(path, "rb") - else: - self.file = open(path, "rb") - - self.read_complete = False - self.current_index = 0 - self.message_offsets = [0] - self.message_cache = {} if cache_messages else None - - def retrieve_offsets(self, limit=None): - """Retrieve the offsets of the messages from the file.""" - if not self.read_complete: - self.current_index = len(self.message_offsets) - 1 - self.file.seek(self.message_offsets[-1], 0) - while not self.read_complete and ( - not limit or len(self.message_offsets) <= limit - ): - self.retrieve_message(skip=True) - return self.message_offsets - - def retrieve_message(self, index=None, skip=False): - """Retrieve the next message from the file at the current position or given index, or skip it if skip is true.""" - if index is not None: - self.current_index = index - self.file.seek(self.message_offsets[index], 0) - if self.message_cache is not None and self.current_index in self.message_cache: - message = self.message_cache[self.current_index] - self.current_index += 1 - if self.current_index == len(self.message_offsets): - self.file.seek(0, 2) - else: - self.file.seek(self.message_offsets[self.current_index], 0) - if skip: - return self.message_offsets[self.current_index] - else: - return message - start = self.file.tell() - header = self.file.read(self._header_length) - if len(header) < self._header_length: - if start == self.message_offsets[-1]: - self.message_offsets.pop() - self.read_complete = True - self.file.seek(start, 0) - return None - message_length = struct.unpack("= len(self.message_offsets): - self.retrieve_offsets(index) - if self.message_cache is not None and index in self.message_cache: - return self.message_cache[index] - return self.retrieve_message(index=index) - - def get_messages(self): - """ - Yield an iterator over all messages in the file. - """ - return self.get_messages_in_index_range(0, None) - - def get_messages_in_index_range(self, begin, end): - """ - Yield an iterator over messages of indexes between begin and end included. - """ - if begin >= len(self.message_offsets): - self.retrieve_offsets(begin) - self.restart(begin) - current = begin - while end is None or current < end: - if self.message_cache is not None and current in self.message_cache: - yield self.message_cache[current] - else: - message = self.retrieve_message() - if message is None: - break - yield message - current += 1 - - def close(self): - if self.file: - self.file.close() - self.file = None - self.current_index = None - self.message_cache = None - self.message_offsets = None - self.read_complete = False - self.read_limit = None - self.type = None From ce81947b950f2761ca09a79c79a5395befeb53b5 Mon Sep 17 00:00:00 2001 From: "Pierre R. Mai" Date: Tue, 22 Jul 2025 17:01:16 +0200 Subject: [PATCH 33/33] Further workflow adjustments Signed-off-by: Pierre R. Mai --- .github/workflows/protobuf.yml | 29 +++++++++++++++++++---------- requirements_tests.txt | 1 - 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/protobuf.yml b/.github/workflows/protobuf.yml index b4718a086..0c323f8a3 100644 --- a/.github/workflows/protobuf.yml +++ b/.github/workflows/protobuf.yml @@ -1,4 +1,4 @@ -name: ProtoBuf CI Builds +name: OSI CI Builds on: push: @@ -8,7 +8,7 @@ on: jobs: spellcheck: name: Spellcheck - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: rojopolis/spellcheck-github-actions@0.36.0 @@ -17,15 +17,13 @@ jobs: config_path: .github/.pyspelling.yml run-tests: - name: Run tests and build docs + name: Run tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout OSI uses: actions/checkout@v4 - with: - submodules: true - name: Setup Python uses: actions/setup-python@v5 @@ -37,6 +35,18 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements_tests.txt + - name: Run Python Tests + run: python -m unittest discover tests + + build-docs: + name: Build docs + + runs-on: ubuntu-24.04 + + steps: + - name: Checkout OSI + uses: actions/checkout@v4 + - name: Install Doxygen run: sudo apt-get install doxygen graphviz @@ -69,11 +79,10 @@ jobs: working-directory: build run: cmake --build . --config Release -j 4 - - name: Run Python Tests - run: python -m unittest discover tests - - name: Upload documentation uses: actions/upload-artifact@v4 with: name: doxygen-doc - path: doc/ + path: | + doc/html/ + doc/images/ diff --git a/requirements_tests.txt b/requirements_tests.txt index 4872aecca..c3726e8bf 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -1,2 +1 @@ pyyaml -black==24.3.0