Adding First Version of Data Formats Library

- WIP: Pending Integration

tracing/data-formats/CMakeLists.txt

@@ -0,0 +1,28 @@
+cmake_minimum_required(VERSION 3.15.0...3.24.0)
+
+project(HERACLES_DATA_FORMATS VERSION 1.0.0)
+
+set(CMAKE_CXX_STANDARD 17)
+set(HERACLES_DATA_FORMATS_CMAKE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/heracles_data_formats")
+include(CMakePackageConfigHelpers)
+include(GNUInstallDirs)
+
+# Set default output directories
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}")
+
+include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake)
+include(${PROJECT_SOURCE_DIR}/cmake/protobuf.cmake)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -O3 -fPIC")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -O3 -fPIC")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_INSTALL_RPATH "$ORIGIN;$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_INSTALL_LIBDIR})
+
+find_package(OpenMP REQUIRED)
+
+option(BUILD_TEST "Build c++/python tests with ctest" ON)
+enable_testing()
+add_subdirectory(src)

tracing/data-formats/README.md

@@ -0,0 +1,104 @@
+# HERACLES Data formatter interface
+
+## CMake Configure and Build
+```bash
+cmake -S . -B build
+cmake --build build --parallel
+```
+_Note: for now cmake will _not build with `ninja`_ and is only tested for
+(default) `CMAKE_GENERATOR='Unix Makefiles`_
+
+
+## Run test
+```bash
+cmake --build build --target test
+```
+
+Note: Python ```protobuf==4.23.0``` module is required to run the Python test. It can be installed via
+```bash
+pip install -r requirements.txt
+```
+
+## C++
+
+### Importing the **HERACLES-Data-Formats** Library
+
+The C++ library be found and included with cmake by including
+following statements in the cmakefile of the project depending on the
+HERACLES data formats library:
+```cmake
+find_package(HERACLES_DATA_FORMATS 1.0.0 REQUIRED)
+...
+target_link_libraries(<YOUR_LIBRARY> PUBLIC HERACLES_DATA_FORMATS::heracles_data_formats)
+```
+Assuming you follow the convention of having all code
+checked out in the same directory and named by their component name, you
+can then build that project by executing the following:
+
+```bash
+# from project root
+HERACLES_DATA_FORMATS_DIR=$(pwd)/../HERACLES-data-formats/build cmake -S . -B build
+cmake --build build --parallel
+```
+Alternatively, you can also build and install HERACLES-data-formats
+(with the destination chosen, e.g., using the
+`-DCMAKE_INSTALL_PREFIX=/path/to/install` argument, and an `cmake
+--build build --target install` after the build ).  However,  when
+installing be careful in not forgetting to re-install
+after each change and subsequent build or accidentally picking up
+older versions installed elsewhere and earlier searched in CMAKE's
+search paths.
+
+
+### Usage example
+The library can be used in the ```C++``` code, e.g., as followed:
+```c++
+// protobuf headers
+#include "heracles/heracles_proto.h"
+// cpp utility headers
+#include "heracles/heracles_data_formats.h"
+
+int main() {
+  heracles::fhe_trace::Trace trace;
+  heracles::data::InputPolynomials input_polys;
+
+  return 0;
+}
+```
+Refer to the [heracles_test.cpp](src/data_formats/test/heracles_test.cpp) source
+code for additional examples of using Heracles protobuf objects and
+utility functions as well as [Protocol Buffer Basics:
+C++](https://protobuf.dev/getting-started/cpptutorial/) for more
+general information on using generated C++ protobuf code.
+
+
+## Python
+
+### Importing the **HERACLES-Data-Formats** Library
+
+For Python protobuf to work, first the Python ```protobuf``` module is required. It can be installed via
+```bash
+pip install -r requirements.txt
+# or
+pip install protobuf==4.23.4
+```
+
+The ```PYTHONPATH``` environment variable needs to be set to point to the protobuf generated files:
+```bash
+export PYTHONPATH=${HERACLES_DATA_FORMATS_DIR}/python/:${PYTHONPATH}
+```
+(with `HERACLES_DATA_FORMATS_DIR` as defined above for building
+dependent C++ projects and/or pointing to the install location in case
+you installed this project).
+
+### Usage example
+The **HERACLES-Data-Formats** library can be imported via, e.g.,
+```python
+from heracles.proto.common_pb2 import Scheme
+from heracles.proto.fhe_trace_pb2 import Trace, Instruction
+```
+Refer to the [heracles_test.py](src/data_formats/test/heracles_test.py) script for
+examples of using Heracles protobuf objects and utility functions as
+well as [Protocol Buffer Basics:
+Python](https://protobuf.dev/getting-started/pythontutorial/) for more
+general information on using generated python protobuf code.

tracing/data-formats/cmake/heracles_data_formats/HERACLES_DATA_FORMATSConfig.cmake.in

@@ -0,0 +1,25 @@
+# Copyright (C) 2025 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+@PACKAGE_INIT@
+
+include(CMakeFindDependencyMacro)
+
+include(${CMAKE_CURRENT_LIST_DIR}/HERACLES_DATA_FORMATSTargets.cmake)
+
+if(TARGET HERACLES_DATA_FORMATS::heracles_data_formats)
+  set(HERACLES_DATA_FORMATS_FOUND TRUE)
+  message(STATUS "Heracles Data Formats Library found")
+else()
+  message(STATUS "Heracles Data Formats Library not found")
+endif()
+
+# Requirement for protobuf
+find_package(ZLIB REQUIRED)
+
+set(HERACLES_DATA_FORMATS_VERSION "@HERACLES_DATA_FORMATS_VERSION")
+set(HERACLES_DATA_FORMATS_VERSION_MAJOR "@HERACLES_DATA_FORMATS_VERSION_MAJOR")
+set(HERACLES_DATA_FORMATS_VERSION_MINOR "@HERACLES_DATA_FORMATS_VERSION")
+set(HERACLES_DATA_FORMATS_VERSION_PATCH "@HERACLES_DATA_FORMATS_VERSION")
+
+set(HERACLES_DATA_FORMATS_DEBUG "@HERACLES_DATA_FORMATS_DEBUG")

tracing/data-formats/cmake/protobuf-generate.cmake

@@ -0,0 +1,155 @@
+function(protobuf_generate)
+  include(CMakeParseArguments)
+
+  set(_options APPEND_PATH)
+  set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR PLUGIN PLUGIN_OPTIONS DEPENDENCIES)
+  if(COMMAND target_sources)
+    list(APPEND _singleargs TARGET)
+  endif()
+  set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS PROTOC_OPTIONS)
+
+  cmake_parse_arguments(protobuf_generate "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}")
+
+  if(NOT protobuf_generate_PROTOS AND NOT protobuf_generate_TARGET)
+    message(SEND_ERROR "Error: protobuf_generate called without any targets or source files")
+    return()
+  endif()
+
+  if(NOT protobuf_generate_OUT_VAR AND NOT protobuf_generate_TARGET)
+    message(SEND_ERROR "Error: protobuf_generate called without a target or output variable")
+    return()
+  endif()
+
+  if(NOT protobuf_generate_LANGUAGE)
+    set(protobuf_generate_LANGUAGE cpp)
+  endif()
+  string(TOLOWER ${protobuf_generate_LANGUAGE} protobuf_generate_LANGUAGE)
+
+  if(NOT protobuf_generate_PROTOC_OUT_DIR)
+    set(protobuf_generate_PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
+  endif()
+
+  if(protobuf_generate_EXPORT_MACRO AND protobuf_generate_LANGUAGE STREQUAL cpp)
+    set(_dll_export_decl "dllexport_decl=${protobuf_generate_EXPORT_MACRO}")
+  endif()
+
+  foreach(_option ${_dll_export_decl} ${protobuf_generate_PLUGIN_OPTIONS})
+    # append comma - not using CMake lists and string replacement as users
+    # might have semicolons in options
+    if(_plugin_options)
+      set( _plugin_options "${_plugin_options},")
+    endif()
+    set(_plugin_options "${_plugin_options}${_option}")
+  endforeach()
+
+  if(protobuf_generate_PLUGIN)
+      set(_plugin "--plugin=${protobuf_generate_PLUGIN}")
+  endif()
+
+  if(NOT protobuf_generate_GENERATE_EXTENSIONS)
+    if(protobuf_generate_LANGUAGE STREQUAL cpp)
+      set(protobuf_generate_GENERATE_EXTENSIONS .pb.h .pb.cc)
+    elseif(protobuf_generate_LANGUAGE STREQUAL python)
+      set(protobuf_generate_GENERATE_EXTENSIONS _pb2.py)
+    else()
+      message(SEND_ERROR "Error: protobuf_generate given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS")
+      return()
+    endif()
+  endif()
+
+  if(protobuf_generate_TARGET)
+    get_target_property(_source_list ${protobuf_generate_TARGET} SOURCES)
+    foreach(_file ${_source_list})
+      if(_file MATCHES "proto$")
+        list(APPEND protobuf_generate_PROTOS ${_file})
+      endif()
+    endforeach()
+  endif()
+
+  if(NOT protobuf_generate_PROTOS)
+    message(SEND_ERROR "Error: protobuf_generate could not find any .proto files")
+    return()
+  endif()
+
+  if(protobuf_generate_APPEND_PATH)
+    # Create an include path for each file specified
+    foreach(_file ${protobuf_generate_PROTOS})
+      get_filename_component(_abs_file ${_file} ABSOLUTE)
+      get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
+      list(FIND _protobuf_include_path ${_abs_dir} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${_abs_dir})
+      endif()
+    endforeach()
+  endif()
+
+  foreach(DIR ${protobuf_generate_IMPORT_DIRS})
+    get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+    list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+    if(${_contains_already} EQUAL -1)
+        list(APPEND _protobuf_include_path -I ${ABS_PATH})
+    endif()
+  endforeach()
+
+  if(NOT _protobuf_include_path)
+    set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
+  endif()
+
+  set(_generated_srcs_all)
+  foreach(_proto ${protobuf_generate_PROTOS})
+    get_filename_component(_abs_file ${_proto} ABSOLUTE)
+    get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
+
+    get_filename_component(_file_full_name ${_proto} NAME)
+    string(FIND "${_file_full_name}" "." _file_last_ext_pos REVERSE)
+    string(SUBSTRING "${_file_full_name}" 0 ${_file_last_ext_pos} _basename)
+
+    set(_suitable_include_found FALSE)
+    foreach(DIR ${_protobuf_include_path})
+      if(NOT DIR STREQUAL "-I")
+        file(RELATIVE_PATH _rel_dir ${DIR} ${_abs_dir})
+        string(FIND "${_rel_dir}" "../" _is_in_parent_folder)
+        if (NOT ${_is_in_parent_folder} EQUAL 0)
+          set(_suitable_include_found TRUE)
+          break()
+        endif()
+      endif()
+    endforeach()
+
+    if(NOT _suitable_include_found)
+      message(SEND_ERROR "Error: protobuf_generate could not find any correct proto include directory.")
+      return()
+    endif()
+
+    set(_generated_srcs)
+    foreach(_ext ${protobuf_generate_GENERATE_EXTENSIONS})
+      list(APPEND _generated_srcs "${protobuf_generate_PROTOC_OUT_DIR}/${_rel_dir}/${_basename}${_ext}")
+    endforeach()
+    list(APPEND _generated_srcs_all ${_generated_srcs})
+
+    set(_comment "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}")
+    if(protobuf_generate_PROTOC_OPTIONS)
+      set(_comment "${_comment}, protoc-options: ${protobuf_generate_PROTOC_OPTIONS}")
+    endif()
+    if(_plugin_options)
+      set(_comment "${_comment}, plugin-options: ${_plugin_options}")
+    endif()
+
+    add_custom_command(
+      OUTPUT ${_generated_srcs}
+      COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=${protobuf_LIB_DIR}" ${protobuf_PROTOC_EXECUTABLE}#protobuf_executable#protobuf::protoc
+      ARGS ${protobuf_generate_PROTOC_OPTIONS} --${protobuf_generate_LANGUAGE}_out ${_plugin_options}:${protobuf_generate_PROTOC_OUT_DIR} ${_plugin} ${_protobuf_include_path} ${_abs_file}
+      DEPENDS ${_abs_file} ${protobuf_PROTOC_EXE} ${protobuf_generate_DEPENDENCIES} protobuf_executable
+      COMMENT ${_comment}
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE)
+  if(protobuf_generate_OUT_VAR)
+    set(${protobuf_generate_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE)
+  endif()
+  if(protobuf_generate_TARGET)
+    target_sources(${protobuf_generate_TARGET} PRIVATE ${_generated_srcs_all})
+  endif()
+
+endfunction()

tracing/data-formats/cmake/protobuf.cmake

@@ -0,0 +1,73 @@
+# Recent release version
+set(PROTOBUF_EXT_GIT_TAG v4.23.4)
+set(PROTOBUF_EXT_GIT_URL https://github.com/protocolbuffers/protobuf.git)
+set(PROTOBUF_EXT_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/ext_protobuf)
+set(PROTOBUF_EXT_DESTDIR ${PROTOBUF_EXT_PREFIX}/protobuf_install)
+
+include(ExternalProject)
+ExternalProject_Add(
+  ext_protobuf
+  GIT_REPOSITORY ${PROTOBUF_EXT_GIT_URL}
+  GIT_TAG ${PROTOBUF_EXT_GIT_TAG}
+  PREFIX ${PROTOBUF_EXT_PREFIX}
+  CMAKE_ARGS ${CMAKE_CXX_FLAGS}
+             -DCMAKE_INSTALL_PREFIX=${PROTOBUF_EXT_DESTDIR}
+             -Dprotobuf_BUILD_EXAMPLES=OFF
+             -Dprotobuf_BUILD_TESTS=OFF
+             -DABSL_PROPAGATE_CXX_STD=ON
+             -DCMAKE_INSTALL_LIBDIR=lib
+             -DCMAKE_BUILD_TYPE=Release
+             -DCMAKE_INSTALL_RPATH=$ORIGIN
+             -Dprotobuf_BUILD_SHARED_LIBS=ON
+  UPDATE_COMMAND ""
+  INSTALL_COMMAND make install
+)
+
+
+set(protobuf_SOURCE_DIR ${PROTOBUF_EXT_PREFIX}/src/ext_protobuf)
+set(protobuf_INCLUDE_DIR ${PROTOBUF_EXT_DESTDIR}/include)
+set(protobuf_LIB_DIR ${PROTOBUF_EXT_DESTDIR}/lib)
+set(protobuf_BIN_DIR ${PROTOBUF_EXT_DESTDIR}/bin)
+
+# setup protobuf executable
+add_executable(protobuf_executable IMPORTED GLOBAL)
+add_dependencies(protobuf_executable ext_protobuf)
+set_target_properties(protobuf_executable PROPERTIES
+  IMPORTED_LOCATION ${protobuf_BIN_DIR}/protoc
+)
+set(protobuf_PROTOC_EXECUTABLE ${protobuf_BIN_DIR}/protoc)
+
+foreach(_protobuf_lib_name ${protobuf_SHARED_LIB_NAMES})
+  set(_protobuf_lib_filename_shared "lib${_protobuf_lib_name}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  add_library(${_protobuf_lib_name} SHARED IMPORTED GLOBAL)
+  add_dependencies(${_protobuf_lib_name} ext_protobuf)
+  set_target_properties(${_protobuf_lib_name} PROPERTIES
+    IMPORTED_LOCATION ${protobuf_LIB_DIR}/${_protobuf_lib_filename_shared}
+    INCLUDE_DIRECTORIES ${protobuf_INCLUDE_DIR}
+  )
+endforeach()
+
+foreach(_protobuf_lib_name ${protobuf_STATIC_LIB_NAMES})
+  set(_protobuf_lib_filename_static "lib${_protobuf_lib_name}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  add_library(${_protobuf_lib_name} STATIC IMPORTED GLOBAL)
+  add_dependencies(${_protobuf_lib_name} ext_protobuf)
+  set_target_properties(${_protobuf_lib_name} PROPERTIES
+    IMPORTED_LOCATION ${protobuf_LIB_DIR}/${_protobuf_lib_filename_static}
+    INCLUDE_DIRECTORIES ${protobuf_INCLUDE_DIR}
+  )
+endforeach()
+
+install(DIRECTORY ${protobuf_INCLUDE_DIR}/
+  DESTINATION include
+)
+
+# copy library and binary files when installing
+install(DIRECTORY ${protobuf_LIB_DIR}/
+  DESTINATION lib
+  USE_SOURCE_PERMISSIONS
+)
+
+install(DIRECTORY ${protobuf_BIN_DIR}/
+  DESTINATION bin
+  USE_SOURCE_PERMISSIONS
+)