Guest tracing improvements to use tracing crate

.github/workflows/CargoPublish.yml

@@ -45,7 +45,7 @@ jobs:
             VERSION="${VERSION#refs/heads/release/v}"
             echo "VERSION=$VERSION" >> $GITHUB_ENV
           fi
-          ./dev/verify-version.sh "$VERSION" hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing-macro hyperlight-guest-tracing
+          ./dev/verify-version.sh "$VERSION" hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing
 
       - name: Determine which crates need publishing
         run: |
@@ -74,7 +74,6 @@ jobs:
           needs_publish hyperlight-component-util
           needs_publish hyperlight-component-macro
           needs_publish hyperlight-host
-          needs_publish hyperlight-guest-tracing-macro
           needs_publish hyperlight-guest-tracing
 
       - name: Publish hyperlight-common
@@ -84,13 +83,6 @@ jobs:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_PUBLISH_TOKEN }}
         if: env.PUBLISH_HYPERLIGHT_COMMON != 'false'
 
-      - name: Publish hyperlight-guest-tracing-macro
-        continue-on-error: ${{ inputs.dry_run }}
-        run: cargo publish --manifest-path ./src/hyperlight_guest_tracing_macro/Cargo.toml ${{ inputs.dry_run && '--dry-run' || '' }}
-        env:
-          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_PUBLISH_TOKEN }}
-        if: env.PUBLISH_HYPERLIGHT_GUEST_TRACING_MACRO != 'false'
-
       - name: Publish hyperlight-guest-tracing
         continue-on-error: ${{ inputs.dry_run }}
         run: cargo publish --manifest-path ./src/hyperlight_guest_tracing/Cargo.toml ${{ inputs.dry_run && '--dry-run' || '' }}

Cargo.toml

@@ -11,7 +11,6 @@ members = [
     "src/hyperlight_host",
     "src/hyperlight_guest_capi",
     "src/hyperlight_guest_tracing",
-    "src/hyperlight_guest_tracing_macro",
     "src/hyperlight_testing",
     "fuzz",
     "src/hyperlight_guest_bin",
@@ -42,7 +41,6 @@ hyperlight-guest = { path = "src/hyperlight_guest", version = "0.9.0", default-f
 hyperlight-guest-bin = { path = "src/hyperlight_guest_bin", version = "0.9.0", default-features = false }
 hyperlight-testing = { path = "src/hyperlight_testing", default-features = false }
 hyperlight-guest-tracing = { path = "src/hyperlight_guest_tracing", version = "0.9.0", default-features = false }
-hyperlight-guest-tracing-macro = { path = "src/hyperlight_guest_tracing_macro", version = "0.9.0", default-features = false }
 hyperlight-component-util = { path = "src/hyperlight_component_util", version = "0.9.0", default-features = false }
 hyperlight-component-macro = { path = "src/hyperlight_component_macro", version = "0.9.0", default-features = false }
 

Justfile

@@ -110,7 +110,7 @@ like-ci config=default-target hypervisor="kvm":
     {{ if os() == "linux" { "just clippy-exhaustive " + config } else { "" } }}
 
     @# Verify MSRV
-    ./dev/verify-msrv.sh hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing-macro hyperlight-guest-tracing
+    ./dev/verify-msrv.sh hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing
 
     @# Build and move Rust guests
     just build-rust-guests {{config}}
@@ -200,22 +200,10 @@ test-rust-crashdump target=default-target features="":
 test-rust-tracing target=default-target features="":
     # Run tests for the tracing guest and macro
     {{ cargo-cmd }} test -p hyperlight-guest-tracing --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }}
-    {{ cargo-cmd }} test -p hyperlight-guest-tracing-macro --profile={{ if target == "debug" { "dev" } else { target } }}  {{ target-triple-flag }}
 
-    # Prepare the tracing guest for testing
+    # Build the tracing guest to ensure it builds with the tracing feature
     just build-rust-guests {{ target }} trace_guest
     just move-rust-guests {{ target }}
-    # Run hello-world example with tracing enabled to get the trace output
-    # note that trace-dump doesn't run on MUSL target as of now
-    TRACE_OUTPUT="$({{ cargo-cmd }} run --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --example hello-world --features {{ if features =="" {"trace_guest"} else { "trace_guest," + features } }})" && \
-        TRACE_FILE="$(echo "$TRACE_OUTPUT" | grep -oE 'Creating trace file at: [^ ]+' | awk -F': ' '{print $2}')" && \
-        TRACE_FILE="$(echo "$TRACE_OUTPUT" | grep -oE 'Creating trace file at: [^ ]+' | awk -F': ' '{print $2}' | sed -E 's|^(trace/[^ ]+\.trace)$|./\1|; s|.*/(trace/[^ ]+\.trace)$|./\1|')" && \
-        echo "$TRACE_OUTPUT" && \
-        if [ -z "$TRACE_FILE" ]; then \
-            echo "Error: Could not extract trace file path from output." >&2 ; \
-            exit 1 ; \
-        fi && \
-        cargo run -p trace_dump ./{{ simpleguest_source }}/{{ target }}/simpleguest "$TRACE_FILE" list_frames
 
     # Rebuild the tracing guests without the tracing feature
     # This is to ensure that the tracing feature does not affect the other tests
@@ -234,7 +222,7 @@ check:
     {{ cargo-cmd }} check -p hyperlight-host --features crashdump  {{ target-triple-flag }}
     {{ cargo-cmd }} check -p hyperlight-host --features print_debug  {{ target-triple-flag }}
     {{ cargo-cmd }} check -p hyperlight-host --features gdb  {{ target-triple-flag }}
-    {{ cargo-cmd }} check -p hyperlight-host --features trace_guest,unwind_guest,mem_profile  {{ target-triple-flag }}
+    {{ cargo-cmd }} check -p hyperlight-host --features trace_guest,mem_profile  {{ target-triple-flag }}
 
 fmt-check:
     cargo +nightly fmt --all -- --check
@@ -275,7 +263,6 @@ clippy-exhaustive target=default-target: (witguest-wit)
     ./hack/clippy-package-features.sh hyperlight-testing {{ target }} {{ target-triple }}
     ./hack/clippy-package-features.sh hyperlight-component-macro  {{ target }} {{ target-triple }}
     ./hack/clippy-package-features.sh hyperlight-component-util {{ target }} {{ target-triple }}
-    ./hack/clippy-package-features.sh hyperlight-guest-tracing-macro {{ target }}
     ./hack/clippy-package-features.sh hyperlight-guest-tracing {{ target }}
     just clippy-guests {{ target }}
 
@@ -285,7 +272,7 @@ clippy-package package target=default-target: (witguest-wit)
 
 # Verify Minimum Supported Rust Version
 verify-msrv:
-    ./dev/verify-msrv.sh hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing-macro hyperlight-guest-tracing
+    ./dev/verify-msrv.sh hyperlight-common hyperlight-guest hyperlight-guest-bin hyperlight-host hyperlight-component-util hyperlight-component-macro hyperlight-guest-tracing
 
 #####################
 ### RUST EXAMPLES ###

docs/hyperlight-metrics-logs-and-traces.md

@@ -94,12 +94,11 @@ Once the container or the exe is running, the trace output can be viewed in the
 
 ## Guest Tracing, Unwinding, and Memory Profiling
 
-Hyperlight provides advanced observability features for guest code running inside micro virtual machines. You can enable guest-side tracing, stack unwinding, and memory profiling using the `trace_guest`, `unwind_guest`, and `mem_profile` features. This section explains how to build, run, and inspect guest traces.
+Hyperlight provides advanced observability features for guest code running inside micro virtual machines. You can enable guest-side tracing, stack unwinding, and memory profiling using the `trace_guest` and `mem_profile` features. This section explains how to build, run, and inspect guest traces.
 
 The following features are available for guest tracing:
 - `trace_guest`: Enables tracing for guest code, capturing function calls and execution time.
-- `unwind_guest`: Enables stack unwinding for guest code, allowing you to capture stack traces.
-- `mem_profile`: Enables memory profiling for guest code, capturing memory allocations and usage.
+- `mem_profile`: Enables memory profiling for guest code with stack unwinding, capturing memory allocations and usage.
 
 ### Building a Guest with Tracing Support
 
@@ -110,19 +109,57 @@ just build-rust-guests debug trace_guest
 just move-rust-guests debug
 ```
 
-This will build the guest binaries with the `trace_guest` feature enabled and move them to the appropriate location for use by the host.
+This builds the guest binaries with the `trace_guest` feature enabled and move them to the appropriate location for use by the host.
+
+**NOTE**: To enable the tracing in your application you need to use the `trace_guest` feature on the `hyperlight-guest-bin` and `hyperlight-guest` crates.
 
 ### Running a Hyperlight Example with Guest Tracing
 
 Once the guest is built, you can run a Hyperlight example with guest tracing enabled. For example:
 
 ```bash
-cargo run --example hello-world --features trace_guest
+RUST_LOG="info,hyperlight_host::sandbox=info,hyperlight_guest=trace,hyperlight_guest_bin=trace" cargo run --example tracing-otlp --features trace_guest
 ```
 
-This will execute the `hello-world` example, loading the guest with tracing enabled. During execution, trace data will be collected and written to a file in the `trace` directory.
+This will execute the `tracing-otlp` example, loading the guest with tracing enabled.
+During execution, trace data will be collected on the host and exported as `opentelemetry` spans/events.
+
+You can set up a collector to gather all the traces and inspect the traces from both host and guests.
+
+Due to the nature of execution inside a Sandbox, on a call basis, the guest tracing sets up a stack of spans to keep track of the correct parents for the incoming
+guest spans.
+We start with `call-to-guest` which contains all the spans coming from a guest. Additionally, for each exit into the host, we add another layer marking it with
+a `call-to-host` span to follow the execution in the host and correctly set it as a child of the active span in the guest.
+This logic simulates the propagation of `Opentelemetry` context that is usually done between two services, but cannot be done here seamlessly because the guest side
+runs `no_std` which `opentelemetry` doesn't know.
+
+#### How it works
+
+##### Guest
+
+When the guest starts executing the `entrypoint` function, it receives a `max_log_level` parameter that tells the guest what kind of logging level is expected from it.
+
+The `trace_guest` logic takes advantage of this parameter and when the `max_log_level` is `trace`, it allocates a custom made `GuestSubscriber` that implements the `Subscriber`
+trait from `tracing_core` that allows defining a subscriber for the `tracing` crate to handle new spans and events.
+
+This custom subscriber stores the spans and events in a buffer initialized only when tracing is enabled. For each new span and event, a method is called on the custom subscriber which
+not only stores the data, but also keeps track of the hierarchy and dependencies between the other spans/events.
+
+When the storage space is filled, the guest triggers a VM Exit that sends the guest pointers to the host. The host can access the guest memory, get the data and parse it to create the `spans` and `events` using the `opentelemetry` crate which allows specifying the starting and ending timestamps
+which are captured in the guest using the `TSC`.
+
+To improve performance, for each VMExit, the guest adds metadata for the host to be able to report the tracing data and free space.
+
+##### Host
+
+When a guest exits, the host checks for metadata from the guest reporting tracing data.
+If tracing data is found, the host starts parsing it and reconstructing a tree which represents the spans hierarchy.
+
+Additionally, the host also adds new children `span`s to the guest's reported active span, emphasizing the spans created on the host as a result of a temporary VM Exit. This helps visualize a call into the guest with context propagated across the VM boundary.
+
+The host creates `opentelemetry` spans and events for each guest span and event reported.
 
-### Inspecting Guest Trace Files
+### Inspecting Guest memory Trace Files (for mem_profile)
 
 To inspect the trace file generated by the guest, use the `trace_dump` crate. You will need the path to the guest symbols and the trace file. Run the following command:
 
@@ -140,7 +177,7 @@ This command will list the stack frames and tracing information captured during
 cargo run -p trace_dump ./src/tests/rust_guests/bin/debug/simpleguest ./trace/<UUID>.trace list_frames
 ```
 
-You can use additional features such as `unwind_guest` and `mem_profile` by enabling them during the build and run steps.
+You can use the `mem_profile` additional feature by enabling them during the build and run steps.
 
 > **Note:** Make sure to follow the build and run steps in order, and ensure that the guest binaries are up to date before running the host example.
 

src/hyperlight_common/Cargo.toml

@@ -26,7 +26,6 @@ spin = "0.10.0"
 default = ["tracing"]
 fuzzing = ["dep:arbitrary"]
 trace_guest = []
-unwind_guest = []
 mem_profile = []
 std = []