Support OTLP transform for opentelemetry sink

[vinzee]

A note for the community

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Use Cases

First of all thank you for creating and maintaining such an amazing piece of software. Vector is a very crucial component for our log ingestion pipeline which ingests logs in a variety of formats such otlp, syslog, glog, ndjson, etc, that are then converted to a common vector format and sent to the destination as json. We now want to start writing in the otlp format which is now the new standard and is starting to be supported by more and more vendors.

When using the opentelemetry sink, we need to transform the logs from vector to otlp manually which becomes quite complex due to nested objects/arrays. I saw a few examples shared here, which seem super complex, and as they pointed out for_each does not support recursive iteration which makes it hard.

Is there a straightforward way to provide any way to support this for arbitrarily nested objects? Or are there any plans to support this out the box?

Thank you!

Attempted Solutions

We are currently using this VRL to convert log attributes in the vector format to OTLP format. But as you can see it cannot handle deeply nested objects/arrays very well and does a encode_json on them.

log_attributes = []
for_each(.attributes) -> |k, v| {
  if is_string(k) && !is_null(v) {
    if is_object(v) {
      # Convert object to kvlistValue with nested primitive values
      kv_values = []
      for_each(object!(v)) -> |obj_k, obj_v| {
        if is_string(obj_k) && !is_null(obj_v) {
          nested_val = if is_boolean(obj_v) {
            { "boolValue": obj_v }
          } else if is_integer(obj_v) {
            { "intValue": to_string!(obj_v) }
          } else if is_float(obj_v) {
            { "doubleValue": obj_v }
          } else if is_string(obj_v) {
            { "stringValue": obj_v }
          } else {
            { "stringValue": encode_json(obj_v) }
          }
          kv_values = push(kv_values, { "key": obj_k, "value": nested_val })
        }
      }
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "kvlistValue": { "values": kv_values } }
      })
    } else if is_array(v) {
      # Convert array to arrayValue with nested primitive values
      arr_values = []
      for_each(array!(v)) -> |_idx, item| {
        arr_val = if is_boolean(item) {
          { "boolValue": item }
        } else if is_integer(item) {
          { "intValue": to_string!(item) }
        } else if is_float(item) {
          { "doubleValue": item }
        } else if is_string(item) {
          { "stringValue": item }
        } else {
          { "stringValue": encode_json(item) }
        }
        arr_values = push(arr_values, arr_val)
      }
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "arrayValue": { "values": arr_values } }
      })
    } else if is_boolean(v) {
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "boolValue": v }
      })
    } else if is_integer(v) {
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "intValue": to_string!(v) }
      })
    } else if is_float(v) {
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "doubleValue": v }
      })
    } else {
      log_attributes = push(log_attributes, {
        "key": k,
        "value": { "stringValue": (to_string(v) ?? encode_json(v)) }
      })
    }
  }
}

Proposal

Maybe im shooting too high, but it would be really amazing to see a helper fn which converts objects/arrays to otlp format adhering to the rules mentioned here, like so:

to_otlp_attributes(<json_object>)

Note: this cannot be done today, as VRL, afaik, does not support closures.

# Define closures to convert arbitrary objects to OTLP format
# to_anyvalue recursively converts any value to its OTLP AnyValue representation.
# to_otlp_attributes converts a map to an array of OTLP key-value pairs.

to_anyvalue = |v| {
  if is_null(v) {
    null
  } else if is_boolean(v) {
    { "boolValue": v }
  } else if is_integer(v) {
    { "intValue": to_string!(v) }
  } else if is_float(v) {
    { "doubleValue": v }
  } else if is_string(v) {
    { "stringValue": v }
  } else if is_object(v) {
    # Recursively convert object to kvlistValue
    kv_values = []
    for_each(object!(v)) -> |obj_k, obj_v| {
      if is_string(obj_k) && !is_null(obj_v) {
        kv_values = push(kv_values, { "key": obj_k, "value": to_anyvalue(obj_v) })
      }
    }
    { "kvlistValue": { "values": kv_values } }
  } else if is_array(v) {
    # Recursively convert array to arrayValue
    arr_values = []
    for_each(array!(v)) -> |_idx, item| {
      if !is_null(item) {
        arr_values = push(arr_values, to_anyvalue(item))
      }
    }
    { "arrayValue": { "values": arr_values } }
  } else {
    { "stringValue": encode_json(v) }
  }
}

to_otlp_attributes = |input_map| {
  result = []
  for_each(object(input_map) ?? {}) -> |k, v| {
    if is_string(k) && !is_null(v) {
      result = push(result, { "key": k, "value": to_anyvalue(v) })
    }
  }
  result
}

References

No response

Version

No response

[thomasqueirozb]

Just checking, is the otlp codec not able to achieve this? https://vector.dev/docs/reference/configuration/sinks/opentelemetry/#protocol.encoding.codec