Skip to content
Prev Previous commit
Next Next commit
add comments for observed metrics
  • Loading branch information
WweiL committed Aug 21, 2023
commit 132b2c86da330802b3d45d5c5e2bac355658b29a
6 changes: 4 additions & 2 deletions python/pyspark/sql/streaming/listener.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,8 +487,10 @@ def fromJson(cls, j: Dict[str, Any]) -> "StreamingQueryProgress":
inputRowsPerSecond=j["inputRowsPerSecond"],
processedRowsPerSecond=j["processedRowsPerSecond"],
observedMetrics={
k: Row(row)
if isinstance(row, str)
# in test_parity_listener, observed metrics is serialized into string,
# this won't happen in production.
k: Row(row) # for test only,
if isinstance(row, str) # for test only,
else Row(*row.keys())(*row.values()) # Assume no nested rows
for k, row in j["observedMetrics"].items()
}
Expand Down