From e3ed2af75f56fb68c227df59cffb46a3e07ef74f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Tue, 5 Aug 2025 16:49:56 +0200 Subject: [PATCH 1/2] [SPARK-53128][CORE] Include unmanaged memory bytes in the usage log before execution memory OOM --- .../java/org/apache/spark/memory/TaskMemoryManager.java | 7 +++++-- .../org/apache/spark/memory/UnifiedMemoryManager.scala | 8 ++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index ab9e470e0c2c..20f26abce837 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -339,9 +339,12 @@ public void showMemoryUsage() { MDC.of(LogKeys.MEMORY_SIZE, memoryNotAccountedFor), MDC.of(LogKeys.TASK_ATTEMPT_ID, taskAttemptId)); logger.info( - "{} bytes of memory are used for execution and {} bytes of memory are used for storage", + "{} bytes of memory are used for execution " + + "and {} bytes of memory are used for storage " + + "and {} bytes of memory are used but unmanaged", MDC.of(LogKeys.EXECUTION_MEMORY_SIZE, memoryManager.executionMemoryUsed()), - MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed())); + MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed()), + MDC.of(LogKeys.MEMORY_SIZE, UnifiedMemoryManager$.MODULE$.getUnmanagedMemoryUsed())); } } diff --git a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala index db51f14415e1..6b278c47f32f 100644 --- a/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/UnifiedMemoryManager.scala @@ -273,6 +273,14 @@ object UnifiedMemoryManager extends Logging { // Atomic flag to ensure polling is only started once per JVM private val pollingStarted = new AtomicBoolean(false) + /** + * Returns the total unmanaged memory in bytes, including both + * on-heap unmanaged memory and off-heap unmanaged memory. + */ + private[spark] def getUnmanagedMemoryUsed: Long = { + UnifiedMemoryManager.unmanagedOnHeapUsed.get() + UnifiedMemoryManager.unmanagedOffHeapUsed.get() + } + /** * Register an unmanaged memory consumer to track its memory usage. * From 25d34b7c5d4e6456d30aa7b56021dd294c457b6f Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Mon, 3 Nov 2025 17:07:28 +0100 Subject: [PATCH 2/2] fixup --- .../main/java/org/apache/spark/memory/TaskMemoryManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java index 20f26abce837..8b41df6b269f 100644 --- a/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java +++ b/core/src/main/java/org/apache/spark/memory/TaskMemoryManager.java @@ -341,7 +341,7 @@ public void showMemoryUsage() { logger.info( "{} bytes of memory are used for execution " + "and {} bytes of memory are used for storage " + - "and {} bytes of memory are used but unmanaged", + "and {} bytes of unmanaged memory are used", MDC.of(LogKeys.EXECUTION_MEMORY_SIZE, memoryManager.executionMemoryUsed()), MDC.of(LogKeys.STORAGE_MEMORY_SIZE, memoryManager.storageMemoryUsed()), MDC.of(LogKeys.MEMORY_SIZE, UnifiedMemoryManager$.MODULE$.getUnmanagedMemoryUsed()));