diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 05b2624b4037..2dcd3af7a52d 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -165,7 +165,7 @@ package object config { ConfigBuilder("spark.eventLog.compress") .version("1.0.0") .booleanConf - .createWithDefault(false) + .createWithDefault(true) private[spark] val EVENT_LOG_BLOCK_UPDATES = ConfigBuilder("spark.eventLog.logBlockUpdates.enabled") diff --git a/docs/configuration.md b/docs/configuration.md index 8fda9317bc77..e9ed2a8aa37d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1311,7 +1311,7 @@ Apart from these, the following properties are also available, and may be useful spark.eventLog.compress - false + true Whether to compress logged events, if spark.eventLog.enabled is true. diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 3f97a484e1a6..765c3494f669 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -22,6 +22,10 @@ license: | * Table of contents {:toc} +## Upgrading from Core 3.4 to 4.0 + +- Since Spark 4.0, Spark will compress event logs. To restore the behavior before Spark 4.0, you can set `spark.eventLog.compress` to `false`. + ## Upgrading from Core 3.3 to 3.4 - Since Spark 3.4, Spark driver will own `PersistentVolumnClaim`s and try to reuse if they are not assigned to live executors. To restore the behavior before Spark 3.4, you can set `spark.kubernetes.driver.ownPersistentVolumeClaim` to `false` and `spark.kubernetes.driver.reusePersistentVolumeClaim` to `false`.