From 4a6c746da8f6318b0d77570d175345a4da0dfd03 Mon Sep 17 00:00:00 2001 From: beliefer Date: Thu, 18 Sep 2025 10:44:14 +0800 Subject: [PATCH 1/2] [GLUTEN-10707] Improve SparkDirectoryUtil by reduce synchronized --- .../spark/util/SparkDirectoryUtil.scala | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala index 24ce3e929089..ddc78d9f37fb 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala @@ -67,30 +67,31 @@ class SparkDirectoryUtil private (val roots: Array[String]) extends Logging { } object SparkDirectoryUtil extends Logging { - private var INSTANCE: SparkDirectoryUtil = _ + @volatile private var targetRoots: Array[String] = _ + private lazy val INSTANCE: SparkDirectoryUtil = { + if (targetRoots == null) { + throw new IllegalStateException("SparkDirectoryUtil not initialized") + } + new SparkDirectoryUtil(targetRoots) + } - def init(conf: SparkConf): Unit = synchronized { + def init(conf: SparkConf): Unit = { val roots = Utils.getConfiguredLocalDirs(conf) init(roots) } private def init(roots: Array[String]): Unit = synchronized { - if (INSTANCE == null) { - INSTANCE = new SparkDirectoryUtil(roots) - return - } - if (INSTANCE.roots.toSet != roots.toSet) { + if (targetRoots == null) { + targetRoots = roots + } else if (targetRoots.toSet != roots.toSet) { throw new IllegalArgumentException( - s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${INSTANCE.ROOTS + s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${targetRoots .mkString("Array(", ", ", ")")}, new: ${roots.mkString("Array(", ", ", ")")}" ) } } - def get(): SparkDirectoryUtil = synchronized { - assert(INSTANCE != null, "Default instance of SparkDirectoryUtil was not set yet") - INSTANCE - } + def get(): SparkDirectoryUtil = INSTANCE } class Namespace(private val parents: Array[File], private val name: String) { From bbcc3e48a2534621a841d3ae62df94650b195d8b Mon Sep 17 00:00:00 2001 From: beliefer Date: Sat, 11 Oct 2025 10:38:57 +0800 Subject: [PATCH 2/2] Replace targetRoots with roots --- .../org/apache/spark/util/SparkDirectoryUtil.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala index ddc78d9f37fb..3ce41dc7f1ca 100644 --- a/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala +++ b/gluten-core/src/main/scala/org/apache/spark/util/SparkDirectoryUtil.scala @@ -67,12 +67,12 @@ class SparkDirectoryUtil private (val roots: Array[String]) extends Logging { } object SparkDirectoryUtil extends Logging { - @volatile private var targetRoots: Array[String] = _ + @volatile private var roots: Array[String] = _ private lazy val INSTANCE: SparkDirectoryUtil = { - if (targetRoots == null) { + if (this.roots == null) { throw new IllegalStateException("SparkDirectoryUtil not initialized") } - new SparkDirectoryUtil(targetRoots) + new SparkDirectoryUtil(this.roots) } def init(conf: SparkConf): Unit = { @@ -81,11 +81,11 @@ object SparkDirectoryUtil extends Logging { } private def init(roots: Array[String]): Unit = synchronized { - if (targetRoots == null) { - targetRoots = roots - } else if (targetRoots.toSet != roots.toSet) { + if (this.roots == null) { + this.roots = roots + } else if (this.roots.toSet != roots.toSet) { throw new IllegalArgumentException( - s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${targetRoots + s"Reinitialize SparkDirectoryUtil with different root dirs: old: ${this.roots .mkString("Array(", ", ", ")")}, new: ${roots.mkString("Array(", ", ", ")")}" ) }