diff --git a/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader b/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
new file mode 100644
index 000000000000..5e0a0fe939bf
--- /dev/null
+++ b/backends-velox/src/main/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gluten.spi.SharedLibraryLoaderCentos7
+org.apache.gluten.spi.SharedLibraryLoaderCentos8
+org.apache.gluten.spi.SharedLibraryLoaderCentos9
+org.apache.gluten.spi.SharedLibraryLoaderDebian11
+org.apache.gluten.spi.SharedLibraryLoaderDebian12
+org.apache.gluten.spi.SharedLibraryLoaderMacOS
+org.apache.gluten.spi.SharedLibraryLoaderOpenEuler2403
+org.apache.gluten.spi.SharedLibraryLoaderUbuntu2004
+org.apache.gluten.spi.SharedLibraryLoaderUbuntu2204
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 5468ad2c56a3..3488ef56533a 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -210,7 +210,7 @@ class VeloxListenerApi extends ListenerApi with Logging {
     val loader = JniWorkspace.getDefault.libLoader
 
     // Load shared native libraries the backend libraries depend on.
-    SharedLibraryLoader.load(conf, loader)
+    SharedLibraryLoaderUtils.load(conf, loader)
 
     // Load backend libraries.
     val libPath = conf.get(GlutenConfig.GLUTEN_LIB_PATH)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala
new file mode 100755
index 000000000000..7841e67ed495
--- /dev/null
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoader.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.spi
+
+import org.apache.gluten.jni.JniLibLoader
+
+/**
+ * :: DeveloperApi ::
+ *
+ * Interface for loading shared libraries based on the operating system name and version.
+ */
+trait SharedLibraryLoader {
+
+  /**
+   * Check if this loader can load libraries for the given OS name and version.
+   *
+   * @param osName
+   *   OS name
+   * @param osVersion
+   *   OS version
+   * @return
+   *   true if this loader can load libraries for the given OS name and version, false otherwise
+   */
+  def accepts(osName: String, osVersion: String): Boolean
+
+  /**
+   * Load the required shared libraries using the given JniLibLoader.
+   *
+   * @param loader
+   *   JniLibLoader to load the shared libraries
+   */
+  def loadLib(loader: JniLibLoader): Unit
+}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
old mode 100755
new mode 100644
similarity index 80%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
index 0ae9bbd5eb4e..2ee087d1853e
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
@@ -14,11 +14,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    (osName.contains("CentOS") && osVersion.startsWith("7")) ||
+    (osName.contains("Oracle") && osVersion.startsWith("7")) ||
+    (osName.contains("Anolis") && osVersion.startsWith("7")) ||
+    (osName.contains("Red Hat") && osVersion.startsWith("7")) ||
+    (osName.contains("Alibaba Cloud Linux") && osVersion.startsWith("2")) ||
+    (osName.contains("tencentos") && osVersion.contains("2.4"))
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
     loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
@@ -37,4 +46,5 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
     loader.loadAndCreateLink("liblz4.so.1", "liblz4.so")
     loader.loadAndCreateLink("libgeos.so.3.10.7", "libgeos.so")
   }
+
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
similarity index 82%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
index f0882ea8f19f..854777d774b1 100755
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
@@ -14,11 +14,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    (osName.contains("CentOS") && osVersion.startsWith("8")) ||
+    (osName.contains("Oracle") && osVersion.startsWith("8")) ||
+    (osName.contains("Anolis") && osVersion.startsWith("8")) ||
+    (osName.contains("Red Hat") && osVersion.startsWith("8")) ||
+    (osName.contains("Alibaba Cloud Linux") && osVersion.startsWith("3")) ||
+    (osName.contains("tencentos") && osVersion.contains("3.2"))
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
     loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
similarity index 91%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
index dea336cb367e..f8268066c7b6 100755
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
@@ -14,11 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("CentOS") && osVersion.startsWith("9") ||
+    osName.contains("Red Hat") && osVersion.startsWith("9")
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
     loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
similarity index 93%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
index 611ca9091fda..9e51ddd34aea 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
@@ -14,11 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("Debian") && osVersion.startsWith("11")
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libicudata.so.67", "libicudata.so")
     loader.loadAndCreateLink("libre2.so.9", "libre2.so")
@@ -45,4 +50,5 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
     loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
   }
+
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
similarity index 94%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
index 9f7650ed1f95..d6b36ae9f3c3 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
@@ -14,11 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("Debian") && osVersion.startsWith("12")
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so")
     loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so")
@@ -51,4 +56,5 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
     loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
   }
+
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala
new file mode 100644
index 000000000000..03059579051c
--- /dev/null
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderMacOS.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.spi
+import org.apache.gluten.jni.JniLibLoader
+
+class SharedLibraryLoaderMacOS extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.startsWith("Mac OS X") || osName.startsWith("macOS")
+  }
+
+  override def loadLib(loader: JniLibLoader): Unit = {
+    // Placeholder for loading shared libs on MacOS if user needs.
+  }
+}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
similarity index 93%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
index ebf5c95a7adb..ca0db50d4844 100755
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderOpenEuler2403.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
@@ -14,11 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderOpenEuler2403 extends SharedLibraryLoader {
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("openEuler") && osVersion.startsWith("24.03")
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
     loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
@@ -42,4 +46,5 @@ class SharedLibraryLoaderOpenEuler2403 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libre2.so.11", "libre2.so")
     loader.loadAndCreateLink("libsodium.so.26", "libsodium.so")
   }
+
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
similarity index 95%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
index 6f463f34274e..7aaa07853e92 100755
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2004.scala
@@ -14,11 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("Ubuntu") && osVersion.startsWith("20.04")
+  }
+
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libroken.so.18", "libroken.so")
     loader.loadAndCreateLink("libasn1.so.8", "libasn1.so")
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
similarity index 93%
rename from backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
rename to backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
index 029ddf45fa7e..a0535f188b68 100755
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
@@ -14,11 +14,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.gluten.utils
+package org.apache.gluten.spi
 
 import org.apache.gluten.jni.JniLibLoader
 
 class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
+
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName.contains("Ubuntu") && osVersion.startsWith("22.04")
+  }
   override def loadLib(loader: JniLibLoader): Unit = {
     loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so")
     loader.loadAndCreateLink("libicudata.so.70", "libicudata.so")
@@ -46,4 +50,5 @@ class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
     loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
     loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so")
   }
+
 }
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
deleted file mode 100755
index cb0c9cd89136..000000000000
--- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoader.scala
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gluten.utils
-
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.exception.GlutenException
-import org.apache.gluten.jni.JniLibLoader
-
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.internal.SparkConfigUtil._
-
-import scala.sys.process._
-
-trait SharedLibraryLoader {
-  def loadLib(loader: JniLibLoader): Unit
-}
-
-object SharedLibraryLoader {
-  def load(conf: SparkConf, jni: JniLibLoader): Unit = {
-    val shouldLoad = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_FROM_JAR)
-    if (!shouldLoad) {
-      return
-    }
-    val osName = System.getProperty("os.name")
-    if (osName.startsWith("Mac OS X") || osName.startsWith("macOS")) {
-      loadLibWithMacOS(jni)
-    } else {
-      loadLibWithLinux(conf, jni)
-    }
-  }
-
-  private def loadLibWithLinux(conf: SparkConf, jni: JniLibLoader): Unit = {
-    val loader = find(conf)
-    loader.loadLib(jni)
-  }
-
-  private def loadLibWithMacOS(jni: JniLibLoader): Unit = {
-    // Placeholder for loading shared libs on MacOS if user needs.
-  }
-
-  private def find(conf: SparkConf): SharedLibraryLoader = {
-    val systemName = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_OS)
-    val loader = if (systemName.isDefined) {
-      val systemVersion = conf.get(GlutenConfig.GLUTEN_LOAD_LIB_OS_VERSION)
-      if (systemVersion.isEmpty) {
-        throw new GlutenException(
-          s"${GlutenConfig.GLUTEN_LOAD_LIB_OS_VERSION.key} must be specified when specifies the " +
-            s"${GlutenConfig.GLUTEN_LOAD_LIB_OS.key}")
-      }
-      getForOS(systemName.get, systemVersion.get, "")
-    } else {
-      val system = "cat /etc/os-release".!!
-      val systemNamePattern = "^NAME=\"?(.*)\"?".r
-      val systemVersionPattern = "^VERSION=\"?(.*)\"?".r
-      val systemInfoLines = system.stripMargin.split("\n")
-      val systemNamePattern(systemName) =
-        systemInfoLines.find(_.startsWith("NAME=")).getOrElse("")
-      val systemVersionPattern(systemVersion) =
-        systemInfoLines.find(_.startsWith("VERSION=")).getOrElse("")
-      if (systemName.isEmpty || systemVersion.isEmpty) {
-        throw new GlutenException("Failed to get OS name and version info.")
-      }
-      getForOS(systemName, systemVersion, system)
-    }
-    loader
-  }
-
-  private def getForOS(
-      systemName: String,
-      systemVersion: String,
-      system: String): SharedLibraryLoader = {
-    if (systemName.contains("Ubuntu") && systemVersion.startsWith("20.04")) {
-      new SharedLibraryLoaderUbuntu2004
-    } else if (systemName.contains("Ubuntu") && systemVersion.startsWith("22.04")) {
-      new SharedLibraryLoaderUbuntu2204
-    } else if (systemName.contains("CentOS") && systemVersion.startsWith("9")) {
-      new SharedLibraryLoaderCentos9
-    } else if (
-      (systemName.contains("CentOS") || systemName.contains("Oracle"))
-      && systemVersion.startsWith("8")
-    ) {
-      new SharedLibraryLoaderCentos8
-    } else if (
-      (systemName.contains("CentOS") || systemName.contains("Oracle"))
-      && systemVersion.startsWith("7")
-    ) {
-      new SharedLibraryLoaderCentos7
-    } else if (systemName.contains("openEuler") && systemVersion.startsWith("24.03")) {
-      new SharedLibraryLoaderOpenEuler2403
-    } else if (systemName.contains("Alibaba Cloud Linux") && systemVersion.startsWith("3")) {
-      new SharedLibraryLoaderCentos8
-    } else if (systemName.contains("Alibaba Cloud Linux") && systemVersion.startsWith("2")) {
-      new SharedLibraryLoaderCentos7
-    } else if (systemName.contains("Anolis") && systemVersion.startsWith("8")) {
-      new SharedLibraryLoaderCentos8
-    } else if (systemName.contains("Anolis") && systemVersion.startsWith("7")) {
-      new SharedLibraryLoaderCentos7
-    } else if (system.contains("tencentos") && system.contains("2.4")) {
-      new SharedLibraryLoaderCentos7
-    } else if (system.contains("tencentos") && system.contains("3.2")) {
-      new SharedLibraryLoaderCentos8
-    } else if (systemName.contains("Red Hat") && systemVersion.startsWith("9")) {
-      new SharedLibraryLoaderCentos9
-    } else if (systemName.contains("Red Hat") && systemVersion.startsWith("8")) {
-      new SharedLibraryLoaderCentos8
-    } else if (systemName.contains("Red Hat") && systemVersion.startsWith("7")) {
-      new SharedLibraryLoaderCentos7
-    } else if (systemName.contains("Debian") && systemVersion.startsWith("11")) {
-      new SharedLibraryLoaderDebian11
-    } else if (systemName.contains("Debian") && systemVersion.startsWith("12")) {
-      new SharedLibraryLoaderDebian12
-    } else {
-      throw new GlutenException(
-        s"Found unsupported OS($systemName, $systemVersion)! Currently, Gluten's Velox backend" +
-          " only supports Ubuntu 20.04/22.04, CentOS 7/8, Oracle 7/8, openEuler 24.03, " +
-          "Alibaba Cloud Linux 2/3 & Anolis 7/8, tencentos 2.4/3.2, RedHat 7/8, " +
-          "Debian 11/12.")
-    }
-  }
-}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala
new file mode 100755
index 000000000000..a66a351d9ee9
--- /dev/null
+++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUtils.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.config.GlutenConfig._
+import org.apache.gluten.exception.GlutenException
+import org.apache.gluten.jni.JniLibLoader
+import org.apache.gluten.spi.SharedLibraryLoader
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.internal.SparkConfigUtil._
+
+import java.io.FileInputStream
+import java.util.{Properties, ServiceLoader}
+
+import scala.collection.JavaConverters._
+
+object SharedLibraryLoaderUtils {
+  private def isMacOS: Boolean = {
+    val osName = System.getProperty("os.name")
+    osName.startsWith("Mac OS X") || osName.startsWith("macOS")
+  }
+
+  def load(conf: SparkConf, jni: JniLibLoader): Unit = {
+    val shouldLoad = conf.get(GLUTEN_LOAD_LIB_FROM_JAR)
+    if (!shouldLoad) {
+      return
+    }
+
+    val (osName, osVersion) = conf.get(GLUTEN_LOAD_LIB_OS) match {
+      case Some(os) =>
+        (
+          os,
+          conf
+            .get(GLUTEN_LOAD_LIB_OS_VERSION)
+            .getOrElse(
+              throw new GlutenException(
+                s"${GLUTEN_LOAD_LIB_OS_VERSION.key} must be specified when specifies the " +
+                  s"${GLUTEN_LOAD_LIB_OS.key}")))
+      case None if isMacOS =>
+        (System.getProperty("os.name"), System.getProperty("os.version"))
+      case None =>
+        val props = new Properties()
+        val in = new FileInputStream("/etc/os-release")
+        props.load(in)
+        (props.getProperty("NAME"), props.getProperty("VERSION"))
+    }
+
+    val loaders = ServiceLoader
+      .load(classOf[SharedLibraryLoader])
+      .asScala
+      .filter(loader => loader.accepts(osName, osVersion))
+      .toSeq
+
+    if (loaders.isEmpty) {
+      throw new GlutenException(
+        s"Cannot find SharedLibraryLoader for $osName $osVersion, please" +
+          "check whether your custom SharedLibraryLoader is implemented and loadable.")
+    }
+
+    if (loaders.size > 1) {
+      throw new GlutenException(
+        s"Found more than one SharedLibraryLoader for $osName $osVersion:" +
+          s" ${loaders.mkString(",")}, " +
+          "please check whether your custom SharedLibraryLoader is implemented correctly.")
+    }
+
+    val loader = loaders.head
+    try {
+      loader.loadLib(jni)
+    } catch {
+      case e: Throwable =>
+        throw new GlutenException(
+          s"Failed to load shared libraries for $osName $osVersion using $loader",
+          e)
+    }
+  }
+}
diff --git a/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader b/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
new file mode 100644
index 000000000000..0b3ef55dda1b
--- /dev/null
+++ b/backends-velox/src/test/resources/META-INF/services/org.apache.gluten.spi.SharedLibraryLoader
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.gluten.utils.MySharedLibraryLoader
diff --git a/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala b/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala
new file mode 100644
index 000000000000..c2902d652c44
--- /dev/null
+++ b/backends-velox/src/test/scala/org/apache/gluten/utils/MySharedLibraryLoader.scala
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.jni.JniLibLoader
+import org.apache.gluten.spi.SharedLibraryLoader
+import org.apache.gluten.utils.MySharedLibraryLoader.LOADED
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+class MySharedLibraryLoader extends SharedLibraryLoader {
+  override def accepts(osName: String, osVersion: String): Boolean = {
+    osName == "My OS" && osVersion == "1.0"
+  }
+
+  /**
+   * Load the required shared libraries using the given JniLibLoader.
+   *
+   * @param loader
+   *   JniLibLoader to load the shared libraries
+   */
+  override def loadLib(loader: JniLibLoader): Unit = {
+    LOADED.set(true)
+  }
+}
+
+object MySharedLibraryLoader {
+  val LOADED = new AtomicBoolean(false)
+}
diff --git a/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala
new file mode 100644
index 000000000000..c0c16eff746b
--- /dev/null
+++ b/backends-velox/src/test/scala/org/apache/gluten/utils/SharedLibraryLoaderUtilsSuite.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.utils
+
+import org.apache.gluten.config.GlutenConfig.{GLUTEN_LOAD_LIB_FROM_JAR, GLUTEN_LOAD_LIB_OS, GLUTEN_LOAD_LIB_OS_VERSION}
+
+import org.apache.spark.SparkConf
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class SharedLibraryLoaderUtilsSuite extends AnyFunSuite {
+
+  test("Load SharedLibraryLoader with SPI") {
+    val sparkConf = new SparkConf()
+      .set(GLUTEN_LOAD_LIB_FROM_JAR.key, "true")
+      .set(GLUTEN_LOAD_LIB_OS_VERSION.key, "1.0")
+      .set(GLUTEN_LOAD_LIB_OS.key, "My OS")
+
+    SharedLibraryLoaderUtils.load(sparkConf, null)
+    assert(MySharedLibraryLoader.LOADED.get())
+  }
+}