build.sbt

/*
 * Copyright (2021) The Delta Lake Project Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// scalastyle:off line.size.limit

import java.nio.file.Files

import sbtprotoc.ProtocPlugin.autoImport._

import xsbti.compile.CompileAnalysis

import Checkstyle._
import ShadedIcebergBuild._
import Mima._
import Unidoc._

// Scala versions
val scala213 = "2.13.17"
val all_scala_versions = Seq(scala213)

// Due to how publishArtifact is determined for javaOnlyReleaseSettings, incl. storage
// It was necessary to change default_scala_version to scala213 in build.sbt
// to build the project with Scala 2.13 only
// As a setting, it's possible to set it on command line easily
// sbt 'set default_scala_version := 2.13.16' [commands]
// FIXME Why not use scalaVersion?
val default_scala_version = settingKey[String]("Default Scala version")
Global / default_scala_version := scala213

// Scala version to use for all projects
scalaVersion := default_scala_version.value

// crossScalaVersions must be set to Nil on the root project to avoid conflicts
crossScalaVersions := Nil

val internalModuleNames = settingKey[Set[String]]("Internal module artifact names to exclude from POM")

// Spark compatibility version to delta-spark and its dependent modules.
// This is also a marker used by CrossSparkVersions.runOnlyForReleasableSparkModules.
// For more information see CrossSparkVersions.scala
val sparkVersion = settingKey[String]("Spark compatibility version").withRank(KeyRanks.Invisible)
val sparkArtifactVersion = settingKey[String]("Spark Maven artifact version")
ThisBuild / sparkArtifactVersion := CrossSparkVersions.getSparkArtifactVersion()

// Dependent library versions
val defaultSparkVersion = SparkVersionSpec.DEFAULT.fullVersion // Spark version to use for testing in non-delta-spark related modules
val hadoopVersion = "3.4.2"
val sparkVersionForKernelTest = "4.0.0"
val scalaTestVersion = "3.2.15"
val scalaTestVersionForConnectors = "3.0.8"
val parquet4sVersion = "1.9.4"
val protoVersion = "3.25.1"
val grpcVersion = "1.62.2"
val flinkVersion = CrossFlinkVersions.getFlinkArtifactVersion()
val gcsConnectorVersion = "4.0.4"
val junit5Version = "5.14.0"
val junit5TestDependencies = Seq(
  "org.junit.jupiter" % "junit-jupiter-api" % junit5Version % "test",
  "org.junit.jupiter" % "junit-jupiter-engine" % junit5Version % "test",
  "org.junit.jupiter" % "junit-jupiter-params" % junit5Version % "test",
  "com.github.sbt.junit" % "jupiter-interface" % "0.16.0" % "test"
)

// Optional kernel version override. See `project/KernelVersion.scala` for the
// resolution rule and `-DkernelVersion=<v>` semantics.
val kernelVersionOpt: Option[String] = KernelVersion.kernelVersionOpt

// Define the ecosystem support flags.
val supportIceberg = CrossSparkVersions.getSparkVersionSpec().supportIceberg
val supportHudi = CrossSparkVersions.getSparkVersionSpec().supportHudi

// For Java 11 use the following on command line
// sbt 'set targetJvm := "11"' [commands]
val targetJvm = settingKey[String]("Target JVM version")
Global / targetJvm := "11"

lazy val javaVersion = sys.props.getOrElse("java.version", "Unknown")
lazy val javaVersionInt = javaVersion.split("\\.")(0).toInt

lazy val commonSettings = Seq(
  organization := "io.delta",
  scalaVersion := default_scala_version.value,
  crossScalaVersions := all_scala_versions,
  fork := true,
  scalacOptions ++= Seq("-Ywarn-unused:imports"),
  javacOptions ++= {
    if (javaVersion.startsWith("1.8")) {
      Seq.empty // `--release` is supported since JDK 9 and the minimum supported JDK is 8
    } else {
      Seq("--release", targetJvm.value) // generated bytecode should be usable with JVM 1.8
    }
  },

  // Make sure any tests in any project that uses Spark is configured for running well locally
  Test / javaOptions ++= Seq(
    "-Dspark.ui.enabled=false",
    "-Dspark.ui.showConsoleProgress=false",
    "-Dspark.databricks.delta.snapshotPartitions=2",
    "-Dspark.sql.shuffle.partitions=5",
    "-Ddelta.log.cacheSize=3",
    "-Dspark.databricks.delta.delta.log.cacheSize=3",
    "-Dspark.sql.sources.parallelPartitionDiscovery.parallelism=5",
    "-Xmx1024m"
  ) ++ {
    if (javaVersionInt >= 17) {
      Seq(  // For Java 17 +
        "--add-opens=java.base/java.nio=ALL-UNNAMED",
        "--add-opens=java.base/java.lang=ALL-UNNAMED",
        "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED",
        "--add-opens=java.base/java.net=ALL-UNNAMED",
        "--add-opens=java.base/java.util=ALL-UNNAMED",
        "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
        "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
      )
    } else {
      Seq.empty
    }
  },

  testOptions += Tests.Argument("-oF"),

  // Unidoc settings: by default dont document any source file
  unidocSourceFilePatterns := Nil,
)

////////////////////////////
// START: Code Formatting //
////////////////////////////

/** Enforce java code style on compile. */
def javafmtCheckSettings(): Seq[Def.Setting[Task[CompileAnalysis]]] = Seq(
  (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value
)

/** Enforce scala code style on compile. */
def scalafmtCheckSettings(): Seq[Def.Setting[Task[CompileAnalysis]]] = Seq(
  (Compile / compile) := ((Compile / compile) dependsOn (Compile / scalafmtCheckAll)).value,
)

// TODO: define fmtAll and fmtCheckAll tasks that run both scala and java fmts/checks

//////////////////////////
// END: Code Formatting //
//////////////////////////

/**
 * Note: we cannot access sparkVersion.value here, since that can only be used within a task or
 *       setting macro.
 */
def runTaskOnlyOnSparkMaster[T](
    task: sbt.TaskKey[T],
    taskName: String,
    projectName: String,
    emptyValue: => T): Def.Initialize[Task[T]] = {
  if (CrossSparkVersions.getSparkVersionSpec().isMaster) {
    Def.task(task.value)
  } else {
    Def.task {
      // scalastyle:off println
      val masterVersion = SparkVersionSpec.MASTER.map(_.fullVersion).getOrElse("(no master version configured)")
      println(s"Project $projectName: Skipping `$taskName` as Spark version " +
        s"${CrossSparkVersions.getSparkVersion()} does not equal $masterVersion.")
      // scalastyle:on println
      emptyValue
    }
  }
}

lazy val connectCommon = (project in file("spark-connect/common"))
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings(
    name := "delta-connect-common",
    commonSettings,
    CrossSparkVersions.sparkDependentSettings(sparkVersion),
    releaseSettings,
    // Export as JAR instead of classes directory. This ensures protobuf-generated classes
    // (e.g., io.delta.connect.proto.DeltaCommand) are available as a JAR file in fullClasspath,
    // which can be symlinked and picked up by Spark Submit's jars/* wildcard in connectClient tests.
    exportJars := true,
    libraryDependencies ++= Seq(
      "io.grpc" % "protoc-gen-grpc-java" % grpcVersion asProtocPlugin(),
      "io.grpc" % "grpc-protobuf" % grpcVersion,
      "io.grpc" % "grpc-stub" % grpcVersion,
      "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf",
      "javax.annotation" % "javax.annotation-api" % "1.3.2",

      "org.apache.spark" %% "spark-connect-common" % sparkArtifactVersion.value % "provided",
    ),
    PB.protocVersion := protoVersion,
    Compile / PB.targets := Seq(
      PB.gens.java -> (Compile / sourceManaged).value,
      PB.gens.plugin("grpc-java") -> (Compile / sourceManaged).value
    )
  )

lazy val connectClient = (project in file("spark-connect/client"))
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .dependsOn(connectCommon % "compile->compile;test->test;provided->provided")
  .settings(
    name := "delta-connect-client",
    commonSettings,
    releaseSettings,
    CrossSparkVersions.sparkDependentSettings(sparkVersion),
    // Shared refresh test traits compiled into both this module and the `spark`
    // module. See io.delta.tables.shared.
    Test / unmanagedSourceDirectories += {
      (LocalRootProject / baseDirectory).value /
        "spark-shared-tests" / "src" / "test" / "scala-shared"
    },
    libraryDependencies ++= Seq(
      "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf",
      "org.apache.spark" %% "spark-connect-client-jvm" % sparkArtifactVersion.value % "provided",

      // Test deps
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.apache.spark" %% "spark-connect-client-jvm" % sparkArtifactVersion.value % "test" classifier "tests"
    ),
    (Test / javaOptions) += {
      // Create a (mini) Spark Distribution based on the server classpath.
      val serverClassPath = (connectServer / Compile / fullClasspath).value
      val distributionDir = crossTarget.value / "test-dist"
      val jarsDir = distributionDir / "jars"

      if (!distributionDir.exists()) {
        IO.createDirectory(jarsDir)
        // V2/STRICT mode requires the Delta Kernel engine at runtime
        // (io.delta.kernel.defaults.engine.DefaultEngine), so add the packaged kernel jars.
        val kernelJars = Seq(
          (kernelApi / Compile / packageBin).value,
          (kernelDefaults / Compile / packageBin).value)
        // Create symlinks for all dependencies (filter to only JAR files)
        (serverClassPath.map(_.data).filter(_.isFile) ++ kernelJars).distinct.foreach { jarFile =>
          val linkedJarFile = jarsDir / jarFile.getName
          if (!java.nio.file.Files.exists(linkedJarFile.toPath)) {
            Files.createSymbolicLink(linkedJarFile.toPath, jarFile.toPath)
          }
        }
        // Create a symlink for the log4j properties
        val confDir = distributionDir / "conf"
        IO.createDirectory(confDir)
        val log4jProps = (sparkV1 / Test / resourceDirectory).value / "log4j2.properties"
        val linkedLog4jProps = confDir / "log4j2.properties"
        if (!java.nio.file.Files.exists(linkedLog4jProps.toPath)) {
          Files.createSymbolicLink(linkedLog4jProps.toPath, log4jProps.toPath)
        }
      }
      // Return the location of the distribution directory.
      "-Ddelta.spark.home=" + distributionDir
    },
    // Required for testing addFeatureSupport/dropFeatureSupport.
    Test / envVars += ("DELTA_TESTING", "1")
  )

lazy val connectServer = (project in file("spark-connect/server"))
  .dependsOn(connectCommon % "compile->compile;test->test;provided->provided")
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings(
    name := "delta-connect-server",
    commonSettings,
    releaseSettings,
    CrossSparkVersions.sparkDependentSettings(sparkVersion),
    // Export as JAR instead of classes directory. Required for connectClient test setup so that
    // classes like SimpleDeltaConnectService are available as a JAR file that can be symlinked
    // and picked up by Spark Submit's jars/* wildcard. Also prevents classpath conflicts.
    exportJars := true,
    assembly / assemblyMergeStrategy := {
      // Discard module-info.class files from Java 9+ modules and multi-release JARs
      case "module-info.class" => MergeStrategy.discard
      case PathList("META-INF", "versions", _, "module-info.class") => MergeStrategy.discard
      case x =>
        val oldStrategy = (assembly / assemblyMergeStrategy).value
        oldStrategy(x)
    },
    libraryDependencies ++= Seq(
      "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf",

      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-connect" % sparkArtifactVersion.value % "provided",

      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-connect" % sparkArtifactVersion.value % "test" classifier "tests",
    ),
    excludeDependencies ++= Seq(
      // Exclude connect common because a properly shaded version of it is included in the
      // spark-connect jar. Including it causes classpath problems.
      ExclusionRule("org.apache.spark", "spark-connect-common_2.13"),
      // Exclude connect shims because we have spark-core on the classpath. The shims are only
      // needed for the client. Including it causes classpath problems.
      ExclusionRule("org.apache.spark", "spark-connect-shims_2.13")
    ),
    // Required for testing addFeatureSupport/dropFeatureSupport.
    Test / envVars += ("DELTA_TESTING", "1"),
    // Force Spark to bind to localhost to avoid network issues
    Test / envVars += ("SPARK_LOCAL_IP", "127.0.0.1")
  )

lazy val deltaSuiteGenerator = (project in file("spark/delta-suite-generator"))
  .disablePlugins(ScalafmtPlugin)
  .settings (
    name := "delta-suite-generator",
    commonSettings,
    scalaStyleSettings,
    skipReleaseSettings, // Internal module - not published to Maven
    libraryDependencies ++= Seq(
      "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0",
      "org.scalameta" %% "scalameta" % "4.13.5",
      "org.scalameta" %% "scalafmt-core" % "3.9.6",
      "commons-cli" % "commons-cli" % "1.9.0",
      "commons-codec" % "commons-codec" % "1.17.2",
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
    ),
    Compile / mainClass := Some("io.delta.suitegenerator.ModularSuiteGenerator"),
    Test / baseDirectory := (ThisBuild / baseDirectory).value,
  )

// ============================================================
// Spark Module 1: sparkV1 (prod code only, no tests)
// ============================================================
lazy val sparkV1 = (project in file("spark"))
  .dependsOn(storage)
  .enablePlugins(Antlr4Plugin)
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-spark-v1",
    commonSettings,
    scalaStyleSettings,
    skipReleaseSettings, // Internal module - not published to Maven
    CrossSparkVersions.sparkDependentSettings(sparkVersion),

    // Export as JAR instead of classes directory. This prevents dependent projects
    // (e.g., connectServer) from seeing multiple 'classes' directories with the same
    // name in their classpath, which would cause FileAlreadyExistsException.
    exportJars := true,

    // Tests are compiled in the final 'spark' module to avoid circular dependencies
    Test / sources := Seq.empty,
    Test / resources := Seq.empty,

    libraryDependencies ++= Seq(
      // Adding test classifier seems to break transitive resolution of the core dependencies
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "provided",
      // For DynamoDBCommitStore
      "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided",

      // Test deps
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test",
      "junit" % "junit" % "4.13.2" % "test",
      "com.novocode" % "junit-interface" % "0.11" % "test",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.mockito" % "mockito-inline" % "4.11.0" % "test",
    ),
    Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++
        listPythonFiles(baseDirectory.value.getParentFile / "python"),
    Antlr4 / antlr4PackageName := Some("io.delta.sql.parser"),
    Antlr4 / antlr4GenListener := true,
    Antlr4 / antlr4GenVisitor := true,

    // Introduced in https://github.com/delta-io/delta/commit/d2990624d34b6b86fa5cf230e00a89b095fde254
    //
    // Hack to avoid errors related to missing repo-root/target/scala-2.13/classes/
    // In multi-module sbt projects, some dependencies may attempt to locate this directory
    // at the repository root, causing build failures if it doesn't exist.
    createTargetClassesDir := {
      val dir = baseDirectory.value.getParentFile / "target" / "scala-2.13" / "classes"
      Files.createDirectories(dir.toPath)
    },
    // Generate Python version.py file with hardcoded version.
    // This file is committed to git and auto-updated during build.
    generatePythonVersion := {
      val versionValue = version.value
      // Trim -SNAPSHOT suffix to get PyPI-compatible version (like setup.py does)
      val trimmedVersion = versionValue.split("-SNAPSHOT")(0)
      val versionFile = baseDirectory.value.getParentFile / "python" / "delta" / "version.py"
      val content =
        s"""#
           |# Copyright (2026) The Delta Lake Project Authors.
           |#
           |# Licensed under the Apache License, Version 2.0 (the "License");
           |# you may not use this file except in compliance with the License.
           |# You may obtain a copy of the License at
           |#
           |# http://www.apache.org/licenses/LICENSE-2.0
           |#
           |# Unless required by applicable law or agreed to in writing, software
           |# distributed under the License is distributed on an "AS IS" BASIS,
           |# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
           |# See the License for the specific language governing permissions and
           |# limitations under the License.
           |#
           |
           |# This file is auto-generated by the build.sbt generatePythonVersion task.
           |# Do not edit manually - edit version.sbt instead and run:
           |#   build/sbt sparkV1/generatePythonVersion
           |
           |__version__ = "$trimmedVersion"
           |""".stripMargin
      IO.write(versionFile, content)
      versionFile
    },
    // Hook both createTargetClassesDir and generatePythonVersion into compile task
    Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir dependsOn generatePythonVersion).value,
    // Generate the package object to provide the version information in runtime.
    Compile / sourceGenerators += Def.task {
      val file = (Compile / sourceManaged).value / "io" / "delta" / "package.scala"
      IO.write(file,
        s"""package io
           |
           |package object delta {
           |  val VERSION = "${version.value}"
           |}
           |""".stripMargin)
      Seq(file)
    },
  )

// ============================================================
// Spark Module 2: sparkV1Filtered (v1 without DeltaLog for v2 dependency)
// This filtered version of sparkV1 is needed because sparkV2 (spark/v2) depends on some
// V1 classes for utilities and common functionality, but must NOT have access to DeltaLog,
// Snapshot, OptimisticTransaction, or actions that belongs to core V1 delta libraries.
// We should use Kernel as the Delta implementation.
// ============================================================
lazy val sparkV1Filtered = (project in file("spark-v1-filtered"))
  .dependsOn(sparkV1)
  .dependsOn(storage)
  .settings(
    name := "delta-spark-v1-filtered",
    commonSettings,
    skipReleaseSettings, // Internal module - not published to Maven
    exportJars := true,  // Export as JAR to avoid classpath conflicts

    // No source code - just repackage sparkV1 without DeltaLog classes
    Compile / sources := Seq.empty,
    Test / sources := Seq.empty,

    // Repackage sparkV1 jar but exclude DeltaLog and related classes
    Compile / packageBin / mappings := {
      val v1Mappings = (sparkV1 / Compile / packageBin / mappings).value

      // Filter out DeltaLog, Snapshot, OptimisticTransaction, and actions.scala classes
      v1Mappings.filterNot { case (file, path) =>
        path.contains("org/apache/spark/sql/delta/DeltaLog")
      }
    },
  )

// ============================================================
// Spark Module 3: sparkV2 (Kernel-based DSv2 connector, depends on v1-filtered)
// ============================================================
lazy val sparkV2 = {
  // When kernelVersionOpt is set, consume the kernel from Maven instead of the
  // local source projects: drop the project dependsOn / kernelApi unmanagedJars
  // wiring and add the matching Maven artifacts at that version.
  val kernelSourceDeps: Project => Project = kernelVersionOpt match {
    case None => p => p
      .dependsOn(kernelDefaults)
      .dependsOn(kernelUnityCatalog % "compile->compile;test->test")
    case Some(_) => p => p
  }
  val kernelDepSettings: Seq[Setting[_]] = kernelVersionOpt match {
    case None => Seq(
      // make sure shaded kernel-api jar exists before compiling/testing
      Compile / compile := (Compile / compile)
        .dependsOn(kernelApi / Compile / packageBin).value,
      Test / test := (Test / test)
        .dependsOn(kernelApi / Compile / packageBin).value,
      Test / unmanagedJars += (kernelApi / Test / packageBin).value,
      Compile / unmanagedJars ++= Seq(
        (kernelApi / Compile / packageBin).value
      )
    )
    case Some(v) => Seq(
      libraryDependencies ++= Seq(
        "io.delta" % "delta-kernel-api" % v,
        "io.delta" % "delta-kernel-defaults" % v,
        "io.delta" % "delta-kernel-unitycatalog" % v,
        // sparkV2 tests depend on UC test helpers (InMemoryUCClient,
        // UCCatalogManagedTestUtils) that live in kernel-unitycatalog's test sources.
        // Consume them via the published tests-classifier jar.
        "io.delta" % "delta-kernel-unitycatalog" % v % Test classifier "tests"
      )
    )
  }

  kernelSourceDeps(Project("sparkV2", file("spark/v2")).dependsOn(sparkV1Filtered))
    .dependsOn(goldenTables % "test")
    .settings(
      name := "delta-spark-v2",
      commonSettings,
      javafmtCheckSettings,
      skipReleaseSettings, // Internal module - not published to Maven
      CrossSparkVersions.sparkDependentSettings(sparkVersion),
      exportJars := true,  // Export as JAR to avoid classpath conflicts

      Test / javaOptions ++= Seq("-ea"),
      libraryDependencies ++= junit5TestDependencies ++ Seq(
        "org.mockito" % "mockito-inline" % "4.11.0" % "test",
        "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided",
        "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided",
        "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "provided",

        // Spark test classes for Scala/Java test utilities
        "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests",
        "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests",
        "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests",
        // ScalaTest for test utilities (needed by Spark test classes)
        "org.scalatest" %% "scalatest" % scalaTestVersion % "test"
      ),
      Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
      TestParallelization.settings
    )
    .settings(kernelDepSettings: _*)
}


// ============================================================
// Spark Module 4: delta-spark (final published module - unified v1+v2)
// ============================================================
lazy val spark = (project in file("spark-unified"))
  .dependsOn(sparkV1)
  .dependsOn(sparkV2)
  .dependsOn(storage)
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-spark",
    commonSettings,
    scalaStyleSettings,
    sparkMimaSettings,
    releaseSettings, // Published to Maven as delta-spark.jar

    // Set Test baseDirectory before sparkDependentSettings() so it uses the correct directory
    Test / baseDirectory := (sparkV1 / baseDirectory).value,

    // Test sources from spark/ directory (sparkV1's directory) AND spark-unified's own directory,
    // plus this version's shim directories (e.g. `src/test/scala-shims/spark-4.2` and any
    // cross-version shared dir like `src/test/scala-shims/spark-4.1-4.2`).
    // MUST be set BEFORE crossSparkSettings() to avoid overwriting version-specific directories.
    Test / unmanagedSourceDirectories := {
      val sparkDir = (sparkV1 / baseDirectory).value
      val unifiedDir = baseDirectory.value
      val shimDirs = CrossSparkVersions.getSparkVersionSpec()
        .additionalSourceDirs
        .map(dir => unifiedDir / "src" / "test" / dir)
      Seq(
        sparkDir / "src" / "test" / "scala",
        sparkDir / "src" / "test" / "java",
        unifiedDir / "src" / "test" / "scala",
        unifiedDir / "src" / "test" / "java"
      ) ++ shimDirs
    },
    Test / unmanagedResourceDirectories := Seq(
      (sparkV1 / baseDirectory).value / "src" / "test" / "resources",
      baseDirectory.value / "src" / "test" / "resources"
    ),
    // Shared refresh test traits compiled into both this module and the
    // `connectClient` module. See io.delta.tables.shared.
    Test / unmanagedSourceDirectories += {
      (LocalRootProject / baseDirectory).value /
        "spark-shared-tests" / "src" / "test" / "scala-shared"
    },

    CrossSparkVersions.sparkDependentSettings(sparkVersion),

    // MiMa should use the generated JAR (not classDirectory) because we merge classes at package time
    mimaCurrentClassfiles := (Compile / packageBin).value,

    // Export as JAR to dependent projects (e.g., connectServer, connectClient).
    // This prevents classpath conflicts from internal module 'classes' directories.
    exportJars := true,

    // Internal module artifact names to exclude from published POM
    internalModuleNames := Set("delta-spark-v1", "delta-spark-v1-shaded", "delta-spark-v2"),

    // Merge classes from internal modules (v1, v2) into final JAR
    // kernel modules are kept as separate JARs and listed as dependencies in POM
    Compile / packageBin / mappings ++= {
      val log = streams.value.log

      // Collect mappings from internal modules
      val v1Mappings = (sparkV1 / Compile / packageBin / mappings).value
      val v2Mappings = (sparkV2 / Compile / packageBin / mappings).value

      // Include Python files (from spark/ directory)
      val pythonMappings = listPythonFiles(baseDirectory.value.getParentFile / "python")

      // Combine all mappings
      val allMappings = v1Mappings ++ v2Mappings ++ pythonMappings

      // Detect duplicate class files
      val classFiles = allMappings.filter(_._2.endsWith(".class"))
      val duplicates = classFiles.groupBy(_._2).filter(_._2.size > 1)

      if (duplicates.nonEmpty) {
        log.error(s"Found ${duplicates.size} duplicate class(es) in packageBin mappings:")
        duplicates.foreach { case (className, entries) =>
          log.error(s"  - $className:")
          entries.foreach { case (file, path) => log.error(s"      from: $file") }
        }
        sys.error("Duplicate classes found. This indicates overlapping code between sparkV1, sparkV2, and storage modules.")
      }

      allMappings.distinct
    },

    // Exclude internal modules from published POM and add kernel dependencies.
    // Kernel modules are transitive through sparkV2 (an internal module), so they
    // are lost when sparkV2 is filtered out. We re-add them explicitly here.
    pomPostProcess := { node =>
      val internalModules = internalModuleNames.value
      val ver = version.value
      import scala.xml._
      import scala.xml.transform._

      def kernelDependencyNode(artifactId: String): Elem = {
        <dependency>
          <groupId>io.delta</groupId>
          <artifactId>{artifactId}</artifactId>
          <version>{ver}</version>
        </dependency>
      }

      val kernelDeps = Seq(
        kernelDependencyNode("delta-kernel-api"),
        kernelDependencyNode("delta-kernel-defaults"),
        kernelDependencyNode("delta-kernel-unitycatalog")
      )

      new RuleTransformer(new RewriteRule {
        override def transform(n: Node): Seq[Node] = n match {
          case e: Elem if e.label == "dependencies" =>
            val filtered = e.child.filter {
              case child: Elem if child.label == "dependency" =>
                val artifactId = (child \ "artifactId").text
                !internalModules.exists(module => artifactId.startsWith(module))
              case _ => true
            }
            Seq(e.copy(child = filtered ++ kernelDeps))
          case _ => Seq(n)
        }
      }).transform(node).head
    },

    pomIncludeRepository := { _ => false },

    // Filter internal modules from project dependencies
    // This works together with pomPostProcess to ensure internal modules
    // (sparkV1, sparkV2, sparkV1Filtered) are not listed as dependencies in POM
    projectDependencies := {
      val internalModules = internalModuleNames.value
      projectDependencies.value.filterNot(dep => internalModules.contains(dep.name))
    },

    libraryDependencies ++= junit5TestDependencies ++ Seq(
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "provided",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "provided",
      "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided",

      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test",
      "junit" % "junit" % "4.13.2" % "test",
      "com.novocode" % "junit-interface" % "0.11" % "test",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.mockito" % "mockito-inline" % "4.11.0" % "test",
    ),

    Test / testOptions += Tests.Argument("-oDF"),
    Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),

    // Don't execute in parallel since we can't have multiple Sparks in the same JVM
    Test / parallelExecution := false,

    javaOptions += "-Xmx1024m",

    // Configurations to speed up tests and reduce memory footprint
    Test / javaOptions ++= Seq(
      "-Dspark.ui.enabled=false",
      "-Dspark.ui.showConsoleProgress=false",
      "-Dspark.databricks.delta.snapshotPartitions=2",
      "-Dspark.sql.shuffle.partitions=5",
      "-Ddelta.log.cacheSize=3",
      "-Dspark.databricks.delta.delta.log.cacheSize=3",
      "-Dspark.sql.sources.parallelPartitionDiscovery.parallelism=5",
      "-Xmx1024m"
    ),

    // Required for testing table features see https://github.com/delta-io/delta/issues/1602
    Test / envVars += ("DELTA_TESTING", "1"),

    TestParallelization.settings,
  )
  .configureUnidoc(
    generatedJavaDoc = CrossSparkVersions.getSparkVersionSpec().generateDocs,
    generateScalaDoc = CrossSparkVersions.getSparkVersionSpec().generateDocs,
    // spark-connect has classes with the same name as spark-core, this causes compilation issues
    // with unidoc since it concatenates the classpaths from all modules
    // ==> thus we exclude such sources
    // (mostly) relevant github issue: https://github.com/sbt/sbt-unidoc/issues/77
    classPathToSkip = "spark-connect"
  )

lazy val contribs = (project in file("contribs"))
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-contribs",
    commonSettings,
    scalaStyleSettings,
    releaseSettings,
    // Set sparkVersion directly (not sparkDependentModuleName) so that
    // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark
    // suffix to the artifact name. delta-contribs is only published as delta-contribs_2.13.
    sparkVersion := CrossSparkVersions.getSparkVersion(),
    Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++
      listPythonFiles(baseDirectory.value.getParentFile / "python"),

    Test / testOptions += Tests.Argument("-oDF"),
    Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),

    // Don't execute in parallel since we can't have multiple Sparks in the same JVM
    Test / parallelExecution := false,

    javaOptions += "-Xmx1024m",

    // Configurations to speed up tests and reduce memory footprint
    Test / javaOptions ++= Seq(
      "-Dspark.ui.enabled=false",
      "-Dspark.ui.showConsoleProgress=false",
      "-Dspark.databricks.delta.snapshotPartitions=2",
      "-Dspark.sql.shuffle.partitions=5",
      "-Ddelta.log.cacheSize=3",
      "-Dspark.databricks.delta.delta.log.cacheSize=3",
      "-Dspark.sql.sources.parallelPartitionDiscovery.parallelism=5",
      "-Xmx1024m"
    ),

    // Introduced in https://github.com/delta-io/delta/commit/d2990624d34b6b86fa5cf230e00a89b095fde254
    //
    // Hack to avoid errors related to missing repo-root/target/scala-2.13/classes/
    // In multi-module sbt projects, some dependencies may attempt to locate this directory
    // at the repository root, causing build failures if it doesn't exist.
    createTargetClassesDir := {
      val dir = baseDirectory.value.getParentFile / "target" / "scala-2.13" / "classes"
      Files.createDirectories(dir.toPath)
    },
    Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir).value,
    TestParallelization.settings
  ).configureUnidoc()


// Unity Catalog version. Three modes, in priority order:
//
//  1. `-DuseDefaultUnityCatalogReleaseVersion=true`: use `defaultUnityCatalogReleaseVersion`
//     below -- the last released UC version on Maven Central. For workflows that don't actually
//     need DRC APIs (e.g. unidoc, lint) and want to skip the pinned UC build. Shared across
//     workflows by reading this single constant, so bumping is a one-line change here.
//
//  2. Release mode: set `unityCatalogReleaseVersion = Some("0.5.0")` (or whatever released
//     version the release branch ships against). sbt resolves the coordinate from Maven Central
//     like any other dependency.
//
//  3. Pinned mode (default): leave `unityCatalogReleaseVersion = None`. The version string
//     comes from `setup_unitycatalog_main.sh --print-version`, which encodes both the pinned
//     UC main SHA and UC's declared base version; the script is the single source of truth.
//     The same script (without the flag) publishes the matching jars to ~/.ivy2/local when
//     `ensurePinnedUnityCatalog` decides they're missing.
//
// Override with -DunityCatalogVersion=<anything> for ad-hoc experiments.
val unityCatalogReleaseVersion: Option[String] = None
val defaultUnityCatalogReleaseVersion = "0.5.0"
val useDefaultUnityCatalogReleaseVersion: Boolean =
  sys.props.getOrElse("useDefaultUnityCatalogReleaseVersion", "false").toBoolean
val unityCatalogSetupScript = "project/scripts/setup_unitycatalog_main.sh"

// Lazy so release-mode / useDefaultUnityCatalogReleaseVersion builds never shell out.
lazy val pinnedUnityCatalogVersion: String = {
  import scala.sys.process._
  Process(Seq("bash", unityCatalogSetupScript, "--print-version")).!!.trim
}
val unityCatalogVersion: String = sys.props.getOrElse(
  "unityCatalogVersion",
  if (useDefaultUnityCatalogReleaseVersion) defaultUnityCatalogReleaseVersion
  else unityCatalogReleaseVersion.getOrElse(pinnedUnityCatalogVersion))

// UC publishes its Spark connector per Spark major.minor (e.g. unitycatalog-spark_4.1). This
// is the artifact name without the Scala suffix - sbt's `%%` appends `_2.13` for dep
// resolution; the canary check below appends `_2.13` explicitly for the Ivy/Maven path.
val unityCatalogSparkArtifactName: String =
  s"unitycatalog-spark_${CrossSparkVersions.getSparkVersionSpec().shortVersion}"

/**
 * Returns true when `current` is at least `target`. Numeric segments only; suffix after
 * the first `-` (e.g. `-SNAPSHOT-abc1234`) is stripped before comparison.
 */
def isAtLeastVersion(current: String, target: String): Boolean = {
  def parts(v: String): Seq[Int] =
    v.takeWhile(_ != '-').split('.').iterator
      .map(p => scala.util.Try(p.toInt).getOrElse(0)).toSeq
  val cur = parts(current)
  val tgt = parts(target)
  val n = math.max(cur.length, tgt.length)
  (0 until n).iterator
    .map(i => (cur.lift(i).getOrElse(0), tgt.lift(i).getOrElse(0)))
    .find { case (a, b) => a != b }
    .forall { case (a, b) => a >= b }
}

val sparkUnityCatalogJacksonVersion = "2.15.4" // We are using Spark 4.0's Jackson version 2.15.x, to override Unity Catalog 0.3.0's version 2.18.x

// Publishes the pinned UC jars to ~/.ivy2/local if they're not already cached there. Hooked
// into `update` on the UC-dependent projects below, so plain `sbt testOnly ...` on a clean
// checkout just works. No-op in release mode. Opt out with
// `-Ddelta.autoBuildPinnedUnityCatalog=false`, in which case sbt errors with a pointer to the
// setup script.
val ensurePinnedUnityCatalog = taskKey[Unit](
  "Publish the pinned UC jars locally if the Ivy coordinate isn't already cached.")

// Extracted so the task body can read as a short guard rather than three nested ifs.
def publishPinnedUnityCatalog(log: sbt.util.Logger, canary: java.io.File): Unit = {
  val shouldAutoBuild =
    sys.props.getOrElse("delta.autoBuildPinnedUnityCatalog", "true").toBoolean
  if (!shouldAutoBuild) {
    sys.error(
      s"""|Pinned Unity Catalog jars are not published locally for coordinate
          |$unityCatalogVersion.
          |Auto-build is disabled (-Ddelta.autoBuildPinnedUnityCatalog=false).
          |Run: bash $unityCatalogSetupScript""".stripMargin)
  }
  log.info(s"[UC] Pinned UC jars not found for coordinate $unityCatalogVersion.")
  log.info(
    s"[UC] Running $unityCatalogSetupScript - takes ~3-5 minutes on a cold cache, <1s on a warm one.")
  import scala.sys.process._
  val procLogger = ProcessLogger(
    line => log.info(s"[UC setup] $line"),
    line => log.warn(s"[UC setup] $line"))
  // SPARK_VERSION tells the script which Spark variant to build (forwarded to UC's sbt as
  // -DsparkVersion).
  val exit = Process(
    Seq("bash", unityCatalogSetupScript),
    None,
    "SPARK_VERSION" -> CrossSparkVersions.getSparkVersionSpec().shortVersion).!(procLogger)
  if (exit != 0) {
    sys.error(
      s"[UC] $unityCatalogSetupScript exited with code $exit. Run it manually to see full output.")
  }
  if (!canary.exists) {
    sys.error(
      s"[UC] $unityCatalogSetupScript succeeded but ${canary.getAbsolutePath} is still missing - " +
        "the publish target layout may have changed.")
  }
}

Global / ensurePinnedUnityCatalog := {
  // Resolve the .value dependencies eagerly - sbt's task macro warns when
  // `.value` appears inside conditional branches.
  val log = streams.value.log
  // No-op whenever the effective version resolves to something Maven Central can serve:
  // release mode, -DuseDefaultUnityCatalogReleaseVersion=true, or -DunityCatalogVersion=<released>.
  val usingReleasedVersion = useDefaultUnityCatalogReleaseVersion ||
    sys.props.contains("unityCatalogVersion")
  if (unityCatalogReleaseVersion.isEmpty && !usingReleasedVersion) {
    val home = file(sys.props("user.home"))
    // Canary on the spark artifact, not client/server: those are Spark-version-independent and
    // would short-circuit the trigger when only the active Spark version changed, leaving the
    // needed unitycatalog-spark_${X.Y}_2.13 unpublished.
    val sparkArtifact = s"${unityCatalogSparkArtifactName}_2.13"
    // Check both layouts: a restored sbt cache can pre-populate ivy alone, leaving m2 empty -
    // checking only ivy would silently skip the slow publish and break mvn-based consumers.
    val ivy2Canary = home / ".ivy2" / "local" / "io.unitycatalog" /
      sparkArtifact / unityCatalogVersion / "ivys" / "ivy.xml"
    val m2Canary = home / ".m2" / "repository" / "io" / "unitycatalog" /
      sparkArtifact / unityCatalogVersion /
      s"$sparkArtifact-$unityCatalogVersion.pom"
    if (!ivy2Canary.exists || !m2Canary.exists) {
      publishPinnedUnityCatalog(log, ivy2Canary)
    }
  }
}

lazy val sparkUnityCatalog = (project in file("spark/unitycatalog"))
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(ScalafmtPlugin)
  .settings(
    name := "delta-spark-unitycatalog",
    commonSettings,
    skipReleaseSettings,
    javafmtCheckSettings(),
    CrossSparkVersions.sparkDependentSettings(sparkVersion),

    // Publish the pinned UC jars before sbt tries to resolve them.
    update := update.dependsOn(ensurePinnedUnityCatalog).value,

    // This is a test-only module - no production sources
    Compile / sources := Seq.empty,

    // Ensure Java sources are picked up
    Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "java",

    Test / javaOptions ++= Seq("-ea"),

    // Don't execute in parallel since we can't have multiple Sparks in the same JVM
    Test / parallelExecution := false,

    // Force ALL Jackson dependencies to match Spark's Jackson version
    // This overrides Jackson from Unity Catalog's transitive dependencies (e.g., Armeria)
    dependencyOverrides ++= Seq(
      "com.fasterxml.jackson.core" % "jackson-core" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-annotations" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-databind" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.module" %% "jackson-module-scala" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % sparkUnityCatalogJacksonVersion,
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % sparkUnityCatalogJacksonVersion
    ),

    libraryDependencies ++= junit5TestDependencies ++ Seq(
      "org.assertj" % "assertj-core" % "3.26.3" % "test",
      // Lombok for generating boilerplate code
      "org.projectlombok" % "lombok" % "1.18.34" % "test",

      // Unity Catalog dependencies - per-Spark-version artifact (unitycatalog-spark_4.X_2.13).
      // Exclude Jackson to use Spark's Jackson 2.15.x
      "io.unitycatalog" %% s"unitycatalog-spark_${CrossSparkVersions.getSparkVersionSpec().shortVersion}" % unityCatalogVersion % "test" excludeAll(
        ExclusionRule(organization = "com.fasterxml.jackson.core"),
        ExclusionRule(organization = "com.fasterxml.jackson.module"),
        ExclusionRule(organization = "com.fasterxml.jackson.datatype"),
        ExclusionRule(organization = "com.fasterxml.jackson.dataformat")
      ),
      "io.unitycatalog" % "unitycatalog-server" % unityCatalogVersion % "test" excludeAll(
        ExclusionRule(organization = "com.fasterxml.jackson.core"),
        ExclusionRule(organization = "com.fasterxml.jackson.module"),
        ExclusionRule(organization = "com.fasterxml.jackson.datatype"),
        ExclusionRule(organization = "com.fasterxml.jackson.dataformat")
      ),

      // Spark test dependencies
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test",
    ),

    // Conditionally add hadoop-aws dependency only when UC_REMOTE=true
    // Please see: https://github.com/delta-io/delta/issues/5624#issuecomment-3673383736
    // Once we release the relocated unitycatalog-server, we can remove this.
    libraryDependencies ++= {
      if (sys.env.get("UC_REMOTE").contains("true")) {
        Seq(
          "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "test",
          "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "test",
          "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion % "test",
          "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion % "test"
        )
      } else {
        Seq.empty
      }
    },

    // Add delta-iceberg compiled classes so that UniForm Iceberg conversion works in integration
    // tests.
    Test / unmanagedJars ++= {
      if (supportIceberg) Seq(
        Attributed.blank((iceberg / Compile / packageBin).value)
      ) else Seq.empty
    },

    Test / javaOptions += s"-DsupportIceberg=${supportIceberg}",
    Test / testOptions += Tests.Argument("-oDF"),
    Test / testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a")
  )

lazy val sharing = (project in file("sharing"))
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings(
    name := "delta-sharing-spark",
    commonSettings,
    scalaStyleSettings,
    releaseSettings,
    CrossSparkVersions.sparkDependentSettings(sparkVersion),
    Test / javaOptions ++= Seq("-ea"),
    libraryDependencies ++= Seq(
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "provided",

      "io.delta" %% "delta-sharing-client" % "1.4.1",

      // Test deps
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.scalatestplus" %% "scalacheck-1-15" % "3.2.9.0" % "test",
      "junit" % "junit" % "4.13.2" % "test",
      "com.novocode" % "junit-interface" % "0.11" % "test",
      "org.apache.spark" %% "spark-catalyst" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % sparkArtifactVersion.value % "test" classifier "tests",
      "org.apache.spark" %% "spark-hive" % sparkArtifactVersion.value % "test" classifier "tests",
    ),
    TestParallelization.settings
  ).configureUnidoc()

lazy val kernelApi = (project in file("kernel/kernel-api"))
  .enablePlugins(ScalafmtPlugin)
  .settings(
    name := "delta-kernel-api",
    commonSettings,
    scalaStyleSettings,
    javaOnlyReleaseSettings,
    javafmtCheckSettings,
    scalafmtCheckSettings,

    // Use unique classDirectory name to avoid conflicts in connectClient test setup
    // This allows connectClient to create symlinks without FileAlreadyExistsException
    Compile / classDirectory := target.value / "scala-2.13" / "kernel-api-classes",

    Test / javaOptions ++= Seq("-ea"),

    // Also publish a test-jar (classifier = "tests") so consumers (e.g. kernelDefault)
    // can depend on test utilities via a published artifact instead of depending on raw class directories.
    Test / publishArtifact := true,
    Test / packageBin / artifactClassifier := Some("tests"),
    libraryDependencies ++= junit5TestDependencies ++ Seq(
      "org.roaringbitmap" % "RoaringBitmap" % "0.9.25",
      "org.slf4j" % "slf4j-api" % "1.7.36",

      "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5",
      "com.fasterxml.jackson.core" % "jackson-core" % "2.13.5",
      "com.fasterxml.jackson.core" % "jackson-annotations" % "2.13.5",
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % "2.13.5",

      // JSR-305 annotations for @Nullable
      "com.google.code.findbugs" % "jsr305" % "3.0.2",

      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test",
      "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test",
      "org.assertj" % "assertj-core" % "3.26.3" % "test",
      // JMH dependencies allow writing micro-benchmarks for testing performance of components.
      // JMH has framework to define benchmarks and takes care of many common functionalities
      // such as warm runs, cold runs, defining benchmark parameter variables etc.
      "org.openjdk.jmh" % "jmh-core" % "1.37" % "test",
      "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test"
    ),
    // Shade jackson libraries so that connector developers don't have to worry
    // about jackson version conflicts.
    Compile / packageBin := assembly.value,
    assembly / assemblyJarName := s"${name.value}-${version.value}.jar",
    assembly / logLevel := Level.Info,
    assembly / test := {},
    assembly / assemblyExcludedJars := {
      val cp = (assembly / fullClasspath).value
      val allowedPrefixes = Set("META_INF", "io", "jackson")
      cp.filter { f =>
        !allowedPrefixes.exists(prefix => f.data.getName.startsWith(prefix))
      }
    },
     assembly / assemblyShadeRules := Seq(
      ShadeRule.rename("com.fasterxml.jackson.**" -> "io.delta.kernel.shaded.com.fasterxml.jackson.@1").inAll
    ),
    assembly / assemblyMergeStrategy := {
      // Discard `module-info.class` to fix the `different file contents found` error.
      case "module-info.class" => MergeStrategy.discard
      case PathList("META-INF", "versions", _*) => MergeStrategy.discard
      case PathList("META-INF", "services", xs @ _*) => MergeStrategy.discard
      case x =>
        val oldStrategy = (assembly / assemblyMergeStrategy).value
        oldStrategy(x)
    },
    // Generate the package object to provide the version information in runtime.
    Compile / sourceGenerators += Def.task {
      val file = (Compile / sourceManaged).value / "io" / "delta" / "kernel" / "Meta.java"
      IO.write(file,
        s"""/*
           | * Copyright (2024) The Delta Lake Project Authors.
           | *
           | * Licensed under the Apache License, Version 2.0 (the "License");
           | * you may not use this file except in compliance with the License.
           | * You may obtain a copy of the License at
           | *
           | * http://www.apache.org/licenses/LICENSE-2.0
           | *
           | * Unless required by applicable law or agreed to in writing, software
           | * distributed under the License is distributed on an "AS IS" BASIS,
           | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
           | * See the License for the specific language governing permissions and
           | * limitations under the License.
           | */
           |package io.delta.kernel;
           |
           |public final class Meta {
           |    public static final String KERNEL_VERSION = "${version.value}";
           |}
           |""".stripMargin)
      Seq(file)
    },
    MultiShardMultiJVMTestParallelization.settings,
    javaCheckstyleSettings("dev/kernel-checkstyle.xml"),
    // Unidoc settings
    unidocSourceFilePatterns := Seq(SourceFilePattern("io/delta/kernel/")),
  ).configureUnidoc(docTitle = "Delta Kernel")

lazy val kernelDefaults = (project in file("kernel/kernel-defaults"))
  .enablePlugins(ScalafmtPlugin)
  .dependsOn(storage)
  .dependsOn(storage % "test->test") // Required for InMemoryCommitCoordinator for tests
  .dependsOn(goldenTables % "test")
  .settings(
    name := "delta-kernel-defaults",
    commonSettings,
    scalaStyleSettings,
    javaOnlyReleaseSettings,
    javafmtCheckSettings,
    scalafmtCheckSettings,

    // Use unique classDirectory name to avoid conflicts in connectClient test setup
    // This allows connectClient to create symlinks without FileAlreadyExistsException
    Compile / classDirectory := target.value / "scala-2.13" / "kernel-defaults-classes",

    Test / javaOptions ++= Seq("-ea"),
    // This allows generating tables with unsupported test table features in delta-spark
    Test / envVars += ("DELTA_TESTING", "1"),

    // Put the shaded kernel-api JAR on the classpath (compile & test)
    Compile / unmanagedJars += (kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Compile / packageBin).value,

    // Make sure the shaded JAR is produced before we compile/run tests
    Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value,
    Test / test       := (Test    / test).dependsOn(kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Test / packageBin).value,

    libraryDependencies ++= Seq(
      "org.assertj" % "assertj-core" % "3.26.3" % Test,
      "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion,
      "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.5",
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jdk8" % "2.13.5",
      "org.apache.parquet" % "parquet-hadoop" % "1.16.0",

      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "junit" % "junit" % "4.13.2" % "test",
      "commons-io" % "commons-io" % "2.8.0" % "test",
      "com.novocode" % "junit-interface" % "0.11" % "test",
      "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test",
      "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test",
      // JMH dependencies allow writing micro-benchmarks for testing performance of components.
      // JMH has framework to define benchmarks and takes care of many common functionalities
      // such as warm runs, cold runs, defining benchmark parameter variables etc.
      "org.openjdk.jmh" % "jmh-core" % "1.37" % "test",
      "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test",

      // The delta-spark and spark dependencies are mainly used for catalog-based table creation.
      // Instead of using the latest snapshot, those are fine to use the released 4.0.0.
      "io.delta" %% "delta-spark" % "4.0.0" % "test",
      "org.apache.spark" %% "spark-hive" % sparkVersionForKernelTest % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % sparkVersionForKernelTest % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % sparkVersionForKernelTest % "test" classifier "tests",
      "org.apache.spark" %% "spark-catalyst" % sparkVersionForKernelTest % "test" classifier "tests",
    ),
    MultiShardMultiJVMTestParallelization.settings,
    javaCheckstyleSettings("dev/kernel-checkstyle.xml"),
      // Unidoc settings
    unidocSourceFilePatterns += SourceFilePattern("io/delta/kernel/"),
  ).configureUnidoc(docTitle = "Delta Kernel Defaults")

lazy val kernelBenchmarks = (project in file("kernel/kernel-benchmarks"))
  .enablePlugins(ScalafmtPlugin)
  .dependsOn(kernelDefaults % "test->test")
  .dependsOn(kernelApi % "test->test")
  .dependsOn(storage % "test->test")
  .dependsOn(kernelUnityCatalog % "test->test")
  .settings(
    name := "delta-kernel-benchmarks",
    commonSettings,
    skipReleaseSettings,
    exportJars := false,
    javafmtCheckSettings,
    scalafmtCheckSettings,

    libraryDependencies ++= Seq(
      "org.openjdk.jmh" % "jmh-core" % "1.37" % "test",
      "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test",
    ),
  )

lazy val kernelUnityCatalog = (project in file("kernel/unitycatalog"))
  .enablePlugins(ScalafmtPlugin)
  .dependsOn(kernelDefaults % "test->test")
  .dependsOn(storage)
  .settings (
    name := "delta-kernel-unitycatalog",
    commonSettings,
    javaOnlyReleaseSettings,
    javafmtCheckSettings,
    javaCheckstyleSettings("dev/kernel-checkstyle.xml"),
    scalaStyleSettings,
    scalafmtCheckSettings,

    // Publish the pinned UC jars before sbt tries to resolve them.
    update := update.dependsOn(ensurePinnedUnityCatalog).value,

    // Also publish a test-jar (classifier = "tests") so consumers (e.g. sparkV2 in
    // Maven mode) can depend on UC test helpers (InMemoryUCClient,
    // UCCatalogManagedTestUtils) via a published artifact.
    Test / publishArtifact := true,

    // Put the shaded kernel-api JAR on the classpath (compile & test)
    Compile / unmanagedJars += (kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Compile / packageBin).value,

    // Make sure the shaded JAR is produced before we compile/run tests
    Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value,
    Test / test       := (Test    / test).dependsOn(kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Test / packageBin).value,

    libraryDependencies ++= Seq(
      "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.apache.logging.log4j" % "log4j-slf4j-impl" % "2.25.3" % "test",
      "org.apache.logging.log4j" % "log4j-core" % "2.25.3" % "test",
    ),
    unidocSourceFilePatterns += SourceFilePattern("src/main/java/io/delta/unity/"),
  ).configureUnidoc()

// TODO javastyle tests
// TODO unidoc
// TODO(scott): figure out a better way to include tests in this project
lazy val storage = (project in file("storage"))
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-storage",
    commonSettings,
    exportJars := true,
    javaOnlyReleaseSettings,
    libraryDependencies ++= Seq(
      // User can provide any 2.x or 3.x version. We don't use any new fancy APIs. Watch out for
      // versions with known vulnerabilities.
      "org.apache.hadoop" % "hadoop-common" % hadoopVersion % "provided",

      // Note that the org.apache.hadoop.fs.s3a.Listing::createFileStatusListingIterator 3.3.1 API
      // is not compatible with 3.3.2.
      "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "provided",
      "io.unitycatalog" % "unitycatalog-client" % unityCatalogVersion excludeAll(
        ExclusionRule(organization = "org.openapitools"),
        ExclusionRule(organization = "com.fasterxml.jackson.core"),
        ExclusionRule(organization = "com.fasterxml.jackson.module"),
        ExclusionRule(organization = "com.fasterxml.jackson.datatype"),
        ExclusionRule(organization = "com.fasterxml.jackson.dataformat")
      ),

      // Test Deps
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      // Jackson datatype module needed for UC SDK tests (excluded from main compile scope)
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % "2.15.4" % "test",
    ) ++ (
      // unitycatalog-hadoop ships from UC 0.5.0 onward; older versions don't publish the
      // artifact, so resolving it would fail. Used by UCDeltaTokenBasedRestClient for
      // credential vending via UCCredentialHadoopConfs.
      if (isAtLeastVersion(unityCatalogVersion, "0.5.0")) {
        Seq("io.unitycatalog" % "unitycatalog-hadoop" % unityCatalogVersion excludeAll(
          ExclusionRule(organization = "org.openapitools"),
          ExclusionRule(organization = "com.fasterxml.jackson.core"),
          ExclusionRule(organization = "com.fasterxml.jackson.module"),
          ExclusionRule(organization = "com.fasterxml.jackson.datatype"),
          ExclusionRule(organization = "com.fasterxml.jackson.dataformat")
        ))
      } else Nil
    ),

    // Publish the pinned UC jars before sbt tries to resolve them. storage is the transitive
    // UC-client entry point for most of the build graph (sparkV1, sparkV2, kernelDefaults, etc.
    // all .dependsOn(storage)), so hooking here covers nearly every compile path.
    update := update.dependsOn(ensurePinnedUnityCatalog).value,

    // Unidoc settings
    unidocSourceFilePatterns += SourceFilePattern("/LogStore.java", "/CloseableIterator.java"),
    TestParallelization.settings
  ).configureUnidoc()

lazy val storageS3DynamoDB = (project in file("storage-s3-dynamodb"))
  .dependsOn(storage % "compile->compile;test->test;provided->provided")
  .dependsOn(spark % "test->test")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-storage-s3-dynamodb",
    commonSettings,
    javaOnlyReleaseSettings,

    // uncomment only when testing FailingS3DynamoDBLogStore. this will include test sources in
    // a separate test jar.
    // Test / publishArtifact := true,

    libraryDependencies ++= Seq(
      "com.amazonaws" % "aws-java-sdk" % "1.12.262" % "provided",

      // Test Deps
      "org.apache.hadoop" % "hadoop-aws" % hadoopVersion % "test", // RemoteFileChangedException
    ),
    TestParallelization.settings
  ).configureUnidoc()

val icebergSparkRuntimeArtifactName = {
 val currentSparkVersion = CrossSparkVersions.getSparkVersion()
 val (expMaj, expMin, _) = getMajorMinorPatch(currentSparkVersion)
 s"iceberg-spark-runtime-$expMaj.$expMin"
}

lazy val testDeltaIcebergJar = (project in file("testDeltaIcebergJar"))
  // delta-iceberg depends on delta-spark! So, we need to include it during our test.
  .dependsOn(spark % "test")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings(
    name := "test-delta-iceberg-jar",
    commonSettings,
    skipReleaseSettings,
    exportJars := true,
    Compile / unmanagedJars += (iceberg / assembly).value,
    libraryDependencies ++= Seq(
      "org.apache.hadoop" % "hadoop-client" % hadoopVersion,
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "org.apache.spark" %% "spark-core" % defaultSparkVersion % "test"
    )
  )

val deltaIcebergSparkIncludePrefixes = Seq(
  // We want everything from this package
  "org/apache/spark/sql/delta/icebergShaded",
  // Server-side planning support
  "org/apache/spark/sql/delta/serverSidePlanning",

  // We only want the files in this project from this package. e.g. we want to exclude
  // org/apache/spark/sql/delta/commands/convert/ConvertTargetFile.class (from delta-spark project).
  "org/apache/spark/sql/delta/commands/convert/DataFileWrapper",
  "org/apache/spark/sql/delta/commands/convert/DelegatingIcebergTable",
  "org/apache/spark/sql/delta/commands/convert/IcebergFileManifest",
  "org/apache/spark/sql/delta/commands/convert/IcebergPartitionConverter",
  "org/apache/spark/sql/delta/commands/convert/IcebergSchemaUtils",
  "org/apache/spark/sql/delta/commands/convert/IcebergSparkWrappers",
  "org/apache/spark/sql/delta/commands/convert/IcebergStatsUtils",
  "org/apache/spark/sql/delta/commands/convert/IcebergTable",
  "org/apache/spark/sql/delta/commands/convert/IcebergTableLike",
  "org/apache/spark/sql/delta/commands/convert/ManifestFileWrapper",
  "org/apache/spark/sql/delta/commands/convert/PartitionFieldSummaryWrapper",
  "org/apache/spark/sql/delta/commands/convert/TypeToSparkTypeWithCustomCast"
)

// Build using: build/sbt clean icebergShaded/compile iceberg/compile
// It will fail the first time, just re-run it.
// scalastyle:off println
lazy val iceberg = (project in file("iceberg"))
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-iceberg",
    commonSettings,
    scalaStyleSettings,
    releaseSettings,
    // Set sparkVersion directly (not sparkDependentModuleName) so that
    // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark
    // suffix to the artifact name. delta-iceberg is only published as delta-iceberg_2.13.
    sparkVersion := CrossSparkVersions.getSparkVersion(),
    libraryDependencies ++= {
      if (supportIceberg) {
        Seq(
          // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error
          // due to legacy scala.
          "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1",
          "com.github.ben-manes.caffeine" % "caffeine" % "3.2.4",
          "com.jolbox" % "bonecp" % "0.8.0.RELEASE" % "test",
          "org.eclipse.jetty" % "jetty-server" % "11.0.26" % "test",
          "org.eclipse.jetty" % "jetty-servlet" % "11.0.26" % "test",
          "org.xerial" % "sqlite-jdbc" % "3.45.0.0" % "test",
          "org.apache.httpcomponents.core5" % "httpcore5" % "5.4.3" % "test",
          "org.apache.httpcomponents.client5" % "httpclient5" % "5.6.4" % "test",
          "org.apache.iceberg" %% icebergSparkRuntimeArtifactName % "1.12.0" % "provided",
          // For FixedGcsAccessTokenProvider (GCS server-side planning credentials)
          "com.google.cloud.bigdataoss" % "util-hadoop" % gcsConnectorVersion % "provided"
        )
      } else {
        Seq.empty
      }
    },
    // Skip compilation and publishing when supportIceberg is false
    Compile / skip := !supportIceberg,
    Test / skip := !supportIceberg,
    publish / skip := !supportIceberg,
    publishLocal / skip := !supportIceberg,
    publishM2 / skip := !supportIceberg,
    Compile / unmanagedJars += (icebergShaded / assembly).value,
    // Generate the assembly JAR as the package JAR
    Compile / packageBin := assembly.value,
    Compile / scalacOptions += "-nowarn",
    Test / unmanagedJars += (icebergTestsShaded / assembly).value,
    Test / scalacOptions += "-nowarn",
    assembly / assemblyJarName := {
      s"${moduleName.value}_${scalaBinaryVersion.value}-${version.value}.jar"
    },
    assembly / logLevel := Level.Info,
    assembly / test := {},
    assembly / assemblyExcludedJars := {
      // Note: the input here is only `libraryDependencies` jars, not `.dependsOn(_)` jars.
      val allowedJars = Seq(
        s"iceberg-shaded_${scalaBinaryVersion.value}-${version.value}.jar",
        s"scala-library-${scala213}.jar",
        s"scala-collection-compat_${scalaBinaryVersion.value}-2.1.1.jar",
        "caffeine-3.2.4.jar",
        // Note: We are excluding
        // - antlr4-runtime-4.9.3.jar
        // - checker-qual-3.19.0.jar
        // - error_prone_annotations-2.10.0.jar
      )
      val cp = (assembly / fullClasspath).value

      // Return `true` when we want the jar `f` to be excluded from the assembly jar
      cp.filter { f =>
        val doExclude = !allowedJars.contains(f.data.getName)
        println(s"Excluding jar: ${f.data.getName} ? $doExclude")
        doExclude
      }
    },
    assembly / assemblyMergeStrategy := {
      case "module-info.class" => MergeStrategy.discard
      case PathList("META-INF", "versions", _*) => MergeStrategy.discard
      // Project iceberg `dependsOn` spark and accidentally brings in it, along with its
      // compile-time dependencies (like delta-storage). We want these excluded from the
      // delta-iceberg jar.
      case PathList("io", "delta", xs @ _*) =>
        // - delta-storage will bring in classes: io/delta/storage
        // - delta-spark will bring in classes: io/delta/exceptions/, io/delta/implicits,
        //   io/delta/package, io/delta/sql, io/delta/tables,
        MergeStrategy.discard
      case PathList("com", "databricks", xs @ _*) =>
        // delta-spark will bring in com/databricks/spark/util
        MergeStrategy.discard
      case PathList("org", "apache", "spark", xs @ _*)
        if !deltaIcebergSparkIncludePrefixes.exists { prefix =>
          s"org/apache/spark/${xs.mkString("/")}".startsWith(prefix) } =>
        MergeStrategy.discard
      case PathList("scoverage", xs @ _*) =>
        MergeStrategy.discard
      case x =>
        (assembly / assemblyMergeStrategy).value(x)
    },
    assemblyPackageScala / assembleArtifact := false
  )
// scalastyle:on println

val icebergShadedVersion = "1.12.0"
lazy val icebergShaded = (project in file("icebergShaded"))
  .dependsOn(spark % "provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "iceberg-shaded",
    commonSettings,
    skipReleaseSettings,
    // must exclude all dependencies from Iceberg that delta-spark includes
    libraryDependencies ++= Seq(
      // Fix Iceberg's legacy java.lang.NoClassDefFoundError: scala/jdk/CollectionConverters$ error
      // due to legacy scala.
      "org.scala-lang.modules" %% "scala-collection-compat" % "2.1.1" % "provided",
      "org.apache.iceberg" % "iceberg-core" % icebergShadedVersion excludeAll (
        icebergExclusionRules: _*
      ),
      "org.apache.iceberg" % "iceberg-hive-metastore" % icebergShadedVersion excludeAll (
        icebergExclusionRules: _*
      ),
      // the hadoop client and hive metastore versions come from this file in the
      // iceberg repo of icebergShadedVersion: iceberg/gradle/libs.versions.toml
      "org.apache.hadoop" % "hadoop-client" % "2.7.3" % "provided" excludeAll (
        hadoopClientExclusionRules: _*
      ),
      "org.apache.hive" % "hive-metastore" % "2.3.8" % "provided" excludeAll (
        hiveMetastoreExclusionRules: _*
      )
    ),
    // Generated shaded Iceberg JARs
    Compile / packageBin := assembly.value,
    assembly / assemblyJarName := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar",
    assembly / logLevel := Level.Info,
    assembly / test := {},
    assembly / assemblyShadeRules := Seq(
      ShadeRule.rename("org.apache.iceberg.**" -> "shadedForDelta.@0").inAll
    ),
    assembly / assemblyExcludedJars := {
      val cp = (assembly / fullClasspath).value
      cp.filter { jar =>
        val doExclude = jar.data.getName.contains("jackson-annotations") ||
          jar.data.getName.contains("RoaringBitmap")
        doExclude
      }
    },
    // all following clases have Delta customized implementation under icebergShaded/src and thus
    // require them to be 'first' to replace the class from iceberg jar
    assembly / assemblyMergeStrategy := updateMergeStrategy((assembly / assemblyMergeStrategy).value),
    assemblyPackageScala / assembleArtifact := false,
  )

lazy val icebergTestsShaded = (project in file("icebergTestsShaded"))
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "iceberg-tests-shaded",
    commonSettings,
    skipReleaseSettings,
    // must exclude all dependencies from Iceberg that delta-spark includes
    libraryDependencies ++= Seq(
      "org.apache.iceberg" % "iceberg-core" % icebergShadedVersion classifier "tests" excludeAll (
        icebergExclusionRules: _*
      ),
    ),
    // Generated shaded Iceberg JARs
    Compile / packageBin := assembly.value,
    assembly / assemblyJarName := s"${name.value}_${scalaBinaryVersion.value}-${version.value}.jar",
    assembly / logLevel := Level.Info,
    assembly / test := {},
    assembly / assemblyShadeRules := Seq(
      ShadeRule.rename("org.apache.iceberg.**" -> "shadedForDelta.@0").inAll
    ),
    assembly / assemblyExcludedJars := {
      val cp = (fullClasspath in assembly).value
      cp.filter { jar =>
        val doExclude = jar.data.getName.contains("jackson-annotations") ||
          jar.data.getName.contains("RoaringBitmap")
        doExclude
      }
    },
    assembly / assemblyMergeStrategy := updateMergeStrategy((assembly / assemblyMergeStrategy).value),
    assemblyPackageScala / assembleArtifact := false,
  )


lazy val hudi = (project in file("hudi"))
  .dependsOn(spark % "compile->compile;test->test;provided->provided")
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings (
    name := "delta-hudi",
    commonSettings,
    scalaStyleSettings,
    releaseSettings,
    // Set sparkVersion directly (not sparkDependentModuleName) so that
    // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark
    // suffix to the artifact name. delta-hudi is only published as delta-hudi_2.13.
    sparkVersion := CrossSparkVersions.getSparkVersion(),
    libraryDependencies ++= {
      if (supportHudi) {
        Seq(
          "org.apache.hudi" % "hudi-java-client" % "0.15.0" % "compile" excludeAll(
            ExclusionRule(organization = "org.apache.hadoop"),
            ExclusionRule(organization = "org.apache.zookeeper"),
          ),
          "org.apache.spark" %% "spark-avro" % sparkArtifactVersion.value % "test" excludeAll ExclusionRule(organization = "org.apache.hadoop"),
          "org.apache.parquet" % "parquet-avro" % "1.16.0" % "compile"
        )
      } else {
        Seq.empty
      }
    },
    // Skip compilation and publishing when supportHudi is false
    Compile / skip := !supportHudi,
    Test / skip := !supportHudi,
    publish / skip := !supportHudi,
    publishLocal / skip := !supportHudi,
    publishM2 / skip := !supportHudi,
    assembly / assemblyJarName := s"${name.value}-assembly_${scalaBinaryVersion.value}-${version.value}.jar",
    assembly / logLevel := Level.Info,
    assembly / test := {},
    assembly / assemblyMergeStrategy := {
      // Project hudi `dependsOn` spark and accidentally brings in it, along with its
      // compile-time dependencies (like delta-storage). We want these excluded from the
      // delta-hudi jar.
      case PathList("io", "delta", xs @ _*) =>
        // - delta-storage will bring in classes: io/delta/storage
        // - delta-spark will bring in classes: io/delta/exceptions/, io/delta/implicits,
        //   io/delta/package, io/delta/sql, io/delta/tables,
        MergeStrategy.discard
      case PathList("com", "databricks", xs @ _*) =>
        // delta-spark will bring in com/databricks/spark/util
        MergeStrategy.discard
      case PathList("org", "apache", "spark", "sql", "delta", "hudi", xs @ _*) =>
        MergeStrategy.first
      case PathList("org", "apache", "spark", xs @ _*) =>
        MergeStrategy.discard
      // Discard `module-info.class` to fix the `different file contents found` error.
      // TODO Upgrade SBT to 1.5 which will do this automatically
      case "module-info.class" => MergeStrategy.discard
      // Discard unused `parquet.thrift` so that we don't conflict the file used by the user
      case "parquet.thrift" => MergeStrategy.discard
      // Hudi metadata writer requires this service file to be present on the classpath
      case "META-INF/services/org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory" => MergeStrategy.first
      // Discard the jackson service configs that we don't need. These files are not shaded so
      // adding them may conflict with other jackson version used by the user.
      case PathList("META-INF", "services", xs @ _*) => MergeStrategy.discard
      case x =>
        MergeStrategy.first
    },
    // Make the 'compile' invoke the 'assembly' task to generate the uber jar.
    Compile / packageBin := assembly.value,
    TestParallelization.settings
  )

lazy val flink = (project in file("flink"))
//  .dependsOn(kernelApi)
  .dependsOn(kernelDefaults)
  .dependsOn(kernelUnityCatalog)
  .settings(
    name := "delta-flink",
    // Publish one artifact per compatible Flink minor line, for example delta-flink_2.0.
    moduleName := s"delta-flink_${CrossFlinkVersions.getFlinkVersionSpec().shortVersion}",
    commonSettings,
    releaseSettings,
    javafmtCheckSettings(),
    // The base release publishes non-Flink modules. Flink versions are published separately by
    // CrossFlinkVersions.crossFlinkReleaseSteps.
    publish / skip := sys.props.getOrElse("skipFlinkPublish", "false").toBoolean,
    publishArtifact := scalaBinaryVersion.value == "2.13", // only publish once
    autoScalaLibrary := false, // exclude scala-library from dependencies
    assembly / assemblyJarName := s"delta-flink-$flinkVersion-${version.value}.jar",
    assembly / assemblyMergeStrategy := {
      // Discard module-info.class files from Java 9+ modules and multi-release JARs
      case "module-info.class" => MergeStrategy.discard
      case "parquet.thrift" => MergeStrategy.discard
      case PathList("META-INF", "versions", _, "module-info.class") => MergeStrategy.discard
      case PathList("mozilla", "public-suffix-list.txt") => MergeStrategy.discard
      case x => MergeStrategy.first
    },
    assembly / assemblyExcludedJars := {
      val cp = (assembly / fullClasspath).value
      cp.filter { entry =>
        entry.data.getName.startsWith("bundle-") &&
          entry.data.getName.endsWith(".jar")
      }
    },
    Compile / unmanagedJars += (kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Compile / packageBin).value,

    // Make sure the shaded JAR is produced before we compile/run tests
    Compile / compile := (Compile / compile).dependsOn(kernelApi / Compile / packageBin).value,
    Test / test       := (Test    / test).dependsOn(kernelApi / Compile / packageBin).value,
    Test / unmanagedJars += (kernelApi / Test / packageBin).value,

    Test / publishArtifact := false,
    Test / javaOptions ++= Seq(
      "--add-opens=java.base/java.util=ALL-UNNAMED" // for Flink with Java 17.
    ),
    crossPaths := false,

    // Publish the pinned UC jars before sbt tries to resolve them.
    update := update.dependsOn(ensurePinnedUnityCatalog).value,

    libraryDependencies ++= junit5TestDependencies ++ Seq(
      "org.apache.flink" % "flink-core" % flinkVersion % "provided",
      "org.apache.flink" % "flink-table-common" % flinkVersion % "provided",
      "org.apache.flink" % "flink-streaming-java" % flinkVersion % "provided",
      "org.apache.flink" % "flink-table-api-java-bridge" % flinkVersion % "provided",
      "io.unitycatalog" % "unitycatalog-client" % unityCatalogVersion,
      "org.apache.httpcomponents" % "httpclient" % "4.5.14" % Runtime,
      "dev.failsafe" % "failsafe" % "3.2.0",
      "com.github.ben-manes.caffeine" % "caffeine" % "3.1.8",
      "org.apache.hadoop" % "hadoop-aws" % hadoopVersion,
      "com.google.cloud.bigdataoss" % "gcs-connector" % gcsConnectorVersion % Provided classifier "shaded",
      // kernel-api uses RoaringBitmap but ships as a shaded fat-jar that excludes it, and the
      // unmanagedJars trick used above does not propagate kernel-api's managed deps. Declare it
      // here too -- pinned to the same version kernel-api uses -- so our DV code under
      // io.delta.flink.kernel.dv compiles against the same library on the classpath.
      "org.roaringbitmap" % "RoaringBitmap" % "0.9.25",

      "org.apache.flink" % "flink-test-utils" % flinkVersion % "test",
      "org.apache.flink" % "flink-clients" % flinkVersion % "test",
      "org.apache.flink" % "flink-table-api-java-bridge" % flinkVersion % Test,
      "org.apache.flink" % "flink-table-planner-loader" % flinkVersion % Test,
      "org.apache.flink" % "flink-table-runtime" % flinkVersion % Test,
      "org.apache.flink" % "flink-test-utils-junit" % flinkVersion  % Test,
      "org.slf4j" % "slf4j-log4j12" % "2.0.17" % "test",
      "com.github.tomakehurst" % "wiremock-jre8" % "2.35.0" % Test,
      "com.databricks" %% "databricks-connect" % "18.0.0" % Test,
    ),
    // Use jupiter
    excludeDependencies ++= Seq(
      ExclusionRule("junit", "junit"),
      ExclusionRule("org.junit.vintage", "junit-vintage-engine")
    )
  )


lazy val goldenTables = (project in file("connectors/golden-tables"))
  .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
  .settings(
    name := "golden-tables",
    commonSettings,
    skipReleaseSettings,
    libraryDependencies ++= Seq(
      // Test Dependencies
      "org.scalatest" %% "scalatest" % scalaTestVersion % "test",
      "commons-io" % "commons-io" % "2.8.0" % "test",

      "io.delta" %% "delta-spark" % "3.3.2" % "test",
      "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "test",
      "org.apache.spark" %% "spark-catalyst" % defaultSparkVersion % "test" classifier "tests",
      "org.apache.spark" %% "spark-core" % defaultSparkVersion % "test" classifier "tests",
      "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "test" classifier "tests"
    )
  )

/**
 * Get list of python files and return the mapping between source files and target paths
 * in the generated package JAR.
 */
def listPythonFiles(pythonBase: File): Seq[(File, String)] = {
  val pythonExcludeDirs = pythonBase / "lib" :: pythonBase / "doc" :: pythonBase / "bin" :: Nil
  import scala.collection.JavaConverters._
  val pythonFiles = Files.walk(pythonBase.toPath).iterator().asScala
    .map { path => path.toFile() }
    .filter { file => file.getName.endsWith(".py") && ! file.getName.contains("test") }
    .filter { file => ! pythonExcludeDirs.exists { base => IO.relativize(base, file).nonEmpty} }
    .toSeq

  pythonFiles pair Path.relativeTo(pythonBase)
}

ThisBuild / parallelExecution := false

val createTargetClassesDir = taskKey[Unit]("create target classes dir")
val generatePythonVersion = taskKey[File]("Generate Python version.py file")

/*
 ******************
 * Project groups *
 ******************
 */

// Don't use these groups for any other projects
lazy val sparkGroup = {
  val baseProjects = Seq(spark, sparkV1, sparkV1Filtered, sparkV2, contribs, sparkUnityCatalog, storage, storageS3DynamoDB, sharing, connectCommon, connectClient, connectServer)
  val allProjects = if (supportHudi) {
    baseProjects :+ hudi
  } else {
    baseProjects
  }

  Project("sparkGroup", file("sparkGroup"))
    .aggregate(allProjects.map(_.project): _*)
    .settings(
      // crossScalaVersions must be set to Nil on the aggregating project
      crossScalaVersions := Nil,
      publishArtifact := false,
      publish / skip := true,
    )
}

lazy val icebergGroup = {
  val allProjects = if (supportIceberg) {
    Seq(iceberg, testDeltaIcebergJar)
  } else {
    Seq.empty
  }

  Project("icebergGroup", file("icebergGroup"))
    .aggregate(allProjects.map(_.project): _*)
    .settings(
      // crossScalaVersions must be set to Nil on the aggregating project
      crossScalaVersions := Nil,
      publishArtifact := false,
      publish / skip := true,
    )
}

lazy val kernelGroup = project
  .aggregate(kernelApi, kernelDefaults, kernelBenchmarks)
  .settings(
    // crossScalaVersions must be set to Nil on the aggregating project
    crossScalaVersions := Nil,
    publishArtifact := false,
    publish / skip := true,
    unidocSourceFilePatterns := {
      (kernelApi / unidocSourceFilePatterns).value.scopeToProject(kernelApi) ++
      (kernelDefaults / unidocSourceFilePatterns).value.scopeToProject(kernelDefaults)
    }
  ).configureUnidoc(docTitle = "Delta Kernel")

lazy val flinkGroup = project
  .aggregate(flink)
  .settings(
    // crossScalaVersions must be set to Nil on the aggregating project
    crossScalaVersions := Nil,
    publishArtifact := false,
    publish / skip := true,
  )

/*
 ********************
 * Release settings *
 ********************
 */
import ReleaseTransformations._

lazy val skipReleaseSettings = Seq(
  publishArtifact := false,
  publish / skip := true
)


// Release settings for artifact that contains only Java source code
lazy val javaOnlyReleaseSettings = releaseSettings ++ Seq(
  // drop off Scala suffix from artifact names
  crossPaths := false,

  // we publish jars for each scalaVersion in crossScalaVersions. however, we only need to publish
  // one java jar. thus, only do so when the current scala version == default scala version
  publishArtifact := {
    val (expMaj, expMin, _) = getMajorMinorPatch(default_scala_version.value)
    s"$expMaj.$expMin" == scalaBinaryVersion.value
  },

  // exclude scala-library from dependencies in generated pom.xml
  autoScalaLibrary := false,
)

lazy val releaseSettings = Seq(
  publishMavenStyle := true,
  publishArtifact := true,
  Test / publishArtifact := false,
  releasePublishArtifactsAction := PgpKeys.publishSigned.value,
  releaseCrossBuild := true,
  pgpPassphrase := sys.env.get("PGP_PASSPHRASE").map(_.toArray),
  // Allow local Maven overwrites for release versions. The cross-Spark publish workflow
  // publishes modules like delta-contribs (no Spark suffix) in both the backward-compat
  // step and the per-version steps, producing identical artifacts. SBT 1.9+ blocks
  // overwriting release artifacts by default; this restores the prior behavior for local
  // publishing only (remote publish via publishSigned is unaffected).
  publishLocalConfiguration := publishLocalConfiguration.value.withOverwrite(true),
  publishM2Configuration := publishM2Configuration.value.withOverwrite(true),

  // TODO: This isn't working yet ...
  sonatypeProfileName := "io.delta", // sonatype account domain name prefix / group ID
  credentials += Credentials(
    "OSSRH Staging API Service",
    "ossrh-staging-api.central.sonatype.com",
    sys.env.getOrElse("SONATYPE_USERNAME", ""),
    sys.env.getOrElse("SONATYPE_PASSWORD", "")
  ),
  credentials += Credentials(
    "Sonatype Nexus Repository Manager",
    "central.sonatype.com",
    sys.env.getOrElse("SONATYPE_USERNAME", ""),
    sys.env.getOrElse("SONATYPE_PASSWORD", "")
  ),
  publishTo := {
    val ossrhBase = "https://ossrh-staging-api.central.sonatype.com/"
    val centralSnapshots = "https://central.sonatype.com/repository/maven-snapshots/"
    if (isSnapshot.value) {
      Some("snapshots" at centralSnapshots)
    } else {
      Some("releases"  at ossrhBase + "service/local/staging/deploy/maven2")
    }
  },
  licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0")),
  pomExtra :=
    <url>https://delta.io/</url>
      <scm>
        <url>git@github.com:delta-io/delta.git</url>
        <connection>scm:git:git@github.com:delta-io/delta.git</connection>
      </scm>
      <developers>
        <developer>
          <id>marmbrus</id>
          <name>Michael Armbrust</name>
          <url>https://github.com/marmbrus</url>
        </developer>
        <developer>
          <id>brkyvz</id>
          <name>Burak Yavuz</name>
          <url>https://github.com/brkyvz</url>
        </developer>
        <developer>
          <id>jose-torres</id>
          <name>Jose Torres</name>
          <url>https://github.com/jose-torres</url>
        </developer>
        <developer>
          <id>liwensun</id>
          <name>Liwen Sun</name>
          <url>https://github.com/liwensun</url>
        </developer>
        <developer>
          <id>mukulmurthy</id>
          <name>Mukul Murthy</name>
          <url>https://github.com/mukulmurthy</url>
        </developer>
        <developer>
          <id>tdas</id>
          <name>Tathagata Das</name>
          <url>https://github.com/tdas</url>
        </developer>
        <developer>
          <id>zsxwing</id>
          <name>Shixiong Zhu</name>
          <url>https://github.com/zsxwing</url>
        </developer>
        <developer>
          <id>scottsand-db</id>
          <name>Scott Sandre</name>
          <url>https://github.com/scottsand-db</url>
        </developer>
        <developer>
          <id>windpiger</id>
          <name>Jun Song</name>
          <url>https://github.com/windpiger</url>
        </developer>
      </developers>
)

// Looks like some of release settings should be set for the root project as well.
publishArtifact := false  // Don't release the root project
publish / skip := true
publishTo := Some("snapshots" at "https://central.sonatype.com/repository/maven-snapshots/")
releaseCrossBuild := false  // Don't use sbt-release's cross facility
releaseProcess := Seq[ReleaseStep](
  checkSnapshotDependencies,
  inquireVersions,
  runTest,
  setReleaseVersion,
  commitReleaseVersion,
  tagRelease
) ++ CrossSparkVersions.crossSparkReleaseSteps("publishSigned") ++
  CrossFlinkVersions.crossFlinkReleaseSteps("publishSigned") ++ Seq[ReleaseStep](

  // Do NOT use `sonatypeBundleRelease` - it will actually release to Maven! We want to do that
  // manually.
  //
  // Do NOT use `sonatypePromote` - it will promote the closed staging repository (i.e. sync to
  //                                Maven central)
  //
  // See https://github.com/xerial/sbt-sonatype#publishing-your-artifact.
  //
  // - sonatypePrepare: Drop the existing staging repositories (if exist) and create a new staging
  //                    repository using sonatypeSessionName as a unique key
  // - sonatypeBundleUpload: Upload your local staging folder contents to a remote Sonatype
  //                         repository
  // - sonatypeClose: closes your staging repository at Sonatype. This step verifies Maven central
  //                  sync requirement, GPG-signature, javadoc and source code presence, pom.xml
  //                  settings, etc
  // TODO: this isn't working yet
  // releaseStepCommand("sonatypePrepare; sonatypeBundleUpload; sonatypeClose"),
  setNextVersion,
  commitNextVersion
)