Skip to content

Commit 115ad5a

Browse files
[Cherry-pick to branch-1.3] [#9374] fix(spark): Fix MySQL datetime filter pushdown in Spark JDBC connector (#11451) (#11508)
**Cherry-pick Information:** - Original commit: eb2e2c9 - Target branch: `branch-1.3` - Status: ✅ Clean cherry-pick (no conflicts) Co-authored-by: Yuhui <hui@datastrato.com>
1 parent f76124e commit 115ad5a

3 files changed

Lines changed: 62 additions & 10 deletions

File tree

‎spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/jdbc/SparkJdbcMysqlCatalogIT.java‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
*
1010
* http://www.apache.org/licenses/LICENSE-2.0
1111
*
12-
* Unless required by applicable law or agreed to in writing,
13-
* software distributed under the License is distributed on an
14-
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15-
* KIND, either express or implied. See the License for the
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
15+
* either express or implied. See the License for the
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
@@ -21,14 +21,20 @@
2121
import static org.apache.gravitino.integration.test.util.TestDatabaseName.MYSQL_CATALOG_MYSQL_IT;
2222

2323
import com.google.common.collect.Maps;
24+
import java.sql.Connection;
25+
import java.sql.DriverManager;
26+
import java.sql.Statement;
27+
import java.util.List;
2428
import java.util.Map;
2529
import org.apache.gravitino.credential.CredentialConstants;
2630
import org.apache.gravitino.credential.JdbcCredential;
2731
import org.apache.gravitino.integration.test.container.ContainerSuite;
2832
import org.apache.gravitino.spark.connector.integration.test.SparkCommonIT;
2933
import org.apache.gravitino.spark.connector.integration.test.util.SparkTableInfoChecker;
3034
import org.apache.gravitino.spark.connector.jdbc.JdbcPropertiesConstants;
35+
import org.junit.jupiter.api.Assertions;
3136
import org.junit.jupiter.api.Tag;
37+
import org.junit.jupiter.api.Test;
3238

3339
@Tag("gravitino-docker-test")
3440
public abstract class SparkJdbcMysqlCatalogIT extends SparkCommonIT {
@@ -114,4 +120,45 @@ protected Map<String, String> getCatalogConfigs() {
114120
CredentialConstants.CREDENTIAL_PROVIDERS, JdbcCredential.JDBC_CREDENTIAL_TYPE);
115121
return catalogProperties;
116122
}
123+
124+
@Test
125+
void testDatetimeFilterPushdown() throws Exception {
126+
// Regression test for https://github.com/apache/gravitino/issues/9374:
127+
// MySQL datetime columns must be usable as filter predicates via the Gravitino Spark JDBC
128+
// catalog. Previously, datetime was mapped to TimestampNTZType (Spark 3.4+), causing the
129+
// MySQL JDBC dialect to generate invalid SQL literals during filter pushdown.
130+
//
131+
// The table is created directly via JDBC to use MySQL-native datetime type, exactly
132+
// reproducing the original bug scenario where tables exist with datetime columns in MySQL.
133+
//
134+
// Note: Uses subquery filter to avoid timezone conversion issues between JVM (UTC+8) and
135+
// MySQL container (UTC). The subquery approach validates filter pushdown functionality
136+
// while being resilient to timezone configuration differences in test environments.
137+
String db = getDefaultDatabase();
138+
String tableName = "datetime_filter_test";
139+
String jdbcUrl = mysqlUrl + "/" + db;
140+
try (Connection conn = DriverManager.getConnection(jdbcUrl, mysqlUsername, mysqlPassword);
141+
Statement stmt = conn.createStatement()) {
142+
stmt.execute("DROP TABLE IF EXISTS " + tableName);
143+
stmt.execute("CREATE TABLE " + tableName + " (id INT, create_time DATETIME)");
144+
stmt.execute(
145+
"INSERT INTO "
146+
+ tableName
147+
+ " VALUES"
148+
+ " (1, '2025-12-01 08:00:00'),"
149+
+ " (2, '2025-12-04 09:07:43'),"
150+
+ " (3, '2025-12-10 12:00:00')");
151+
}
152+
153+
// Use subquery filter to avoid timezone-related assertion failures
154+
// This validates filter pushdown works correctly after the TimestampType fix
155+
List<String> result =
156+
getQueryData(
157+
String.format(
158+
"SELECT id FROM %s WHERE create_time >= (SELECT create_time FROM %s WHERE id = 2) ORDER BY id",
159+
tableName, tableName));
160+
Assertions.assertEquals(2, result.size());
161+
Assertions.assertTrue(result.contains("2"));
162+
Assertions.assertTrue(result.contains("3"));
163+
}
117164
}

‎spark-connector/v3.4/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter34.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99
*
1010
* http://www.apache.org/licenses/LICENSE-2.0
1111
*
12-
* Unless required by applicable law or agreed to in writing,
13-
* software distributed under the License is distributed on an
14-
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15-
* KIND, either express or implied. See the License for the
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
15+
* either express or implied. See the License for the
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
@@ -32,6 +32,12 @@ public DataType toSparkType(Type gravitinoType) {
3232
// type varchar.
3333
if (gravitinoType instanceof Types.VarCharType) {
3434
return DataTypes.StringType;
35+
} else if (gravitinoType instanceof Types.TimestampType
36+
&& !((Types.TimestampType) gravitinoType).hasTimeZone()) {
37+
// MySQL datetime (no timezone) must map to TimestampType, not TimestampNTZType, for JDBC
38+
// filter pushdown compatibility. The MySQL JDBC dialect only handles TimestampType literals
39+
// correctly; TimestampNTZType produces invalid SQL syntax errors during filter pushdown.
40+
return DataTypes.TimestampType;
3541
} else {
3642
return super.toSparkType(gravitinoType);
3743
}

‎spark-connector/v3.5/spark/src/main/java/org/apache/gravitino/spark/connector/jdbc/GravitinoJdbcCatalogSpark35.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.gravitino.spark.connector.SparkTableChangeConverter;
2424
import org.apache.gravitino.spark.connector.SparkTableChangeConverter34;
2525
import org.apache.gravitino.spark.connector.SparkTypeConverter;
26-
import org.apache.gravitino.spark.connector.SparkTypeConverter34;
2726
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
2827
import org.apache.spark.sql.connector.catalog.Identifier;
2928
import org.apache.spark.sql.connector.catalog.Table;
@@ -33,7 +32,7 @@ public class GravitinoJdbcCatalogSpark35 extends GravitinoJdbcCatalog {
3332

3433
@Override
3534
protected SparkTypeConverter getSparkTypeConverter() {
36-
return new SparkTypeConverter34();
35+
return new SparkJdbcTypeConverter34();
3736
}
3837

3938
@Override

0 commit comments

Comments
 (0)