From f3bf716a2318ddb3c251590c410f5dbd5d231906 Mon Sep 17 00:00:00 2001 From: marvelshan Date: Wed, 17 Jun 2026 02:08:31 +0000 Subject: [PATCH 1/2] fix: classify unsupported format patterns as Unsupported in CometFromUnixTime CometFromUnixTime previously reported all limitations as Incompatible, but non-default format patterns cannot be executed natively at all and should be Unsupported. The DataFusion timestamp range difference is the genuine Incompatible case. - Split getIncompatibleReasons to only contain the timestamp range issue - Add getUnsupportedReasons for the format pattern limitation - Make getSupportLevel return Unsupported for non-default patterns, Incompatible for the default pattern --- .../scala/org/apache/comet/serde/unixtime.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala index 1a3a4ca677..9851437f6e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala @@ -30,11 +30,19 @@ import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, optExprWithFa object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] with CodegenDispatchFallback { override def getIncompatibleReasons(): Seq[String] = Seq( - "Only supports the default datetime format pattern `yyyy-MM-dd HH:mm:ss`." + - " DataFusion's valid timestamp range differs from Spark" + + "DataFusion's valid timestamp range differs from Spark" + " (https://github.com/apache/datafusion/issues/16594)") - override def getSupportLevel(expr: FromUnixTime): SupportLevel = Incompatible(None) + override def getUnsupportedReasons(): Seq[String] = Seq( + "Only the default datetime format pattern `yyyy-MM-dd HH:mm:ss` is supported.") + + override def getSupportLevel(expr: FromUnixTime): SupportLevel = { + if (expr.format != Literal(TimestampFormatter.defaultPattern())) { + Unsupported(Some("Only the default datetime pattern `yyyy-MM-dd HH:mm:ss` is supported")) + } else { + Incompatible(None) + } + } override def convert( expr: FromUnixTime, From 35ebc98a2a8d27ab6695a770ffb9e12f725d3817 Mon Sep 17 00:00:00 2001 From: marvelshan Date: Wed, 17 Jun 2026 02:08:40 +0000 Subject: [PATCH 2/2] test: add SQL tests for from_unixtime fallback on non-default format patterns Verify that non-default format patterns fall back to Spark with the expected reason, and that the default pattern runs natively when allowIncompatible=true. --- spark/src/main/scala/org/apache/comet/serde/unixtime.scala | 7 +++++-- .../sql-tests/expressions/datetime/from_unix_time.sql | 4 ++-- .../expressions/datetime/from_unix_time_enabled.sql | 7 ++++--- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala index 9851437f6e..b37c8404c6 100644 --- a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala +++ b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala @@ -34,11 +34,14 @@ object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] with Codegen " (https://github.com/apache/datafusion/issues/16594)") override def getUnsupportedReasons(): Seq[String] = Seq( - "Only the default datetime format pattern `yyyy-MM-dd HH:mm:ss` is supported.") + "Only the default datetime format pattern `yyyy-MM-dd HH:mm:ss` is supported natively." + + " Non-default patterns are handled by the JVM codegen dispatcher when" + + " `spark.comet.exec.scalaUDF.codegen.enabled=true`.") override def getSupportLevel(expr: FromUnixTime): SupportLevel = { if (expr.format != Literal(TimestampFormatter.defaultPattern())) { - Unsupported(Some("Only the default datetime pattern `yyyy-MM-dd HH:mm:ss` is supported")) + Incompatible( + Some("Only the default datetime pattern `yyyy-MM-dd HH:mm:ss` is supported natively")) } else { Incompatible(None) } diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql index 66b94307ab..1cdf82dfab 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time.sql @@ -26,12 +26,12 @@ INSERT INTO test_from_unix_time VALUES (0), (1718451045), (-1), (NULL), (2147483 query SELECT from_unixtime(t) FROM test_from_unix_time -query +query spark_answer_only SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time -- literal arguments query SELECT from_unixtime(0) -query +query spark_answer_only SELECT from_unixtime(1718451045, 'yyyy-MM-dd') diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql index 1821a6a01e..709476951e 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/from_unix_time_enabled.sql @@ -26,15 +26,16 @@ CREATE TABLE test_from_unix_time_enabled(t long) USING parquet statement INSERT INTO test_from_unix_time_enabled VALUES (0), (1718451045), (-1), (NULL), (2147483647) --- Even with allowIncompatible=true, the default datetime pattern is unsupported natively -query spark_answer_only +-- With allowIncompatible=true, the default datetime pattern runs natively +query SELECT from_unixtime(t) FROM test_from_unix_time_enabled +-- Non-default format patterns are handled by the JVM codegen dispatcher query spark_answer_only SELECT from_unixtime(t, 'yyyy-MM-dd') FROM test_from_unix_time_enabled -- literal arguments -query spark_answer_only +query SELECT from_unixtime(0) query spark_answer_only