Skip to content

feat: remove constraint on array of nested elements #4714

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hsiang-c wants to merge 4 commits into
base: main
Choose a base branch
from
Open

feat: remove constraint on array of nested elements #4714

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8bab0c7
Remove constraint on array of nested elements
hsiang-c Jun 23, 2026
a2f5e4f
Merge branch 'main' into sort_nested_array
hsiang-c Jun 24, 2026
b04f537
Remove unsupported reasons
hsiang-c Jun 25, 2026
844ede5
Merge branch 'main' into sort_nested_array
hsiang-c Jun 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 4 additions & 13 deletions spark/src/main/scala/org/apache/comet/serde/arrays.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,22 +120,13 @@ object CometSortArray extends CometExpressionSerde[SortArray] with CodegenDispat
"When `" + CometConf.COMET_EXEC_STRICT_FLOATING_POINT.key + "=true`, sorting on" +
" floating-point types is not 100% compatible with Spark")

override def getUnsupportedReasons(): Seq[String] = Seq(
"Nested arrays with `Struct` or `Null` child values are not supported natively and will" +
" fall back to Spark.")

private def supportedSortArrayElementType(
dt: DataType,
nestedInArray: Boolean = false): Boolean = {
private def supportedSortArrayElementType(dt: DataType): Boolean = {
dt match {
// DataFusion's array_sort compares nested arrays through Arrow's rank kernel.
// That kernel does not support Struct or Null child values,
// so array<array<struct<...>>> and array<array<null>> would fail at runtime.
case _: NullType if !nestedInArray =>
case _: NullType =>
true
case ArrayType(elementType, _) =>
supportedSortArrayElementType(elementType, nestedInArray = true)
case StructType(fields) if !nestedInArray =>
supportedSortArrayElementType(elementType)
case StructType(fields) =>
fields.forall(f => supportedSortArrayElementType(f.dataType))
case _ =>
supportedScalarSortElementType(dt)
Expand Down
6 changes: 3 additions & 3 deletions spark/src/test/resources/sql-tests/expressions/array/sort_array.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,10 +295,10 @@ INSERT INTO test_sort_array_nested_struct VALUES
(array()),
(NULL)

query expect_fallback(Sort on array element type ArrayType(StructType(StructField(a,IntegerType)
query
SELECT sort_array(arr) FROM test_sort_array_nested_struct

query expect_fallback(Sort on array element type ArrayType(StructType(StructField(a,IntegerType)
query
SELECT sort_array(arr, false) FROM test_sort_array_nested_struct

-- literal arguments
Expand Down Expand Up @@ -391,7 +391,7 @@ SELECT
sort_array(array(NULL, NULL)),
sort_array(cast(NULL as array<int>))

query expect_fallback(Sort on array element type ArrayType(StructType(StructField(a,IntegerType)
query
SELECT sort_array(
array(
array(named_struct('a', 2)),
Expand Down