From 582d63babab717e4dbe99086bfa809ea4bf2df9f Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Sun, 21 Jun 2026 00:11:19 +0800 Subject: [PATCH 1/4] [spark] Expose external table type to Spark --- .../paimon/spark/PaimonSparkTableBase.scala | 6 ++++- .../spark/SparkCatalogWithHiveTest.java | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala index bd69174be712..017d7f6e1988 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala @@ -21,12 +21,13 @@ package org.apache.paimon.spark import org.apache.paimon.CoreOptions import org.apache.paimon.CoreOptions.BucketFunctionType import org.apache.paimon.options.Options +import org.apache.paimon.options.CatalogOptions.TABLE_TYPE import org.apache.paimon.spark.catalog.functions.BucketFunction import org.apache.paimon.spark.read.PaimonSplitScanBuilder import org.apache.paimon.spark.schema.PaimonMetadataColumn import org.apache.paimon.spark.util.OptionUtils import org.apache.paimon.spark.write.{PaimonV2WriteBuilder, PaimonWriteBuilder} -import org.apache.paimon.table.{Table, _} +import org.apache.paimon.table.{CatalogTableType, Table, _} import org.apache.paimon.table.BucketMode.{BUCKET_UNAWARE, HASH_FIXED, POSTPONE_MODE} import org.apache.spark.sql.connector.catalog._ @@ -84,6 +85,9 @@ abstract class PaimonSparkTableBase(val table: Table) if (properties.containsKey(CoreOptions.PATH.key())) { properties.put(TableCatalog.PROP_LOCATION, properties.get(CoreOptions.PATH.key())) } + if (CatalogTableType.EXTERNAL.toString.equalsIgnoreCase(dataTable.options().get(TABLE_TYPE.key()))) { + properties.put(TableCatalog.PROP_EXTERNAL, "true") + } properties case _ => Collections.emptyMap() } diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java index 583522822b9e..c56705c45418 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java @@ -156,6 +156,30 @@ public void testCreateExternalTable() throws IOException { } } + @Test + public void testDescribeExternalAndManagedTableType() throws IOException { + try (SparkSession spark = createSessionBuilder().getOrCreate()) { + spark.sql("CREATE DATABASE IF NOT EXISTS test_db"); + spark.sql("USE spark_catalog.test_db"); + + spark.sql("CREATE EXTERNAL TABLE external_table (a INT, bb INT, c STRING)"); + assertThat( + spark.sql("DESC FORMATTED external_table") + .filter("col_name = 'Type'") + .head() + .getString(1)) + .isEqualTo("EXTERNAL"); + + spark.sql("CREATE TABLE managed_table (a INT)"); + assertThat( + spark.sql("DESC FORMATTED managed_table") + .filter("col_name = 'Type'") + .head() + .getString(1)) + .isEqualTo("MANAGED"); + } + } + private SparkSession.Builder createSessionBuilder() { Path warehousePath = new Path("file:" + tempDir.toString()); return SparkSession.builder() From 0bd05b5333fa229d89038463a61bf6c0112f9590 Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Sun, 21 Jun 2026 08:19:23 +0800 Subject: [PATCH 2/4] Fix code style --- .../org/apache/paimon/spark/PaimonSparkTableBase.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala index 017d7f6e1988..6be314cee847 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonSparkTableBase.scala @@ -20,8 +20,8 @@ package org.apache.paimon.spark import org.apache.paimon.CoreOptions import org.apache.paimon.CoreOptions.BucketFunctionType -import org.apache.paimon.options.Options import org.apache.paimon.options.CatalogOptions.TABLE_TYPE +import org.apache.paimon.options.Options import org.apache.paimon.spark.catalog.functions.BucketFunction import org.apache.paimon.spark.read.PaimonSplitScanBuilder import org.apache.paimon.spark.schema.PaimonMetadataColumn @@ -85,7 +85,10 @@ abstract class PaimonSparkTableBase(val table: Table) if (properties.containsKey(CoreOptions.PATH.key())) { properties.put(TableCatalog.PROP_LOCATION, properties.get(CoreOptions.PATH.key())) } - if (CatalogTableType.EXTERNAL.toString.equalsIgnoreCase(dataTable.options().get(TABLE_TYPE.key()))) { + if ( + CatalogTableType.EXTERNAL.toString.equalsIgnoreCase( + dataTable.options().get(TABLE_TYPE.key())) + ) { properties.put(TableCatalog.PROP_EXTERNAL, "true") } properties From 35898de12906e02fe8f49d581826c589b9c50d84 Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Tue, 23 Jun 2026 19:06:53 +0800 Subject: [PATCH 3/4] [spark] Fix test case --- .../org/apache/paimon/spark/SparkCatalogWithHiveTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java index c56705c45418..07f2ed151d79 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java @@ -162,17 +162,17 @@ public void testDescribeExternalAndManagedTableType() throws IOException { spark.sql("CREATE DATABASE IF NOT EXISTS test_db"); spark.sql("USE spark_catalog.test_db"); - spark.sql("CREATE EXTERNAL TABLE external_table (a INT, bb INT, c STRING)"); + spark.sql("CREATE EXTERNAL TABLE external_type_table (a INT, bb INT, c STRING)"); assertThat( - spark.sql("DESC FORMATTED external_table") + spark.sql("DESC FORMATTED external_type_table") .filter("col_name = 'Type'") .head() .getString(1)) .isEqualTo("EXTERNAL"); - spark.sql("CREATE TABLE managed_table (a INT)"); + spark.sql("CREATE TABLE managed_type_table (a INT)"); assertThat( - spark.sql("DESC FORMATTED managed_table") + spark.sql("DESC FORMATTED managed_type_table") .filter("col_name = 'Type'") .head() .getString(1)) From 0a518e15fcac1a14888f23e515f85014be7c4916 Mon Sep 17 00:00:00 2001 From: huangxiaoping <1754789345@qq.com> Date: Tue, 23 Jun 2026 20:11:04 +0800 Subject: [PATCH 4/4] [spark] Isolate external table type test database --- .../org/apache/paimon/spark/SparkCatalogWithHiveTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java index 07f2ed151d79..f71e5df3c08f 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkCatalogWithHiveTest.java @@ -159,8 +159,8 @@ public void testCreateExternalTable() throws IOException { @Test public void testDescribeExternalAndManagedTableType() throws IOException { try (SparkSession spark = createSessionBuilder().getOrCreate()) { - spark.sql("CREATE DATABASE IF NOT EXISTS test_db"); - spark.sql("USE spark_catalog.test_db"); + spark.sql("CREATE DATABASE IF NOT EXISTS type_test_db"); + spark.sql("USE spark_catalog.type_test_db"); spark.sql("CREATE EXTERNAL TABLE external_type_table (a INT, bb INT, c STRING)"); assertThat(