revert default reader to iceberg reader

apache · Feb 1, 2025 · d36fea0 · d36fea0
1 parent 2f28af7
commit d36fea0
Show file tree

Hide file tree

Showing 7 changed files with 11 additions and 15 deletions.
diff --git a/spark/v3.4/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java b/spark/v3.4/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
@@ -28,7 +28,6 @@
 import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 public class SmokeTest extends SparkExtensionsTestBase {
@@ -45,7 +44,7 @@ public void dropTable() {
   // Run through our Doc's Getting Started Example
   // TODO Update doc example so that it can actually be run, modifications were required for this
   // test suite to run
-  @Ignore
+  @Test
   public void testGettingStarted() throws IOException {
     // Creating a table
     sql("CREATE TABLE %s (id bigint, data string) USING iceberg", tableName);

diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
@@ -29,7 +29,7 @@ private SparkSQLProperties() {}
 
   // Controls which Parquet reader implementation to use
   public static final String PARQUET_READER_TYPE = "spark.sql.iceberg.parquet.reader-type";
-  public static final ParquetReaderType PARQUET_READER_TYPE_DEFAULT = ParquetReaderType.COMET;
+  public static final ParquetReaderType PARQUET_READER_TYPE_DEFAULT = ParquetReaderType.ICEBERG;
 
   // Controls whether reading/writing timestamps without timezones is allowed
   @Deprecated

diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java
@@ -41,7 +41,6 @@
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 public class TestDataFrameWriterV2 extends SparkTestBaseWithCatalog {
@@ -215,7 +214,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars
     Assert.assertEquals(4, fields.size());
   }
 
-  @Ignore
+  @Test
   public void testMergeSchemaIgnoreCastingLongToInt() throws Exception {
     sql(
         "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')",
@@ -255,7 +254,7 @@ public void testMergeSchemaIgnoreCastingLongToInt() throws Exception {
     assertThat(idField.type().typeId()).isEqualTo(Type.TypeID.LONG);
   }
 
-  @Ignore
+  @Test
   public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception {
     removeTables();
     sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName);
@@ -297,7 +296,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception {
     assertThat(idField.type().typeId()).isEqualTo(Type.TypeID.DOUBLE);
   }
 
-  @Ignore
+  @Test
   public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() throws Exception {
     removeTables();
     sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName);

diff --git a/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java b/spark/v3.5/spark-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java
@@ -26,7 +26,6 @@
 import org.apache.iceberg.Table;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.spark.extensions.ExtensionsTestBase;
-import org.junit.Ignore;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.TestTemplate;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -41,7 +40,7 @@ public void dropTable() {
   // Run through our Doc's Getting Started Example
   // TODO Update doc example so that it can actually be run, modifications were required for this
   // test suite to run
-  @Ignore
+  @TestTemplate
   public void testGettingStarted() throws IOException {
     // Creating a table
     sql("CREATE TABLE %s (id bigint, data string) USING iceberg", tableName);

diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
@@ -29,7 +29,7 @@ private SparkSQLProperties() {}
 
   // Controls which Parquet reader implementation to use
   public static final String PARQUET_READER_TYPE = "spark.sql.iceberg.parquet.reader-type";
-  public static final ParquetReaderType PARQUET_READER_TYPE_DEFAULT = ParquetReaderType.COMET;
+  public static final ParquetReaderType PARQUET_READER_TYPE_DEFAULT = ParquetReaderType.ICEBERG;
   // Controls whether to perform the nullability check during writes
   public static final String CHECK_NULLABILITY = "spark.sql.iceberg.check-nullability";
   public static final boolean CHECK_NULLABILITY_DEFAULT = true;

diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java
@@ -40,7 +40,6 @@
 import org.apache.spark.sql.internal.SQLConf;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.TestTemplate;
 
 public class TestDataFrameWriterV2 extends TestBaseWithCatalog {
@@ -249,7 +248,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception {
         sql("select * from %s order by id", tableName));
   }
 
-  @Disabled
+  @TestTemplate
   public void testMergeSchemaIgnoreCastingLongToInt() throws Exception {
     sql(
         "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')",
@@ -289,7 +288,7 @@ public void testMergeSchemaIgnoreCastingLongToInt() throws Exception {
     assertThat(idField.type().typeId()).isEqualTo(Type.TypeID.LONG);
   }
 
-  @Disabled
+  @TestTemplate
   public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception {
     removeTables();
     sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName);
@@ -331,7 +330,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception {
     assertThat(idField.type().typeId()).isEqualTo(Type.TypeID.DOUBLE);
   }
 
-  @Disabled
+  @TestTemplate
   public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() throws Exception {
     removeTables();
     sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName);

diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java
@@ -670,7 +670,7 @@ public void testEqualityDeleteWithDifferentScanAndDeleteColumns() throws IOExcep
     ParquetBatchReadConf conf =
         ImmutableParquetBatchReadConf.builder()
             .batchSize(7)
-            .readerType(ParquetReaderType.COMET)
+            .readerType(ParquetReaderType.ICEBERG)
             .build();
 
     for (CombinedScanTask task : tasks) {