From 5903e1308f0030c328c7b9962337715ea8791aad Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 16:10:27 +0800 Subject: [PATCH 01/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/datafusion-ext-plans/src/orc_exec.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index d07e4794e..b17b22f31 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -365,13 +365,16 @@ impl SchemaAdapter { } } else { for named_column in file_named_columns { + // Case-insensitive field name matching + let named_column_name_lower = named_column.name().to_lowercase(); if let Some((proj_idx, _)) = - self.projected_schema.fields().find(named_column.name()) - { - field_mappings[proj_idx] = Some(projection.len()); - projection.push(named_column.data_type().column_index()); + self.projected_schema.fields().iter().enumerate() + .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } } - } } Ok(( From a1fbc9fc37ac4a23bea5ce80b46b1dd90e9a96c8 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 17:18:57 +0800 Subject: [PATCH 02/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/auron-jni-bridge/src/conf.rs | 1 + .../datafusion-ext-plans/src/orc_exec.rs | 24 +++++++++++++------ .../SparkAuronConfiguration.java | 6 +++++ .../org/apache/spark/sql/auron/AuronConf.java | 2 ++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/native-engine/auron-jni-bridge/src/conf.rs b/native-engine/auron-jni-bridge/src/conf.rs index e99037ad2..c4e32553c 100644 --- a/native-engine/auron-jni-bridge/src/conf.rs +++ b/native-engine/auron-jni-bridge/src/conf.rs @@ -57,6 +57,7 @@ define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE); define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE_KWAY_MERGE); define_conf!(BooleanConf, ORC_FORCE_POSITIONAL_EVOLUTION); define_conf!(BooleanConf, ORC_TIMESTAMP_USE_MICROSECOND); +define_conf!(BooleanConf, ORC_SCHEMA_ISCASE_SENSITIVE) define_conf!(IntConf, UDAF_FALLBACK_NUM_UDAFS_TRIGGER_SORT_AGG); define_conf!(BooleanConf, PARSE_JSON_ERROR_FALLBACK); define_conf!(StringConf, NATIVE_LOG_LEVEL); diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index b17b22f31..5df19631a 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -160,6 +160,7 @@ impl ExecutionPlan for OrcExec { let force_positional_evolution = conf::ORC_FORCE_POSITIONAL_EVOLUTION.value()?; let use_microsecond_precision = conf::ORC_TIMESTAMP_USE_MICROSECOND.value()?; + let is_case_sensitive = conf::ORC_SCHEMA_ISCASE_SENSITIVE.value()?; let opener: Arc = Arc::new(OrcOpener { projection, @@ -364,17 +365,26 @@ impl SchemaAdapter { } } } else { - for named_column in file_named_columns { - // Case-insensitive field name matching - let named_column_name_lower = named_column.name().to_lowercase(); - if let Some((proj_idx, _)) = - self.projected_schema.fields().iter().enumerate() - .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + if is_case_sensitive { + for named_column in file_named_columns { + if let Some((proj_idx, _)) = + self.projected_schema.fields().find(named_column.name()) { field_mappings[proj_idx] = Some(projection.len()); projection.push(named_column.data_type().column_index()); + } + } + } else { + // Case-insensitive field name matching + let named_column_name_lower = named_column.name().to_lowercase(); + if let Some((proj_idx, _)) = + self.projected_schema.fields() + .iter().enumerate().find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } } - } } Ok(( diff --git a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java index 8d29a0bc3..4f94dc389 100644 --- a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java +++ b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java @@ -27,6 +27,7 @@ import org.apache.spark.internal.config.ConfigEntryWithDefault; import scala.Option; import scala.collection.immutable.List$; +import sun.security.pkcs11.P11Util; /** * Spark configuration proxy for Auron. @@ -237,6 +238,11 @@ public class SparkAuronConfiguration extends AuronConfiguration { .description("use microsecond precision when reading ORC timestamp columns. ") .booleanType() .defaultValue(false); + public static final ConfigOption ORC_SCHEMA_ISCASE_SENSITIVE = ConfigOptions.key( + "auron.orc.schema.iscasesensitive") + .description("does matching ORC file schema distinguish between uppercase and lowercase ") + .booleanType() + .defaultValue(false); private final SparkConf sparkConf; diff --git a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java index b3cbca1a5..5911deebd 100644 --- a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java +++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java @@ -139,6 +139,8 @@ public enum AuronConf { // use microsecond precision when reading ORC timestamp columns ORC_TIMESTAMP_USE_MICROSECOND("spark.auron.orc.timestamp.use.microsecond", false), + ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.iscasesensitive", true), + NATIVE_LOG_LEVEL("spark.auron.native.log.level", "info"); public final String key; From b724889b008421d42a41b503d876fc9d2ecc012d Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 17:30:10 +0800 Subject: [PATCH 03/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/auron-jni-bridge/src/conf.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/native-engine/auron-jni-bridge/src/conf.rs b/native-engine/auron-jni-bridge/src/conf.rs index c4e32553c..6ee7886d8 100644 --- a/native-engine/auron-jni-bridge/src/conf.rs +++ b/native-engine/auron-jni-bridge/src/conf.rs @@ -57,7 +57,7 @@ define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE); define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE_KWAY_MERGE); define_conf!(BooleanConf, ORC_FORCE_POSITIONAL_EVOLUTION); define_conf!(BooleanConf, ORC_TIMESTAMP_USE_MICROSECOND); -define_conf!(BooleanConf, ORC_SCHEMA_ISCASE_SENSITIVE) +define_conf!(BooleanConf, ORC_SCHEMA_ISCASE_SENSITIVE); define_conf!(IntConf, UDAF_FALLBACK_NUM_UDAFS_TRIGGER_SORT_AGG); define_conf!(BooleanConf, PARSE_JSON_ERROR_FALLBACK); define_conf!(StringConf, NATIVE_LOG_LEVEL); From 7e6a629de463485f311292f0f006f24113b52e47 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 17:41:59 +0800 Subject: [PATCH 04/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/datafusion-ext-plans/src/orc_exec.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 5df19631a..7a6c8f716 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -171,6 +171,7 @@ impl ExecutionPlan for OrcExec { metrics: self.metrics.clone(), force_positional_evolution, use_microsecond_precision, + is_case_sensitive }); let file_stream = Box::pin(FileStream::new( @@ -218,6 +219,7 @@ struct OrcOpener { metrics: ExecutionPlanMetricsSet, force_positional_evolution: bool, use_microsecond_precision: bool, + is_case_sensitive: bool } impl FileOpener for OrcOpener { @@ -260,7 +262,7 @@ impl FileOpener for OrcOpener { } let (schema_mapping, projection) = - schema_adapter.map_schema(builder.file_metadata())?; + schema_adapter.map_schema(builder.file_metadata(), is_case_sensitive)?; let projection_mask = ProjectionMask::roots(builder.file_metadata().root_data_type(), projection); @@ -326,6 +328,7 @@ impl SchemaAdapter { fn map_schema( &self, orc_file_meta: &FileMetadata, + is_case_sensitive: bool ) -> Result<(Arc, Vec)> { let mut projection = Vec::with_capacity(self.projected_schema.fields().len()); let mut field_mappings = vec![None; self.projected_schema.fields().len()]; From 271fc41614b4ab1c1714ec37fc8f3834dacba0c8 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 17:50:52 +0800 Subject: [PATCH 05/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/datafusion-ext-plans/src/orc_exec.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 7a6c8f716..1457ac0e2 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -248,6 +248,7 @@ impl FileOpener for OrcOpener { self.force_positional_evolution, ); let use_microsecond = self.use_microsecond_precision; + let is_case = self.is_case_sensitive; Ok(Box::pin(async move { let mut builder = ArrowReaderBuilder::try_new_async(reader) @@ -262,7 +263,7 @@ impl FileOpener for OrcOpener { } let (schema_mapping, projection) = - schema_adapter.map_schema(builder.file_metadata(), is_case_sensitive)?; + schema_adapter.map_schema(builder.file_metadata(), is_case)?; let projection_mask = ProjectionMask::roots(builder.file_metadata().root_data_type(), projection); @@ -382,7 +383,9 @@ impl SchemaAdapter { let named_column_name_lower = named_column.name().to_lowercase(); if let Some((proj_idx, _)) = self.projected_schema.fields() - .iter().enumerate().find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + .iter() + .enumerate() + .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) { field_mappings[proj_idx] = Some(projection.len()); projection.push(named_column.data_type().column_index()); From 440c897522e7ec651b86ee46e3886f4c7c93f932 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Tue, 23 Dec 2025 17:56:57 +0800 Subject: [PATCH 06/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/datafusion-ext-plans/src/orc_exec.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 1457ac0e2..b897ff6f7 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -379,6 +379,7 @@ impl SchemaAdapter { } } } else { + for named_column in file_named_columns { // Case-insensitive field name matching let named_column_name_lower = named_column.name().to_lowercase(); if let Some((proj_idx, _)) = @@ -391,6 +392,7 @@ impl SchemaAdapter { projection.push(named_column.data_type().column_index()); } } + } } Ok(( From a32f4f148fa8a5113f8e956153a208b57c9f9a61 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Wed, 24 Dec 2025 09:42:41 +0800 Subject: [PATCH 07/12] Fix the ORC table written by Hive to read null uppercase fields --- .../datafusion-ext-plans/src/orc_exec.rs | 50 +++++++++---------- .../SparkAuronConfiguration.java | 2 - .../org/apache/spark/sql/auron/AuronConf.java | 2 +- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index b897ff6f7..6e01dc8bf 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -171,7 +171,7 @@ impl ExecutionPlan for OrcExec { metrics: self.metrics.clone(), force_positional_evolution, use_microsecond_precision, - is_case_sensitive + is_case_sensitive, }); let file_stream = Box::pin(FileStream::new( @@ -370,30 +370,30 @@ impl SchemaAdapter { } } else { if is_case_sensitive { - for named_column in file_named_columns { - if let Some((proj_idx, _)) = - self.projected_schema.fields().find(named_column.name()) - { - field_mappings[proj_idx] = Some(projection.len()); - projection.push(named_column.data_type().column_index()); - } - } - } else { - for named_column in file_named_columns { - // Case-insensitive field name matching - let named_column_name_lower = named_column.name().to_lowercase(); - if let Some((proj_idx, _)) = - self.projected_schema.fields() - .iter() - .enumerate() - .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) - { - field_mappings[proj_idx] = Some(projection.len()); - projection.push(named_column.data_type().column_index()); - } - } - } - } + for named_column in file_named_columns { + if let Some((proj_idx, _)) = + self.projected_schema.fields().find(named_column.name()) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } + } + } else { + for named_column in file_named_columns { + // Case-insensitive field name matching + let named_column_name_lower = named_column.name().to_lowercase(); + if let Some((proj_idx, _)) = + self.projected_schema.fields() + .iter() + .enumerate() + .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } + } + } + } Ok(( create_auron_schema_mapper(&self.projected_schema, &field_mappings), diff --git a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java index 4f94dc389..5c85821de 100644 --- a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java +++ b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java @@ -27,8 +27,6 @@ import org.apache.spark.internal.config.ConfigEntryWithDefault; import scala.Option; import scala.collection.immutable.List$; -import sun.security.pkcs11.P11Util; - /** * Spark configuration proxy for Auron. * All configuration prefixes start with spark. diff --git a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java index 5911deebd..f033fd0e2 100644 --- a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java +++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java @@ -139,7 +139,7 @@ public enum AuronConf { // use microsecond precision when reading ORC timestamp columns ORC_TIMESTAMP_USE_MICROSECOND("spark.auron.orc.timestamp.use.microsecond", false), - ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.iscasesensitive", true), + ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.iscasesensitive", false), NATIVE_LOG_LEVEL("spark.auron.native.log.level", "info"); From d6e7a080968c8bfe285e5a048f637951acf9b1f0 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Wed, 24 Dec 2025 10:44:45 +0800 Subject: [PATCH 08/12] Fix the ORC table written by Hive to read null uppercase fields --- .../datafusion-ext-plans/src/orc_exec.rs | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 6e01dc8bf..8b750dcb5 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -368,32 +368,31 @@ impl SchemaAdapter { } } } + } else if is_case_sensitive { + for named_column in file_named_columns { + if let Some((proj_idx, _)) = + self.projected_schema.fields().find(named_column.name()) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } + } } else { - if is_case_sensitive { - for named_column in file_named_columns { - if let Some((proj_idx, _)) = - self.projected_schema.fields().find(named_column.name()) - { - field_mappings[proj_idx] = Some(projection.len()); - projection.push(named_column.data_type().column_index()); - } - } - } else { - for named_column in file_named_columns { - // Case-insensitive field name matching - let named_column_name_lower = named_column.name().to_lowercase(); - if let Some((proj_idx, _)) = - self.projected_schema.fields() - .iter() - .enumerate() - .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) - { - field_mappings[proj_idx] = Some(projection.len()); - projection.push(named_column.data_type().column_index()); - } - } - } - } + for named_column in file_named_columns { + // Case-insensitive field name matching + let named_column_name_lower = named_column.name().to_lowercase(); + if let Some((proj_idx, _)) = self + .projected_schema + .fields() + .iter() + .enumerate() + .find(|(_, f)| f.name().to_lowercase() == named_column_name_lower) + { + field_mappings[proj_idx] = Some(projection.len()); + projection.push(named_column.data_type().column_index()); + } + } + } Ok(( create_auron_schema_mapper(&self.projected_schema, &field_mappings), From 0675b0b52ab0cb8297156abd24f6faff90d900dc Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Wed, 24 Dec 2025 10:50:16 +0800 Subject: [PATCH 09/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/datafusion-ext-plans/src/orc_exec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 8b750dcb5..14fcf0cc7 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -219,7 +219,7 @@ struct OrcOpener { metrics: ExecutionPlanMetricsSet, force_positional_evolution: bool, use_microsecond_precision: bool, - is_case_sensitive: bool + is_case_sensitive: bool, } impl FileOpener for OrcOpener { @@ -329,7 +329,7 @@ impl SchemaAdapter { fn map_schema( &self, orc_file_meta: &FileMetadata, - is_case_sensitive: bool + is_case_sensitive: bool, ) -> Result<(Arc, Vec)> { let mut projection = Vec::with_capacity(self.projected_schema.fields().len()); let mut field_mappings = vec![None; self.projected_schema.fields().len()]; From 715c12dca84a651f780378c5f4c8bb9c295f5387 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Wed, 24 Dec 2025 11:01:00 +0800 Subject: [PATCH 10/12] Fix the ORC table written by Hive to read null uppercase fields --- .../auron/spark/configuration/SparkAuronConfiguration.java | 1 + 1 file changed, 1 insertion(+) diff --git a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java index 5c85821de..d1144bfa7 100644 --- a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java +++ b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java @@ -27,6 +27,7 @@ import org.apache.spark.internal.config.ConfigEntryWithDefault; import scala.Option; import scala.collection.immutable.List$; + /** * Spark configuration proxy for Auron. * All configuration prefixes start with spark. From 53cf3222b016e87c2cc50cf74519348476e57f47 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Fri, 26 Dec 2025 10:21:17 +0800 Subject: [PATCH 11/12] Fix the ORC table written by Hive to read null uppercase fields --- .../auron/spark/configuration/SparkAuronConfiguration.java | 4 ++-- .../src/main/java/org/apache/spark/sql/auron/AuronConf.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java index d1144bfa7..e9cc2d5c7 100644 --- a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java +++ b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java @@ -238,8 +238,8 @@ public class SparkAuronConfiguration extends AuronConfiguration { .booleanType() .defaultValue(false); public static final ConfigOption ORC_SCHEMA_ISCASE_SENSITIVE = ConfigOptions.key( - "auron.orc.schema.iscasesensitive") - .description("does matching ORC file schema distinguish between uppercase and lowercase ") + "auron.orc.schema.caseSensitive.enable") + .description("whether ORC file schema matching distinguishes between uppercase and lowercase. ") .booleanType() .defaultValue(false); diff --git a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java index f033fd0e2..bf113b154 100644 --- a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java +++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java @@ -139,7 +139,7 @@ public enum AuronConf { // use microsecond precision when reading ORC timestamp columns ORC_TIMESTAMP_USE_MICROSECOND("spark.auron.orc.timestamp.use.microsecond", false), - ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.iscasesensitive", false), + ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.caseSensitive.enable", false), NATIVE_LOG_LEVEL("spark.auron.native.log.level", "info"); From 9d981c53ed43de18e9657ddc0dc7fe4fa73641a4 Mon Sep 17 00:00:00 2001 From: duanhao-jk Date: Fri, 26 Dec 2025 15:40:37 +0800 Subject: [PATCH 12/12] Fix the ORC table written by Hive to read null uppercase fields --- native-engine/auron-jni-bridge/src/conf.rs | 2 +- native-engine/datafusion-ext-plans/src/orc_exec.rs | 2 +- .../auron/spark/configuration/SparkAuronConfiguration.java | 2 +- .../src/main/java/org/apache/spark/sql/auron/AuronConf.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/native-engine/auron-jni-bridge/src/conf.rs b/native-engine/auron-jni-bridge/src/conf.rs index 6ee7886d8..383596d60 100644 --- a/native-engine/auron-jni-bridge/src/conf.rs +++ b/native-engine/auron-jni-bridge/src/conf.rs @@ -57,7 +57,7 @@ define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE); define_conf!(IntConf, SUGGESTED_BATCH_MEM_SIZE_KWAY_MERGE); define_conf!(BooleanConf, ORC_FORCE_POSITIONAL_EVOLUTION); define_conf!(BooleanConf, ORC_TIMESTAMP_USE_MICROSECOND); -define_conf!(BooleanConf, ORC_SCHEMA_ISCASE_SENSITIVE); +define_conf!(BooleanConf, ORC_SCHEMA_CASE_SENSITIVE); define_conf!(IntConf, UDAF_FALLBACK_NUM_UDAFS_TRIGGER_SORT_AGG); define_conf!(BooleanConf, PARSE_JSON_ERROR_FALLBACK); define_conf!(StringConf, NATIVE_LOG_LEVEL); diff --git a/native-engine/datafusion-ext-plans/src/orc_exec.rs b/native-engine/datafusion-ext-plans/src/orc_exec.rs index 14fcf0cc7..c53cb6b5f 100644 --- a/native-engine/datafusion-ext-plans/src/orc_exec.rs +++ b/native-engine/datafusion-ext-plans/src/orc_exec.rs @@ -160,7 +160,7 @@ impl ExecutionPlan for OrcExec { let force_positional_evolution = conf::ORC_FORCE_POSITIONAL_EVOLUTION.value()?; let use_microsecond_precision = conf::ORC_TIMESTAMP_USE_MICROSECOND.value()?; - let is_case_sensitive = conf::ORC_SCHEMA_ISCASE_SENSITIVE.value()?; + let is_case_sensitive = conf::ORC_SCHEMA_CASE_SENSITIVE.value()?; let opener: Arc = Arc::new(OrcOpener { projection, diff --git a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java index e9cc2d5c7..88f494142 100644 --- a/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java +++ b/spark-extension/src/main/java/org/apache/auron/spark/configuration/SparkAuronConfiguration.java @@ -237,7 +237,7 @@ public class SparkAuronConfiguration extends AuronConfiguration { .description("use microsecond precision when reading ORC timestamp columns. ") .booleanType() .defaultValue(false); - public static final ConfigOption ORC_SCHEMA_ISCASE_SENSITIVE = ConfigOptions.key( + public static final ConfigOption ORC_SCHEMA_CASE_SENSITIVE = ConfigOptions.key( "auron.orc.schema.caseSensitive.enable") .description("whether ORC file schema matching distinguishes between uppercase and lowercase. ") .booleanType() diff --git a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java index bf113b154..2943d4b2a 100644 --- a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java +++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java @@ -139,7 +139,7 @@ public enum AuronConf { // use microsecond precision when reading ORC timestamp columns ORC_TIMESTAMP_USE_MICROSECOND("spark.auron.orc.timestamp.use.microsecond", false), - ORC_SCHEMA_ISCASE_SENSITIVE("spark.auron.orc.schema.caseSensitive.enable", false), + ORC_SCHEMA_CASE_SENSITIVE("spark.auron.orc.schema.caseSensitive.enable", false), NATIVE_LOG_LEVEL("spark.auron.native.log.level", "info");