diff --git a/arrow-data-source/common/pom.xml b/arrow-data-source/common/pom.xml
index d52187c40..03e44c46c 100644
--- a/arrow-data-source/common/pom.xml
+++ b/arrow-data-source/common/pom.xml
@@ -2,15 +2,15 @@
-
- spark-arrow-datasource
- com.intel.oap
- 1.4.0-SNAPSHOT
- ../pom.xml
-
+
+ spark-arrow-datasource
+ com.intel.oap
+ 1.4.0-SNAPSHOT
+ ../pom.xml
+
4.0.0
- spark-arrow-datasource-common
+ spark-arrow-datasource-common-spark${spark.version}
@@ -23,6 +23,7 @@
org.apache.arrow
arrow-dataset
${arrow.version}
+ compile
io.netty
@@ -41,13 +42,39 @@
jackson-annotations
- compile
+
${project.basedir}/src/main/scala
${project.basedir}/src/test/scala
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.0
+
+
+ process-sources
+
+ compile
+ testCompile
+
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+
org.apache.maven.plugins
maven-source-plugin
@@ -95,6 +122,27 @@
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.2.0
+
+
+ attach-javadocs
+ deploy
+
+ jar
+
+
+
+
+ UTF-8
+ UTF-8
+
+ -Xdoclint:none
+
+
+
diff --git a/arrow-data-source/parquet/pom.xml b/arrow-data-source/parquet/pom.xml
index 28c38e76a..ec60b5e10 100644
--- a/arrow-data-source/parquet/pom.xml
+++ b/arrow-data-source/parquet/pom.xml
@@ -2,32 +2,70 @@
-
- spark-arrow-datasource
- com.intel.oap
- 1.4.0-SNAPSHOT
-
- 4.0.0
+
+ spark-arrow-datasource
+ com.intel.oap
+ 1.4.0-SNAPSHOT
+
+ 4.0.0
- spark-arrow-datasource-parquet
+ spark-arrow-datasource-parquet-spark${spark.version}
-
- ${project.basedir}/src/main/scala
- ${project.basedir}/src/test/scala
-
-
-
-
- com.intel.oap
- spark-arrow-datasource-standard
- ${project.version}
-
-
- com.intel.oap
- spark-sql-columnar-shims-common
- ${project.version}
- provided
-
-
+
+
+ com.intel.oap
+ spark-arrow-datasource-standard-spark${spark.version}
+ ${project.version}
+
+
+ com.intel.oap
+ spark-sql-columnar-shims-common-spark${spark.version}
+ ${project.version}
+ provided
+
+
+
+ ${project.basedir}/src/main/scala
+ ${project.basedir}/src/test/scala
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.0
+
+
+ process-sources
+
+ compile
+ testCompile
+
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+
+
+
+
+
+
+ org.scala-tools
+ maven-scala-plugin
+
+ ${scala.version}
+
+
+
+
diff --git a/arrow-data-source/standard/pom.xml b/arrow-data-source/standard/pom.xml
index 4b37a639f..57e851950 100644
--- a/arrow-data-source/standard/pom.xml
+++ b/arrow-data-source/standard/pom.xml
@@ -2,110 +2,135 @@
-
- spark-arrow-datasource
- com.intel.oap
- 1.4.0-SNAPSHOT
-
- 4.0.0
+
+ spark-arrow-datasource
+ com.intel.oap
+ 1.4.0-SNAPSHOT
+
+ 4.0.0
- spark-arrow-datasource-standard
+ spark-arrow-datasource-standard-spark${spark.version}
+
+
+ com.intel.oap
+ spark-arrow-datasource-common-spark${spark.version}
+ ${project.version}
+
+
+ com.intel.oap
+ spark-sql-columnar-shims-common-spark${spark.version}
+ ${project.version}
+ provided
+
+
-
-
- com.intel.oap
- spark-arrow-datasource-common
- ${project.version}
-
-
- com.intel.oap
- spark-sql-columnar-shims-common
- ${project.version}
- provided
-
-
-
-
- ${project.basedir}/src/main/scala
- ${project.basedir}/src/test/scala
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- 3.8.0
-
-
- 1.8
-
-
-
- compile
-
- compile
-
-
-
-
-
- org.codehaus.mojo
- build-helper-maven-plugin
-
-
- add-src-1
- generate-sources
-
- add-source
-
-
-
-
-
-
-
-
-
-
- maven-assembly-plugin
- 3.3.0
-
-
- jar-with-dependencies
-
-
-
-
- make-assembly
- package
-
- single
-
-
-
-
-
- org.apache.maven.plugins
- maven-source-plugin
-
-
- attach-sources
-
- jar
-
-
-
-
-
-
-
-
-
- org.scala-tools
- maven-scala-plugin
-
- ${scala.version}
-
-
-
-
+
+ ${project.basedir}/src/main/scala
+ ${project.basedir}/src/test/scala
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.8.0
+
+
+ 1.8
+
+
+
+ compile
+
+ compile
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-src-1
+ generate-sources
+
+ add-source
+
+
+
+
+
+
+
+
+
+
+ maven-assembly-plugin
+ 3.3.0
+
+
+ jar-with-dependencies
+
+
+
+
+ make-assembly
+ package
+
+ single
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+ attach-sources
+
+ jar
+
+
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.0
+
+
+ process-sources
+
+ compile
+ testCompile
+
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+
+
+
+
+
+
+ org.scala-tools
+ maven-scala-plugin
+
+ ${scala.version}
+
+
+
+
diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml
index 1cdd279de..63cab3c4d 100644
--- a/native-sql-engine/core/pom.xml
+++ b/native-sql-engine/core/pom.xml
@@ -22,7 +22,7 @@
com.intel.oap
- spark-columnar-core
+ spark-columnar-core-spark${spark.version}
jar
OAP Project Spark Columnar Plugin
@@ -54,7 +54,7 @@
com.intel.oap
- spark-sql-columnar-shims-spark311
+ spark-sql-columnar-shims-spark${spark.version}
${project.version}
@@ -64,7 +64,7 @@
com.intel.oap
- spark-sql-columnar-shims-spark321
+ spark-sql-columnar-shims-spark${spark.version}
${project.version}
@@ -157,7 +157,7 @@
- spark-arrow-datasource-common
+ spark-arrow-datasource-common-spark${spark.version}
com.intel.oap
${project.version}
provided
@@ -282,7 +282,7 @@
com.intel.oap
- spark-arrow-datasource-standard
+ spark-arrow-datasource-standard-spark${spark.version}
${project.version}
test
@@ -324,7 +324,7 @@
com.intel.oap
- spark-sql-columnar-shims-common
+ spark-sql-columnar-shims-common-spark${spark.version}
${project.version}
provided
@@ -360,36 +360,55 @@
target/scala-${scala.binary.version}/test-classes
- exec-maven-plugin
- org.codehaus.mojo
- 1.6.0
-
-
- Build cpp
- generate-resources
-
- exec
-
-
- bash
-
- ${cpp.dir}/compile.sh
- ${nativesql.cpp_tests}
- ${nativesql.build_arrow}
- ${nativesql.static_arrow}
- ${nativesql.build_protobuf}
- ${nativesql.arrow_root}
- ${nativesql.arrow.bfs.install.dir}
- ${nativesql.build_jemalloc}
-
-
-
-
+ exec-maven-plugin
+ org.codehaus.mojo
+ 1.6.0
+
+
+ Build cpp
+ generate-resources
+
+ exec
+
+
+ bash
+
+ ${cpp.dir}/compile.sh
+ ${nativesql.cpp_tests}
+ ${nativesql.build_arrow}
+ ${nativesql.static_arrow}
+ ${nativesql.build_protobuf}
+ ${nativesql.arrow_root}
+ ${nativesql.arrow.bfs.install.dir}
+ ${nativesql.build_jemalloc}
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-resources-plugin
+ 3.0.1
- org.apache.maven.plugins
- maven-resources-plugin
- 3.0.1
+ maven-javadoc-plugin
+ 3.2.0
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
+ UTF-8
+ UTF-8
+
+ -Xdoclint:none
+
+
net.alchim31.maven
@@ -407,6 +426,19 @@
compile
+
+ attach-javadocs
+ verify
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
scala-test-compile
process-test-resources
@@ -526,4 +558,15 @@
+
+
+
+ org.scala-tools
+ maven-scala-plugin
+
+ ${scala.version}
+
+
+
+
diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java
index 8d5baa49c..bf36772dc 100644
--- a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java
+++ b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java
@@ -42,7 +42,7 @@ public class JniUtils {
private static final String ARROW_LIBRARY_NAME = "libarrow.so.400.0.0";
private static final String ARROW_PARENT_LIBRARY_NAME = "libarrow.so.400";
private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400.0.0";
- private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400";
+ private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400";
private static boolean isLoaded = false;
private static boolean isCodegenDependencyLoaded = false;
private static List codegenJarsLoadedCache = new ArrayList<>();
diff --git a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc
index 2288865e8..591e6829e 100644
--- a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc
+++ b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc
@@ -1135,7 +1135,6 @@ class HashAggregateKernel::Impl {
if (post_process_projector_) {
RETURN_NOT_OK(post_process_projector_->Evaluate(&outputs));
}
-
*out = arrow::RecordBatch::Make(result_schema_, out_length, outputs);
return arrow::Status::OK();
}
diff --git a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc
index 2cbcd618a..924619c18 100644
--- a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc
+++ b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc
@@ -1006,4 +1006,4 @@ arrow::Status RowToColumnarConverter::Init(std::shared_ptr*
}
} // namespace rowtocolumnar
-} // namespace sparkcolumnarplugin
\ No newline at end of file
+} // namespace sparkcolumnarplugin
diff --git a/pom.xml b/pom.xml
index fe7e0a2f7..63cab36f3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -20,7 +20,8 @@
pom
Native SQL Engine Parent Pom
- https://github.com/oap-project/native-sql-engine.git
+ https://github.com/oap-project/gazelle_plugin.git
+ Native SQL Engine is an open source plugin for Spark Columnar based Processing Support
@@ -29,6 +30,21 @@
+
+
+ Weiting Chen
+ weiting.chen@intel.com
+ Intel Corporation
+ http://www.intel.com
+
+
+
+
+ scm:git:git://github.com/oap-project/gazelle_plugin.git
+ scm:git:ssh://github.com:oap-project/gazelle_plugin.git
+ http://github.com/oap-project/gazelle_plugin/tree/master
+
+
arrow-data-source
native-sql-engine/core
@@ -118,8 +134,45 @@
arrow-memory-netty
+
+ publish
+
+ false
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+ 1.5
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+
+
+
+
+
+
+
+ ossrh
+ https://oss.sonatype.org/content/repositories/snapshots
+
+
+ ossrh
+ https://oss.sonatype.org/service/local/staging/deploy/maven2/
+
+
+
3.1.1
@@ -321,6 +374,57 @@
+
+ org.sonatype.plugins
+ nexus-staging-maven-plugin
+ 1.6.7
+ true
+
+ ossrh
+ https://oss.sonatype.org/
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ 2.2.1
+
+
+ attach-sources
+
+ jar-no-fork
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.2.0
+
+
+ attach-javadocs
+
+ jar
+
+
+
+ aggregate
+ site
+
+ aggregate
+
+
+
+
+ UTF-8
+ UTF-8
+
+ -Xdoclint:none
+
+
+
org.apache.maven.plugins
maven-jar-plugin
@@ -349,4 +453,28 @@
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+ 3.2.0
+
+
+
+ javadoc
+ test-javadoc
+
+
+
+ aggregate
+ false
+
+ aggregate
+
+
+
+
+
+
diff --git a/shims/common/pom.xml b/shims/common/pom.xml
index 8e8a17831..4ed451ffd 100644
--- a/shims/common/pom.xml
+++ b/shims/common/pom.xml
@@ -15,7 +15,7 @@
- 4.0.0
+ 4.0.0
com.intel.oap
@@ -24,74 +24,86 @@
../pom.xml
- spark-sql-columnar-shims-common
+ spark-sql-columnar-shims-common-spark${spark.version}
${project.name.prefix} Shims Common
1.4.0-SNAPSHOT
jar
-
-
- org.scalastyle
- scalastyle-maven-plugin
-
-
- net.alchim31.maven
- scala-maven-plugin
- 3.2.2
-
-
- scala-compile-first
- process-resources
-
- compile
-
-
-
- scala-test-compile-first
- process-test-resources
-
- testCompile
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- 3.3
-
-
- ${java.version}
- UTF-8
- 1024m
- true
-
- -Xlint:all,-serial,-path
-
-
-
-
+
+
+ org.scalastyle
+ scalastyle-maven-plugin
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.2
+
+
+ scala-compile-first
+ process-resources
+
+ compile
+
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+ scala-test-compile-first
+ process-test-resources
+
+ testCompile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.3
+
+
+ ${java.version}
+ UTF-8
+ 1024m
+ true
+
+ -Xlint:all,-serial,-path
+
+
+
+
-
- org.apache.spark
- spark-sql_${scala.binary.version}
- ${spark311.version}
- provided
-
-
- com.intel.oap
- spark-arrow-datasource-common
- ${project.version}
- provided
-
-
- org.apache.hadoop
- hadoop-mapreduce-client-core
- ${hadoop.version}
- provided
-
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark311.version}
+ provided
+
+
+ com.intel.oap
+ spark-arrow-datasource-common-spark${spark.version}
+ ${project.version}
+ provided
+
+
+ org.apache.hadoop
+ hadoop-mapreduce-client-core
+ ${hadoop.version}
+ provided
+
diff --git a/shims/pom.xml b/shims/pom.xml
index 9e45e9dd6..bc1d9525b 100644
--- a/shims/pom.xml
+++ b/shims/pom.xml
@@ -35,52 +35,52 @@
-
- org.scalatest
- scalatest_${scala.binary.version}
- 3.0.5
- test
-
-
-
- org.slf4j
- slf4j-log4j12
- 1.7.30
- test
-
-
- log4j
- log4j
- 1.2.17
- test
-
-
- org.apache.hadoop
- hadoop-client
- ${hadoop.version}
- test
-
+
+ org.scalatest
+ scalatest_${scala.binary.version}
+ 3.0.5
+ test
+
+
+
+ org.slf4j
+ slf4j-log4j12
+ 1.7.30
+ test
+
+
+ log4j
+ log4j
+ 1.2.17
+ test
+
+
+ org.apache.hadoop
+ hadoop-client
+ ${hadoop.version}
+ test
+
- spark
-
- true
-
-
- common
- spark311
-
+ spark
+
+ true
+
+
+ common
+ spark311
+
- spark-3.1
-
-
-
- common
- spark311
-
+ spark-3.1
+
+
+
+ common
+ spark311
+
spark-3.2
@@ -95,23 +95,37 @@
-
- net.alchim31.maven
- scala-maven-plugin
- ${scala.plugin.version}
-
-
- org.scalatest
- scalatest-maven-plugin
-
-
- test
-
- test
-
-
-
-
+
+ net.alchim31.maven
+ scala-maven-plugin
+ ${scala.plugin.version}
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+
+
+ org.scalatest
+ scalatest-maven-plugin
+
+
+ test
+
+ test
+
+
+
+
diff --git a/shims/spark311/pom.xml b/shims/spark311/pom.xml
index ef9a8cf40..67f547798 100644
--- a/shims/spark311/pom.xml
+++ b/shims/spark311/pom.xml
@@ -15,7 +15,7 @@
- 4.0.0
+ 4.0.0
com.intel.oap
@@ -24,65 +24,14 @@
../pom.xml
- spark-sql-columnar-shims-spark311
+ spark-sql-columnar-shims-spark${spark.version}
${project.name.prefix} Shims for Spark 3.1.1
jar
-
-
-
- org.scalastyle
- scalastyle-maven-plugin
-
-
- net.alchim31.maven
- scala-maven-plugin
- 3.2.2
-
-
- scala-compile-first
- process-resources
-
- compile
-
-
-
- scala-test-compile-first
- process-test-resources
-
- testCompile
-
-
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
- 3.3
-
-
- ${java.version}
- UTF-8
- 1024m
- true
-
- -Xlint:all,-serial,-path
-
-
-
-
-
-
-
- src/main/resources
-
-
-
-
com.intel.oap
- ${project.prefix}-shims-common
+ ${project.prefix}-shims-common-spark${spark.version}
${project.version}
compile
@@ -94,7 +43,7 @@
com.intel.oap
- spark-arrow-datasource-common
+ spark-arrow-datasource-common-spark${spark.version}
${project.version}
provided
@@ -117,4 +66,66 @@
provided
+
+
+
+
+ org.scalastyle
+ scalastyle-maven-plugin
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+ 3.2.2
+
+
+ scala-compile-first
+ process-resources
+
+ compile
+
+
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
+
+ scala-test-compile-first
+ process-test-resources
+
+ testCompile
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+ 3.3
+
+
+ ${java.version}
+ UTF-8
+ 1024m
+ true
+
+ -Xlint:all,-serial,-path
+
+
+
+
+
+
+ src/main/resources
+
+
+
diff --git a/shims/spark321/pom.xml b/shims/spark321/pom.xml
index aab60e3d5..2f091eabc 100644
--- a/shims/spark321/pom.xml
+++ b/shims/spark321/pom.xml
@@ -24,10 +24,43 @@
../pom.xml
- spark-sql-columnar-shims-spark321
+ spark-sql-columnar-shims-spark${spark.version}
${project.name.prefix} Shims for Spark 3.2.1
jar
+
+
+ com.intel.oap
+ ${project.prefix}-shims-common-spark${spark.version}
+ ${project.version}
+ compile
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark321.version}
+ provided
+
+
+ com.intel.oap
+ spark-arrow-datasource-common-spark${spark.version}
+ ${project.version}
+ provided
+
+
+ com.google.guava
+ guava
+ 11.0.2
+ provided
+
+
+ org.apache.hadoop
+ hadoop-common
+ ${hadoop.version}
+ provided
+
+
+
@@ -46,6 +79,18 @@
compile
+
+ attach-javadocs
+
+ doc-jar
+
+
+
+ attach-sources
+
+ add-source
+
+
scala-test-compile-first
process-test-resources
@@ -71,44 +116,10 @@
-
src/main/resources
-
-
-
- com.intel.oap
- ${project.prefix}-shims-common
- ${project.version}
- compile
-
-
- org.apache.spark
- spark-sql_${scala.binary.version}
- ${spark321.version}
- provided
-
-
- com.intel.oap
- spark-arrow-datasource-common
- ${project.version}
- provided
-
-
- com.google.guava
- guava
- 11.0.2
- provided
-
-
- org.apache.hadoop
- hadoop-common
- ${hadoop.version}
- provided
-
-