diff --git a/arrow-data-source/common/pom.xml b/arrow-data-source/common/pom.xml index d52187c40..03e44c46c 100644 --- a/arrow-data-source/common/pom.xml +++ b/arrow-data-source/common/pom.xml @@ -2,15 +2,15 @@ - - spark-arrow-datasource - com.intel.oap - 1.4.0-SNAPSHOT - ../pom.xml - + + spark-arrow-datasource + com.intel.oap + 1.4.0-SNAPSHOT + ../pom.xml + 4.0.0 - spark-arrow-datasource-common + spark-arrow-datasource-common-spark${spark.version} @@ -23,6 +23,7 @@ org.apache.arrow arrow-dataset ${arrow.version} + compile io.netty @@ -41,13 +42,39 @@ jackson-annotations - compile + ${project.basedir}/src/main/scala ${project.basedir}/src/test/scala + + net.alchim31.maven + scala-maven-plugin + 3.2.0 + + + process-sources + + compile + testCompile + + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + org.apache.maven.plugins maven-source-plugin @@ -95,6 +122,27 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + attach-javadocs + deploy + + jar + + + + + UTF-8 + UTF-8 + + -Xdoclint:none + + + diff --git a/arrow-data-source/parquet/pom.xml b/arrow-data-source/parquet/pom.xml index 28c38e76a..ec60b5e10 100644 --- a/arrow-data-source/parquet/pom.xml +++ b/arrow-data-source/parquet/pom.xml @@ -2,32 +2,70 @@ - - spark-arrow-datasource - com.intel.oap - 1.4.0-SNAPSHOT - - 4.0.0 + + spark-arrow-datasource + com.intel.oap + 1.4.0-SNAPSHOT + + 4.0.0 - spark-arrow-datasource-parquet + spark-arrow-datasource-parquet-spark${spark.version} - - ${project.basedir}/src/main/scala - ${project.basedir}/src/test/scala - - - - - com.intel.oap - spark-arrow-datasource-standard - ${project.version} - - - com.intel.oap - spark-sql-columnar-shims-common - ${project.version} - provided - - + + + com.intel.oap + spark-arrow-datasource-standard-spark${spark.version} + ${project.version} + + + com.intel.oap + spark-sql-columnar-shims-common-spark${spark.version} + ${project.version} + provided + + + + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + + + net.alchim31.maven + scala-maven-plugin + 3.2.0 + + + process-sources + + compile + testCompile + + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + + + + + + + org.scala-tools + maven-scala-plugin + + ${scala.version} + + + + diff --git a/arrow-data-source/standard/pom.xml b/arrow-data-source/standard/pom.xml index 4b37a639f..57e851950 100644 --- a/arrow-data-source/standard/pom.xml +++ b/arrow-data-source/standard/pom.xml @@ -2,110 +2,135 @@ - - spark-arrow-datasource - com.intel.oap - 1.4.0-SNAPSHOT - - 4.0.0 + + spark-arrow-datasource + com.intel.oap + 1.4.0-SNAPSHOT + + 4.0.0 - spark-arrow-datasource-standard + spark-arrow-datasource-standard-spark${spark.version} + + + com.intel.oap + spark-arrow-datasource-common-spark${spark.version} + ${project.version} + + + com.intel.oap + spark-sql-columnar-shims-common-spark${spark.version} + ${project.version} + provided + + - - - com.intel.oap - spark-arrow-datasource-common - ${project.version} - - - com.intel.oap - spark-sql-columnar-shims-common - ${project.version} - provided - - - - - ${project.basedir}/src/main/scala - ${project.basedir}/src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.0 - - 1.8 - 1.8 - - - - compile - - compile - - - - - - org.codehaus.mojo - build-helper-maven-plugin - - - add-src-1 - generate-sources - - add-source - - - - ${project.basedir}/src/main/java - - - - - - - maven-assembly-plugin - 3.3.0 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - - - - org.scala-tools - maven-scala-plugin - - ${scala.version} - - - - + + ${project.basedir}/src/main/scala + ${project.basedir}/src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + + + + compile + + compile + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-src-1 + generate-sources + + add-source + + + + ${project.basedir}/src/main/java + + + + + + + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.0 + + + process-sources + + compile + testCompile + + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + + + + + + + org.scala-tools + maven-scala-plugin + + ${scala.version} + + + + diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml index 1cdd279de..63cab3c4d 100644 --- a/native-sql-engine/core/pom.xml +++ b/native-sql-engine/core/pom.xml @@ -22,7 +22,7 @@ com.intel.oap - spark-columnar-core + spark-columnar-core-spark${spark.version} jar OAP Project Spark Columnar Plugin @@ -54,7 +54,7 @@ com.intel.oap - spark-sql-columnar-shims-spark311 + spark-sql-columnar-shims-spark${spark.version} ${project.version} @@ -64,7 +64,7 @@ com.intel.oap - spark-sql-columnar-shims-spark321 + spark-sql-columnar-shims-spark${spark.version} ${project.version} @@ -157,7 +157,7 @@ - spark-arrow-datasource-common + spark-arrow-datasource-common-spark${spark.version} com.intel.oap ${project.version} provided @@ -282,7 +282,7 @@ com.intel.oap - spark-arrow-datasource-standard + spark-arrow-datasource-standard-spark${spark.version} ${project.version} test @@ -324,7 +324,7 @@ com.intel.oap - spark-sql-columnar-shims-common + spark-sql-columnar-shims-common-spark${spark.version} ${project.version} provided @@ -360,36 +360,55 @@ target/scala-${scala.binary.version}/test-classes - exec-maven-plugin - org.codehaus.mojo - 1.6.0 - - - Build cpp - generate-resources - - exec - - - bash - - ${cpp.dir}/compile.sh - ${nativesql.cpp_tests} - ${nativesql.build_arrow} - ${nativesql.static_arrow} - ${nativesql.build_protobuf} - ${nativesql.arrow_root} - ${nativesql.arrow.bfs.install.dir} - ${nativesql.build_jemalloc} - - - - + exec-maven-plugin + org.codehaus.mojo + 1.6.0 + + + Build cpp + generate-resources + + exec + + + bash + + ${cpp.dir}/compile.sh + ${nativesql.cpp_tests} + ${nativesql.build_arrow} + ${nativesql.static_arrow} + ${nativesql.build_protobuf} + ${nativesql.arrow_root} + ${nativesql.arrow.bfs.install.dir} + ${nativesql.build_jemalloc} + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.0.1 - org.apache.maven.plugins - maven-resources-plugin - 3.0.1 + maven-javadoc-plugin + 3.2.0 + + + attach-javadocs + + jar + + + + + UTF-8 + UTF-8 + + -Xdoclint:none + + net.alchim31.maven @@ -407,6 +426,19 @@ compile + + attach-javadocs + verify + + doc-jar + + + + attach-sources + + add-source + + scala-test-compile process-test-resources @@ -526,4 +558,15 @@ + + + + org.scala-tools + maven-scala-plugin + + ${scala.version} + + + + diff --git a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java index 8d5baa49c..bf36772dc 100644 --- a/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java +++ b/native-sql-engine/core/src/main/java/com/intel/oap/vectorized/JniUtils.java @@ -42,7 +42,7 @@ public class JniUtils { private static final String ARROW_LIBRARY_NAME = "libarrow.so.400.0.0"; private static final String ARROW_PARENT_LIBRARY_NAME = "libarrow.so.400"; private static final String GANDIVA_LIBRARY_NAME = "libgandiva.so.400.0.0"; - private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400"; + private static final String GANDIVA_PARENT_LIBRARY_NAME = "libgandiva.so.400"; private static boolean isLoaded = false; private static boolean isCodegenDependencyLoaded = false; private static List codegenJarsLoadedCache = new ArrayList<>(); diff --git a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc index 2288865e8..591e6829e 100644 --- a/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc +++ b/native-sql-engine/cpp/src/codegen/arrow_compute/ext/hash_aggregate_kernel.cc @@ -1135,7 +1135,6 @@ class HashAggregateKernel::Impl { if (post_process_projector_) { RETURN_NOT_OK(post_process_projector_->Evaluate(&outputs)); } - *out = arrow::RecordBatch::Make(result_schema_, out_length, outputs); return arrow::Status::OK(); } diff --git a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc index 2cbcd618a..924619c18 100644 --- a/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc +++ b/native-sql-engine/cpp/src/operators/row_to_columnar_converter.cc @@ -1006,4 +1006,4 @@ arrow::Status RowToColumnarConverter::Init(std::shared_ptr* } } // namespace rowtocolumnar -} // namespace sparkcolumnarplugin \ No newline at end of file +} // namespace sparkcolumnarplugin diff --git a/pom.xml b/pom.xml index fe7e0a2f7..63cab36f3 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,8 @@ pom Native SQL Engine Parent Pom - https://github.com/oap-project/native-sql-engine.git + https://github.com/oap-project/gazelle_plugin.git + Native SQL Engine is an open source plugin for Spark Columnar based Processing Support @@ -29,6 +30,21 @@ + + + Weiting Chen + weiting.chen@intel.com + Intel Corporation + http://www.intel.com + + + + + scm:git:git://github.com/oap-project/gazelle_plugin.git + scm:git:ssh://github.com:oap-project/gazelle_plugin.git + http://github.com/oap-project/gazelle_plugin/tree/master + + arrow-data-source native-sql-engine/core @@ -118,8 +134,45 @@ arrow-memory-netty + + publish + + false + + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + verify + + sign + + + + + + + + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + 3.1.1 @@ -321,6 +374,57 @@ + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.7 + true + + ossrh + https://oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + attach-javadocs + + jar + + + + aggregate + site + + aggregate + + + + + UTF-8 + UTF-8 + + -Xdoclint:none + + + org.apache.maven.plugins maven-jar-plugin @@ -349,4 +453,28 @@ + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + + javadoc + test-javadoc + + + + aggregate + false + + aggregate + + + + + + diff --git a/shims/common/pom.xml b/shims/common/pom.xml index 8e8a17831..4ed451ffd 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -15,7 +15,7 @@ - 4.0.0 + 4.0.0 com.intel.oap @@ -24,74 +24,86 @@ ../pom.xml - spark-sql-columnar-shims-common + spark-sql-columnar-shims-common-spark${spark.version} ${project.name.prefix} Shims Common 1.4.0-SNAPSHOT jar - - - org.scalastyle - scalastyle-maven-plugin - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - - scala-compile-first - process-resources - - compile - - - - scala-test-compile-first - process-test-resources - - testCompile - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - ${java.version} - ${java.version} - UTF-8 - 1024m - true - - -Xlint:all,-serial,-path - - - - + + + org.scalastyle + scalastyle-maven-plugin + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + scala-compile-first + process-resources + + compile + + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + scala-test-compile-first + process-test-resources + + testCompile + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + ${java.version} + ${java.version} + UTF-8 + 1024m + true + + -Xlint:all,-serial,-path + + + + - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark311.version} - provided - - - com.intel.oap - spark-arrow-datasource-common - ${project.version} - provided - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - provided - + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark311.version} + provided + + + com.intel.oap + spark-arrow-datasource-common-spark${spark.version} + ${project.version} + provided + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + diff --git a/shims/pom.xml b/shims/pom.xml index 9e45e9dd6..bc1d9525b 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -35,52 +35,52 @@ - - org.scalatest - scalatest_${scala.binary.version} - 3.0.5 - test - - - - org.slf4j - slf4j-log4j12 - 1.7.30 - test - - - log4j - log4j - 1.2.17 - test - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - test - + + org.scalatest + scalatest_${scala.binary.version} + 3.0.5 + test + + + + org.slf4j + slf4j-log4j12 + 1.7.30 + test + + + log4j + log4j + 1.2.17 + test + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + test + - spark - - true - - - common - spark311 - + spark + + true + + + common + spark311 + - spark-3.1 - - - - common - spark311 - + spark-3.1 + + + + common + spark311 + spark-3.2 @@ -95,23 +95,37 @@ - - net.alchim31.maven - scala-maven-plugin - ${scala.plugin.version} - - - org.scalatest - scalatest-maven-plugin - - - test - - test - - - - + + net.alchim31.maven + scala-maven-plugin + ${scala.plugin.version} + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + + + org.scalatest + scalatest-maven-plugin + + + test + + test + + + + diff --git a/shims/spark311/pom.xml b/shims/spark311/pom.xml index ef9a8cf40..67f547798 100644 --- a/shims/spark311/pom.xml +++ b/shims/spark311/pom.xml @@ -15,7 +15,7 @@ - 4.0.0 + 4.0.0 com.intel.oap @@ -24,65 +24,14 @@ ../pom.xml - spark-sql-columnar-shims-spark311 + spark-sql-columnar-shims-spark${spark.version} ${project.name.prefix} Shims for Spark 3.1.1 jar - - - - org.scalastyle - scalastyle-maven-plugin - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - - scala-compile-first - process-resources - - compile - - - - scala-test-compile-first - process-test-resources - - testCompile - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.3 - - ${java.version} - ${java.version} - UTF-8 - 1024m - true - - -Xlint:all,-serial,-path - - - - - - - - src/main/resources - - - - com.intel.oap - ${project.prefix}-shims-common + ${project.prefix}-shims-common-spark${spark.version} ${project.version} compile @@ -94,7 +43,7 @@ com.intel.oap - spark-arrow-datasource-common + spark-arrow-datasource-common-spark${spark.version} ${project.version} provided @@ -117,4 +66,66 @@ provided + + + + + org.scalastyle + scalastyle-maven-plugin + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + scala-compile-first + process-resources + + compile + + + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + + + scala-test-compile-first + process-test-resources + + testCompile + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.3 + + ${java.version} + ${java.version} + UTF-8 + 1024m + true + + -Xlint:all,-serial,-path + + + + + + + src/main/resources + + + diff --git a/shims/spark321/pom.xml b/shims/spark321/pom.xml index aab60e3d5..2f091eabc 100644 --- a/shims/spark321/pom.xml +++ b/shims/spark321/pom.xml @@ -24,10 +24,43 @@ ../pom.xml - spark-sql-columnar-shims-spark321 + spark-sql-columnar-shims-spark${spark.version} ${project.name.prefix} Shims for Spark 3.2.1 jar + + + com.intel.oap + ${project.prefix}-shims-common-spark${spark.version} + ${project.version} + compile + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark321.version} + provided + + + com.intel.oap + spark-arrow-datasource-common-spark${spark.version} + ${project.version} + provided + + + com.google.guava + guava + 11.0.2 + provided + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + @@ -46,6 +79,18 @@ compile + + attach-javadocs + + doc-jar + + + + attach-sources + + add-source + + scala-test-compile-first process-test-resources @@ -71,44 +116,10 @@ - src/main/resources - - - - com.intel.oap - ${project.prefix}-shims-common - ${project.version} - compile - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321.version} - provided - - - com.intel.oap - spark-arrow-datasource-common - ${project.version} - provided - - - com.google.guava - guava - 11.0.2 - provided - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - provided - -