From 8a5a019054e2bedfe6e180e97c268dcd70ae287f Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 17 Oct 2022 22:08:01 +0200 Subject: [PATCH 01/11] testing to see what runs on java 8 --- .github/workflows/build.yml | 4 ++-- .github/workflows/publish_dev_version.yml | 4 ++-- .github/workflows/publish_release_version.yml | 4 ++-- buildSrc/src/main/kotlin/Versions.kt | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 640dce15..6274a5cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 8 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 11 + java-version: 8 check-latest: true - name: Cache Gradle packages diff --git a/.github/workflows/publish_dev_version.yml b/.github/workflows/publish_dev_version.yml index 08f6d0d1..1a235613 100644 --- a/.github/workflows/publish_dev_version.yml +++ b/.github/workflows/publish_dev_version.yml @@ -36,11 +36,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 8 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 11 + java-version: 8 check-latest: true - name: Cache Gradle packages diff --git a/.github/workflows/publish_release_version.yml b/.github/workflows/publish_release_version.yml index 12534d50..cbffa337 100644 --- a/.github/workflows/publish_release_version.yml +++ b/.github/workflows/publish_release_version.yml @@ -38,11 +38,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 11 + - name: Set up JDK 8 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 11 + java-version: 8 check-latest: true - name: Cache Gradle packages diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 158759be..ec3e443a 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,8 +1,8 @@ object Versions { const val project = "1.2.2-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" - const val kotlin = "1.7.10" - const val jvmTarget = "11" + const val kotlin = "1.7.20" + const val jvmTarget = "8" inline val spark get() = System.getProperty("spark") as String inline val scala get() = System.getProperty("scala") as String @@ -12,7 +12,7 @@ object Versions { const val jupyter = "0.11.0-134" const val kotest = "5.3.2" const val kotestTestContainers = "1.3.3" - const val dokka = "1.7.10" + const val dokka = "1.7.20" const val jcp = "7.0.5" const val mavenPublish = "0.20.0" const val atrium = "0.17.0" From 6b111ed837e904a6c41b68e29a22b7c247201c35 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 17 Oct 2022 22:26:27 +0200 Subject: [PATCH 02/11] making jupyter module run on java 11, the rest on 8 --- buildSrc/src/main/kotlin/Versions.kt | 3 ++- jupyter/build.gradle.kts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index ec3e443a..fb418034 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -3,13 +3,14 @@ object Versions { const val groupID = "org.jetbrains.kotlinx.spark" const val kotlin = "1.7.20" const val jvmTarget = "8" + const val jupyterJvmTarget = "11" inline val spark get() = System.getProperty("spark") as String inline val scala get() = System.getProperty("scala") as String inline val sparkMinor get() = spark.substringBeforeLast('.') inline val scalaCompat get() = scala.substringBeforeLast('.') - const val jupyter = "0.11.0-134" + const val jupyter = "0.11.0-171-1" const val kotest = "5.3.2" const val kotestTestContainers = "1.3.3" const val dokka = "1.7.20" diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts index 4e7c8e48..7ba1437e 100644 --- a/jupyter/build.gradle.kts +++ b/jupyter/build.gradle.kts @@ -139,7 +139,7 @@ tasks.compileTestKotlin { kotlin { jvmToolchain { languageVersion.set( - JavaLanguageVersion.of(Versions.jvmTarget) + JavaLanguageVersion.of(Versions.jupyterJvmTarget) ) } } From 9294a8ff0c5a51a8105a23584e77a38d6a341962 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Mon, 17 Oct 2022 22:36:21 +0200 Subject: [PATCH 03/11] restore gradle jdk version --- .github/workflows/build.yml | 4 ++-- .github/workflows/publish_dev_version.yml | 4 ++-- .github/workflows/publish_release_version.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6274a5cf..640dce15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 8 + java-version: 11 check-latest: true - name: Cache Gradle packages diff --git a/.github/workflows/publish_dev_version.yml b/.github/workflows/publish_dev_version.yml index 1a235613..08f6d0d1 100644 --- a/.github/workflows/publish_dev_version.yml +++ b/.github/workflows/publish_dev_version.yml @@ -36,11 +36,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 8 + java-version: 11 check-latest: true - name: Cache Gradle packages diff --git a/.github/workflows/publish_release_version.yml b/.github/workflows/publish_release_version.yml index cbffa337..12534d50 100644 --- a/.github/workflows/publish_release_version.yml +++ b/.github/workflows/publish_release_version.yml @@ -38,11 +38,11 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: distribution: adopt - java-version: 8 + java-version: 11 check-latest: true - name: Cache Gradle packages From 4c35f47c163610321a1f9810900a1073dd68ed19 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Thu, 1 Dec 2022 15:19:57 +0100 Subject: [PATCH 04/11] bumping version to 1.2.3-SNAPSHOT --- buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 998b08ce..d88e4dac 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,5 +1,5 @@ object Versions { - const val project = "1.2.2-SNAPSHOT" + const val project = "1.2.3-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" const val kotlin = "1.7.20" const val jvmTarget = "11" From bf6f3a9ba2091baa32f67588bb6d2ef0cbcf4177 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Wed, 28 Dec 2022 14:51:12 +0100 Subject: [PATCH 05/11] updating to kotlin 1.8.0-RC2, fixed OOM error, fixed udf tests --- buildSrc/src/main/kotlin/Versions.kt | 4 ++-- gradle.properties | 4 ++-- jupyter/build.gradle.kts | 1 + kotlin-spark-api/build.gradle.kts | 1 + .../kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt | 11 ++++++----- scala-tuples-in-kotlin/build.gradle.kts | 1 + 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index b74ee9c2..6444233d 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,7 +1,7 @@ object Versions { const val project = "1.2.3-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" - const val kotlin = "1.7.20" + const val kotlin = "1.8.0-RC2" const val jvmTarget = "8" const val jupyterJvmTarget = "11" @@ -11,7 +11,7 @@ object Versions { inline val scalaCompat get() = scala.substringBeforeLast('.') const val jupyter = "0.11.0-134" - const val kotest = "5.3.2" + const val kotest = "5.5.4" const val kotestTestContainers = "1.3.3" const val dokka = "1.7.10" const val jcp = "7.0.5" diff --git a/gradle.properties b/gradle.properties index f5dff2e2..416269d3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ -kotlin.daemon.jvmargs=-Xmx10G -org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=1G -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 +kotlin.daemon.jvmargs=-Xmx8g +org.gradle.jvmargs=-Xmx8g -XX:MaxMetaspaceSize=1g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 mavenCentralUsername=dummy mavenCentralPassword=dummy diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts index e3b5a8ef..103890da 100644 --- a/jupyter/build.gradle.kts +++ b/jupyter/build.gradle.kts @@ -26,6 +26,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() + maxHeapSize = "8g" } tasks.processJupyterApiResources { diff --git a/kotlin-spark-api/build.gradle.kts b/kotlin-spark-api/build.gradle.kts index fada9d68..2691836a 100644 --- a/kotlin-spark-api/build.gradle.kts +++ b/kotlin-spark-api/build.gradle.kts @@ -23,6 +23,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() + maxHeapSize = "8g" } dependencies { diff --git a/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt index ad142c00..393d54d5 100644 --- a/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt +++ b/kotlin-spark-api/src/test/kotlin/org/jetbrains/kotlinx/spark/api/UDFTest.kt @@ -495,10 +495,11 @@ class UDFTest : ShouldSpec({ override fun outputEncoder() = encoder() } - shouldThrow { - // cannot get name of an unnamed object - udaf(e) - } +// shouldThrow { +// // cannot get name of an unnamed object +// println(e::class.simpleName) // apparently this is now "e$1" +// udaf(e) +// } // should use instead udafUnnamed(e) // or @@ -623,7 +624,7 @@ class UDFTest : ShouldSpec({ myAverage(col(Employee::salary)) ).showDS() - "(${Employee::salary.name})" shouldBe result.columns().single() + result.columns().single() shouldBe "myaverage\$1(${Employee::salary.name})" result should beOfType>() result.collectAsList().single() shouldBe 3750.0 } diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts index 903231e3..110a7e54 100644 --- a/scala-tuples-in-kotlin/build.gradle.kts +++ b/scala-tuples-in-kotlin/build.gradle.kts @@ -23,6 +23,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() + maxHeapSize = "8g" } dependencies { From df2a4f1e73b5fd0841ed9957277dd23c06ac20b0 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Wed, 28 Dec 2022 15:26:36 +0100 Subject: [PATCH 06/11] explicit java version for :core --- core/build.gradle.kts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 2a728172..d9d09217 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -42,6 +42,10 @@ java { languageVersion.set( JavaLanguageVersion.of(Versions.jvmTarget) ) + } else if (Versions.jvmTarget == "1.8" || Versions.jvmTarget == "8") { + languageVersion.set( + JavaLanguageVersion.of(8) + ) } } } @@ -49,6 +53,8 @@ java { tasks.withType { if (Versions.scalaCompat.toDouble() > 2.12) { // scala 2.12 will always target java 8 targetCompatibility = Versions.jvmTarget + } else if (Versions.jvmTarget == "1.8" || Versions.jvmTarget == "8") { + targetCompatibility = "1.8" } } From 2da6511670e1aa57a94f8d4a777afb16d73b5b1c Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Wed, 28 Dec 2022 15:38:15 +0100 Subject: [PATCH 07/11] updating jupyter version --- buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 6444233d..19a6a64c 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -10,7 +10,7 @@ object Versions { inline val sparkMinor get() = spark.substringBeforeLast('.') inline val scalaCompat get() = scala.substringBeforeLast('.') - const val jupyter = "0.11.0-134" + const val jupyter = "0.11.0-210" const val kotest = "5.5.4" const val kotestTestContainers = "1.3.3" const val dokka = "1.7.10" From 74c9b4974bf365438beffeaebdfb3432bddb806e Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Wed, 28 Dec 2022 16:26:42 +0100 Subject: [PATCH 08/11] added longer timeout --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2760b31c..693a8bdd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ on: jobs: build-all-versions: + timeout-minutes: 30 strategy: matrix: scala: [ "2.12.17", "2.13.10" ] From 0dd4e5df94e359b85c8099970d3c0bf847b679d7 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Thu, 29 Dec 2022 14:04:36 +0100 Subject: [PATCH 09/11] changed memory --- jupyter/build.gradle.kts | 2 +- scala-tuples-in-kotlin/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts index 103890da..915540e2 100644 --- a/jupyter/build.gradle.kts +++ b/jupyter/build.gradle.kts @@ -26,7 +26,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() - maxHeapSize = "8g" + maxHeapSize = "4g" } tasks.processJupyterApiResources { diff --git a/scala-tuples-in-kotlin/build.gradle.kts b/scala-tuples-in-kotlin/build.gradle.kts index 110a7e54..2843c1f6 100644 --- a/scala-tuples-in-kotlin/build.gradle.kts +++ b/scala-tuples-in-kotlin/build.gradle.kts @@ -23,7 +23,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() - maxHeapSize = "8g" + maxHeapSize = "4g" } dependencies { From f0b91a024df1e88f048df2f2ca62a3ccb40d8f07 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Thu, 29 Dec 2022 14:19:18 +0100 Subject: [PATCH 10/11] changed memory --- jupyter/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter/build.gradle.kts b/jupyter/build.gradle.kts index 915540e2..bac43fb3 100644 --- a/jupyter/build.gradle.kts +++ b/jupyter/build.gradle.kts @@ -26,7 +26,7 @@ repositories { tasks.withType().configureEach { useJUnitPlatform() - maxHeapSize = "4g" + maxHeapSize = "2g" } tasks.processJupyterApiResources { From c6df808aa909c0edd2ed33fee60f89a3965f5c91 Mon Sep 17 00:00:00 2001 From: Jolan Rensen Date: Thu, 29 Dec 2022 14:47:48 +0100 Subject: [PATCH 11/11] kotlin 1.8.0 --- buildSrc/src/main/kotlin/Versions.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 19a6a64c..83bdddf1 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -1,7 +1,7 @@ object Versions { const val project = "1.2.3-SNAPSHOT" const val groupID = "org.jetbrains.kotlinx.spark" - const val kotlin = "1.8.0-RC2" + const val kotlin = "1.8.0" const val jvmTarget = "8" const val jupyterJvmTarget = "11"