Fix mac upload, missed clang-format, add mac CI/CD upload

flipperdevices · Dec 19, 2023 · c075537 · c075537
1 parent a15f8da
commit c075537
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 31 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -83,3 +83,41 @@ jobs:
               -e "INDEXER_TOKEN=${{ secrets.INDEXER_TOKEN }}" \
               flipperzero-toolchain-windows:${IMAGE_TAG} \
               /usr/bin/bundle_upload.sh
+
+  build-mac:
+    runs-on: [self-hosted, macOS, ARM64]
+    steps:
+      - name: 'Wipe workspace'
+        run: find ./ -mount -maxdepth 1 -exec rm -rf {} \;
+
+      - name: 'Checkout code'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: 'Set image tag and name'
+        id: tag
+        run: |
+          IMAGE_TAG="0"
+          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            REF=${{ github.ref }};
+            TAG_FULL=${REF#refs/*/};
+            IMAGE_TAG=${TAG_FULL//\//_};
+          fi
+          echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_ENV
+          echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT
+
+      #- name: 'Build mac toolchain'
+      # run: |
+      #    sudo chmocker build -t flipperzero-toolchain-mac:${IMAGE_TAG} darwin
+
+      - name: 'Bundle and upload mac toolchain'
+        #if: ${{ steps.tag.outputs.image_tag != '0' }}
+        run: |
+          sudo chmocker run --rm \
+              -e "TOOLCHAIN_VERSION=$IMAGE_TAG" \
+              -e "INDEXER_URL=${{ secrets.INDEXER_URL }}" \
+              -e "INDEXER_TOKEN=${{ secrets.INDEXER_TOKEN }}" \
+              flipperzero-toolchain-windows:${IMAGE_TAG} \
+              /usr/bin/bundle_upload.sh
diff --git a/darwin/Dockerfile b/darwin/Dockerfile
@@ -52,9 +52,22 @@
 #RUN bash /toolchain/src/move_newlib_to_nano.sh
 
 #FROM flipperzero-toolchain-binutils AS flipperzero-toolchain-tools
+#RUN brew install cmake automake autoconf
 #ADD https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protobuf-cpp-3.21.7.tar.gz /toolchain/src/src/archives/
 #RUN tar -xvf /toolchain/src/src/archives/protobuf-cpp-3.21.7.tar.gz -C /toolchain/src/src/
 #RUN mv /toolchain/src/src/protobuf-3.21.7 /toolchain/src/src/protobuf
+#ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/llvm-17.0.6.src.tar.xz /toolchain/src/src/archives/
+#RUN mkdir -p /toolchain/src/src/clang-format
+#RUN tar -xvf /toolchain/src/src/archives/llvm-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format
+#ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang-17.0.6.src.tar.xz /toolchain/src/src/archives/
+#RUN tar -xvf /toolchain/src/src/archives/clang-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format/llvm-17.0.6.src/tools/
+#RUN mv /toolchain/src/src/clang-format/llvm-17.0.6.src/tools/clang-17.0.6.src /toolchain/src/src/clang-format/llvm-17.0.6.src/tools/clang
+#ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/cmake-17.0.6.src.tar.xz /toolchain/src/src/archives/
+#RUN tar -xvf /toolchain/src/src/archives/cmake-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format/
+#RUN mv /toolchain/src/src/clang-format/cmake-17.0.6.src /toolchain/src/src/clang-format/cmake
+#ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/third-party-17.0.6.src.tar.xz /toolchain/src/src/archives/
+#RUN tar -xvf /toolchain/src/src/archives/third-party-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format/
+#RUN mv /toolchain/src/src/clang-format/third-party-17.0.6.src /toolchain/src/src/clang-format/third-party
 #ADD scripts/build-mac-flipper-tools.sh /toolchain/src/
 #RUN bash /toolchain/src/build-mac-flipper-tools.sh
 #ADD https://github.com/xpack-dev-tools/openocd-xpack/releases/download/v0.12.0-2/xpack-openocd-0.12.0-2-darwin-arm64.tar.gz /toolchain/src/src/archives/
@@ -74,22 +87,22 @@
 #RUN relink.sh /toolchain/mac-arm64-output-root
 
 FROM MacOSVenturaWithBrew AS flipperzero-toolchain-mac
-#RUN brew install coreutils bison
-#RUN brew link bison --force
-#COPY --from=flipperzero-toolchain-gdb /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
-#COPY --from=flipperzero-toolchain-gdb /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
-#COPY --from=flipperzero-toolchain-gcc /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
-#COPY --from=flipperzero-toolchain-gcc /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
-#COPY --from=flipperzero-toolchain-tools /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
-#COPY --from=flipperzero-toolchain-tools /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
-#ADD requirements.txt /toolchain/
-#RUN /toolchain/mac-x86_64-output-root/bin/python3 -m pip install --upgrade pip
-#RUN /toolchain/mac-x86_64-output-root/bin/python3 -m pip install -r /toolchain/requirements.txt
-#RUN /toolchain/mac-arm64-output-root/bin/python3 -m pip install --upgrade pip
-#RUN /toolchain/mac-arm64-output-root/bin/python3 -m pip install -r /toolchain/requirements.txt
-#ADD scripts/strip.sh /usr/bin/
-#ADD scripts/relink.sh /usr/bin/
-#RUN relink.sh /toolchain/mac-x86_64-output-root
-#RUN relink.sh /toolchain/mac-arm64-output-root
-#RUN strip.sh
+RUN brew install coreutils bison
+RUN brew link bison --force
+COPY --from=flipperzero-toolchain-gdb /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
+COPY --from=flipperzero-toolchain-gdb /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
+COPY --from=flipperzero-toolchain-gcc /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
+COPY --from=flipperzero-toolchain-gcc /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
+COPY --from=flipperzero-toolchain-tools /toolchain/mac-x86_64-output-root /toolchain/mac-x86_64-output-root
+COPY --from=flipperzero-toolchain-tools /toolchain/mac-arm64-output-root /toolchain/mac-arm64-output-root
+ADD requirements.txt /toolchain/
+RUN /toolchain/mac-x86_64-output-root/bin/python3 -m pip install --upgrade pip
+RUN /toolchain/mac-x86_64-output-root/bin/python3 -m pip install -r /toolchain/requirements.txt
+RUN /toolchain/mac-arm64-output-root/bin/python3 -m pip install --upgrade pip
+RUN /toolchain/mac-arm64-output-root/bin/python3 -m pip install -r /toolchain/requirements.txt
+ADD scripts/strip.sh /usr/bin/
+ADD scripts/relink.sh /usr/bin/
+RUN relink.sh /toolchain/mac-x86_64-output-root
+RUN relink.sh /toolchain/mac-arm64-output-root
+RUN strip.sh
 ADD scripts/bundle_upload.sh /usr/bin/
diff --git a/darwin/scripts/build-mac-flipper-tools.sh b/darwin/scripts/build-mac-flipper-tools.sh
@@ -41,7 +41,7 @@ function build_protobuf_x86_64() {
         PATH="$MAC_X86_64_OUTPUT_ROOT/bin:$PATH" \
         make "-j$CPUS";
     PATH="$MAC_X86_64_OUTPUT_ROOT/bin:$PATH" \
-    	make install;
+        make install;
     popd;
 }
 
@@ -72,7 +72,7 @@ function build_protobuf_arm64() {
         PATH="$MAC_ARM64_OUTPUT_ROOT/bin:$PATH" \
         make "-j$CPUS";
     PATH="$MAC_ARM64_OUTPUT_ROOT/bin:$PATH" \
-    	make install;
+        make install;
     popd;
 }
 
@@ -84,11 +84,12 @@ function build_clang_format_x86_64() {
         CXXFLAGS="$MAC_X86_64_FLAGS" \
         CFLAGS="$MAC_X86_64_FLAGS" \
         LDFLAGS="$MAC_X86_64_FLAGS" \
-        cmake -S /tmp/test/src/llvm-17.0.6.src \
+        cmake \
+            -S /toolchain/src/src/clang-format/llvm-17.0.6.src \
             -B build \
             -DLLVM_INCLUDE_BENCHMARKS=OFF \
             -DCMAKE_BUILD_TYPE=Release \
-            -DCMAKE_INSTALL_PREFIX=/tmp/test/out \
+            "-DCMAKE_INSTALL_PREFIX=$MAC_X86_64_OUTPUT_ROOT" \
             -DLLVM_EXTERNAL_PROJECTS=clang \
             -DCMAKE_OSX_ARCHITECTURES=x86_64;
     CPPFLAGS="$MAC_X86_64_FLAGS" \
@@ -100,8 +101,47 @@ function build_clang_format_x86_64() {
             "build" \
             "--target" \
             "clang-format" \
-            -j10;
+            "-j$CPUS";
+    cmake \
+        --install build \
+        --strip \
+        --component clang-format;
+    popd;
 }
 
+function build_clang_format_arm64() {
+    rm -rf "$MAC_ARM64_CONFIGURE_ROOT/clang-format";
+    mkdir -p "$MAC_ARM64_CONFIGURE_ROOT/clang-format";
+    pushd "$MAC_ARM64_CONFIGURE_ROOT/clang-format";
+    CPPFLAGS="$MAC_ARM64_FLAGS" \
+        CXXFLAGS="$MAC_ARM64_FLAGS" \
+        CFLAGS="$MAC_ARM64_FLAGS" \
+        LDFLAGS="$MAC_ARM64_FLAGS" \
+        cmake \
+            -S /toolchain/src/src/clang-format/llvm-17.0.6.src \
+            -B build \
+            -DLLVM_INCLUDE_BENCHMARKS=OFF \
+            -DCMAKE_BUILD_TYPE=Release \
+            "-DCMAKE_INSTALL_PREFIX=$MAC_ARM64_OUTPUT_ROOT" \
+            -DLLVM_EXTERNAL_PROJECTS=clang \
+            -DCMAKE_OSX_ARCHITECTURES=arm64;
+    CPPFLAGS="$MAC_ARM64_FLAGS" \
+        CXXFLAGS="$MAC_ARM64_FLAGS" \
+        CFLAGS="$MAC_ARM64_FLAGS" \
+        LDFLAGS="$MAC_ARM64_FLAGS"\
+        cmake \
+            "--build" \
+            "build" \
+            "--target" \
+            "clang-format" \
+            "-j$CPUS";
+    cmake \
+        --install build \
+        --strip \
+        --component clang-format;
+    popd;
+}
 build_protobuf_x86_64;
 build_protobuf_arm64;
+build_clang_format_x86_64;
+build_clang_format_arm64;
diff --git a/darwin/scripts/bundle_upload.sh b/darwin/scripts/bundle_upload.sh
@@ -47,12 +47,12 @@ function make_bundle_arm64() {
 }
 
 function upload_bundle_x86_64() {
-    curl --fail -L -H "Token: $INDEXER_TOKEN" \
+    curl --fail --http1.1 -L -H "Token: $INDEXER_TOKEN" \
         -F "files=@$MAC_X86_64_OUTPUT_FILE" \
         "$INDEXER_URL"/toolchain/uploadfilesraw;
 }
 function upload_bundle_arm64() {
-    curl --fail -L -H "Token: $INDEXER_TOKEN" \
+    curl --fail --http1.1 -L -H "Token: $INDEXER_TOKEN" \
         -F "files=@$MAC_ARM64_OUTPUT_FILE" \
         "$INDEXER_URL"/toolchain/uploadfilesraw;
 }

diff --git a/linux/Dockerfile b/linux/Dockerfile
@@ -72,12 +72,15 @@ RUN relink.sh /toolchain/linux-output-root
 
 FROM flipperzero-toolchain-linux-build-deps AS flipperzero-toolchain-linux-tools
 RUN DEBIAN_FRONTEND=noninteractive apt update && apt -y install automake autoconf libtool python3 git
+
 ADD https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-x86_64.tar.gz /toolchain/src/src/archives/
 RUN tar -xvf /toolchain/src/src/archives/cmake-3.28.0-linux-x86_64.tar.gz -C /tmp
 RUN rsync -azv /tmp/cmake-3.28.0-linux-x86_64/ /usr
+
 ADD https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protobuf-cpp-3.21.7.tar.gz /toolchain/src/src/archives/
 RUN tar -xvf /toolchain/src/src/archives/protobuf-cpp-3.21.7.tar.gz -C /toolchain/src/src/
 RUN mv /toolchain/src/src/protobuf-3.21.7 /toolchain/src/src/protobuf
+
 ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/llvm-17.0.6.src.tar.xz /toolchain/src/src/archives/
 RUN mkdir -p /toolchain/src/src/clang-format
 RUN tar -xvf /toolchain/src/src/archives/llvm-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format
@@ -90,14 +93,13 @@ RUN mv /toolchain/src/src/clang-format/cmake-17.0.6.src /toolchain/src/src/clang
 ADD https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/third-party-17.0.6.src.tar.xz /toolchain/src/src/archives/
 RUN tar -xvf /toolchain/src/src/archives/third-party-17.0.6.src.tar.xz -C /toolchain/src/src/clang-format/
 RUN mv /toolchain/src/src/clang-format/third-party-17.0.6.src /toolchain/src/src/clang-format/third-party
-ADD https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protobuf-cpp-3.21.7.tar.gz /toolchain/src/src/archives/
-RUN tar -xvf /toolchain/src/src/archives/protobuf-cpp-3.21.7.tar.gz -C /toolchain/src/src/
-RUN mv /toolchain/src/src/protobuf-3.21.7 /toolchain/src/src/protobuf
+
 ADD https://github.com/xpack-dev-tools/openocd-xpack/releases/download/v0.12.0-2/xpack-openocd-0.12.0-2-linux-x64.tar.gz /toolchain/src/src/archives/
 RUN tar -xvf /toolchain/src/src/archives/xpack-openocd-0.12.0-2-linux-x64.tar.gz -C /tmp
 RUN rm -rf /tmp/xpack-openocd-0.12.0-2/README.md
 RUN rm -rf /tmp/xpack-openocd-0.12.0-2/distro-info
 RUN rsync -azv /tmp/xpack-openocd-0.12.0-2/ /toolchain/linux-output-root
+
 ADD scripts/relink.sh /usr/bin/
 ADD scripts/build-linux-flipper-tools.sh /toolchain/src/
 RUN bash /toolchain/src/build-linux-flipper-tools.sh

diff --git a/linux/scripts/bundle_upload.sh b/linux/scripts/bundle_upload.sh
@@ -31,7 +31,7 @@ function make_bundle() {
 }
 
 function upload_bundle() {
-    curl --fail -L -H "Token: $INDEXER_TOKEN" \
+    curl --fail --http1.1 -L -H "Token: $INDEXER_TOKEN" \
         -F "files=@$LINUX_OUTPUT_FILE" \
         "$INDEXER_URL"/toolchain/uploadfilesraw;
 }

diff --git a/windows/scripts/bundle_upload.sh b/windows/scripts/bundle_upload.sh
@@ -24,14 +24,14 @@ function prepare_dir() {
 
 function make_bundle() {
 	pushd "$WINDOWS_OUTPUT_ROOT_BASE_DIR";
-	zip \
+	zip -r \
 		"$WINDOWS_OUTPUT_FILE" \
 		"$WINDOWS_TOOLCHAIN_DIR_NAME";
 	popd;
 }
 
 function upload_bundle() {
-    curl --fail -L -H "Token: $INDEXER_TOKEN" \
+    curl --fail --http1.1 -L -H "Token: $INDEXER_TOKEN" \
         -F "files=@$WINDOWS_OUTPUT_FILE" \
         "$INDEXER_URL"/toolchain/uploadfilesraw;
 }