diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f278ee244..ec30480b7 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -53,7 +53,6 @@ jobs: run: | python -m pip install --upgrade pip pip install setuptools wheel auditwheel auditwheel-symbols build - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi # Build Paddle2ONNX - name: Build Paddle2ONNX diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml new file mode 100644 index 000000000..b34850ad6 --- /dev/null +++ b/.github/workflows/release_linux_aarch64.yml @@ -0,0 +1,44 @@ +name: LinuxRelease_aarch64 + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + # python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: [ '3.8'] + architecture: [ 'x64' ] + + steps: + # Checkout the latest branch of Paddle2ONNX. + - name: Checkout Paddle2ONNX + uses: actions/checkout@v4 + with: + submodules: true + + # setting up qemu for enabling aarch64 binary execution on x86 machine + - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + + - name: Build on manylinux2014_aarch64 + uses: docker://quay.io/pypa/manylinux2014_aarch64:latest + with: + entrypoint: bash + args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_aarch64 CentOS + + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 + with: + name: wheels + path: dist + + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.ZHENG_BICHENG_PYPI_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml index dbad2acc3..0a8e7e02d 100644 --- a/.github/workflows/release_linux_x86_64.yml +++ b/.github/workflows/release_linux_x86_64.yml @@ -1,4 +1,4 @@ -name: Build Package +name: LinuxRelease_x86_64 on: release: @@ -12,11 +12,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12'] + # python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: [ '3.8'] architecture: [ 'x64' ] - # For the sake of simplicity in testing, the Paddle2ONNX packaging program will temporarily only package executable files for Python 3.8. - # In the future, we will need to extend support to cover Python 3.8 through Python 3.10. steps: # Checkout the latest branch of Paddle2ONNX. - name: Checkout Paddle2ONNX @@ -28,7 +27,7 @@ jobs: uses: docker://quay.io/pypa/manylinux2014_x86_64:latest with: entrypoint: bash - args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 + args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 with: @@ -39,4 +38,4 @@ jobs: uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 with: user: __token__ - password: ${{ secrets.ZHENG_BICHENG_PYPI_TOKEN }} \ No newline at end of file + password: ${{ secrets.PADDLE2ONNX_API_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/scripts/build_protobuf_unix.sh b/.github/workflows/scripts/build_protobuf_unix.sh deleted file mode 100644 index 85fb09c98..000000000 --- a/.github/workflows/scripts/build_protobuf_unix.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Copyright (c) ONNX Project Contributors -# -# SPDX-License-Identifier: Apache-2.0 - -export CORE_NUMBER=$1 - -if [[ -z "$CORE_NUMBER" ]]; then - export CORE_NUMBER=1 -fi - -# Build protobuf from source with -fPIC on Unix-like system -ORIGINAL_PATH=$(pwd) -cd .. -git clone https://github.com/protocolbuffers/protobuf.git -b v3.16.0 -cd protobuf -mkdir build_source && cd build_source -cmake ../cmake -DCMAKE_INSTALL_PREFIX=${PWD}/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -make -j$CORE_NUMBER && make install -export PATH=${PWD}/installed_protobuf_lib/bin:${PATH} -cd $ORIGINAL_PATH \ No newline at end of file diff --git a/.github/workflows/scripts/download_protobuf.sh b/.github/workflows/scripts/download_protobuf.sh new file mode 100644 index 000000000..4010f5256 --- /dev/null +++ b/.github/workflows/scripts/download_protobuf.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# Detect the operating system +OS=$(uname -s) +ARCH=$(uname -m) + +# Check if the operating system is Linux +if [ "$OS" = "Linux" ]; then + if [[ "$ARCH" == "x86_64" ]]; then + protobuf_tgz_name="protobuf-linux-x64-3.16.0.tgz" + elif [[ "$ARCH" == "arm"* || "$ARCH" == "aarch64" ]]; then + protobuf_tgz_name="protobuf-linux-aarch64-3.16.0.tgz" + else + echo "When the operating system is Linux, the system architecture only supports (x86_64 and aarch64), but the current architecture is $ARCH." + exit 1 + fi +# Check if the operating system is Darwin (macOS) +elif [ "$OS" = "Darwin" ]; then + if [[ "$ARCH" == "x86_64" ]]; then + protobuf_tgz_name="protobuf-osx-x86_64-3.16.0.tgz" + elif [[ "$ARCH" == "arm64" ]]; then + protobuf_tgz_name="protobuf-osx-arm64-3.16.0.tgz" + else + echo "When the operating system is Darwin, the system architecture only supports (x86_64 and arm64), but the current architecture is $ARCH." + exit 1 + fi +else + echo "The system only supports (Linux and Darwin), but the current system is $OS." +fi + +protobuf_url="https://bj.bcebos.com/paddle2onnx/third_party/$protobuf_tgz_name" +wget $protobuf_url +protobuf_svae_dir="$PWD/installed_protobuf" +mkdir -p $protobuf_svae_dir +tar -zxf $protobuf_tgz_name -C $protobuf_svae_dir +export PATH=$protobuf_svae_dir/bin:${PATH} \ No newline at end of file diff --git a/.github/workflows/scripts/entrypoint.sh b/.github/workflows/scripts/entrypoint.sh index c81d3c6e5..7530e9768 100644 --- a/.github/workflows/scripts/entrypoint.sh +++ b/.github/workflows/scripts/entrypoint.sh @@ -9,28 +9,26 @@ set -e -x # CLI arguments PY_VERSION=$1 PLAT=$2 -GITHUB_EVENT_NAME=$3 +SYSTEM_NAME=$3 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib # Compile wheels # Need to be updated if there is a new Python Version -if [ "$(uname -m)" == "aarch64" ]; then - PIP_INSTALL_COMMAND="$PY_VERSION -m pip install --no-cache-dir -q" - PYTHON_COMMAND="$PY_VERSION" -else - declare -A python_map=( ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39" ["3.10"]="cp310-cp310" ["3.11"]="cp311-cp311" ["3.12"]="cp312-cp312") - PY_VER=${python_map[$PY_VERSION]} - PIP_INSTALL_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q" - PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python" -fi +declare -A python_map=( ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39" ["3.10"]="cp310-cp310" ["3.11"]="cp311-cp311" ["3.12"]="cp312-cp312") +PY_VER=${python_map[$PY_VERSION]} +PIP_INSTALL_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q" +PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python" # Update pip and install cmake $PIP_INSTALL_COMMAND --upgrade pip $PIP_INSTALL_COMMAND cmake # Build protobuf from source -source .github/workflows/scripts/build_protobuf_unix.sh "$(nproc)" +if [[ "$SYSTEM_NAME" == "CentOS" ]]; then + yum install -y wget +fi +source .github/workflows/scripts/download_protobuf.sh # Build Paddle2ONNX wheels $PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; } diff --git a/.gitignore b/.gitignore index 05c8a3f98..657cfb98a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,22 @@ +# System-generated temporary files .DS_Store -*~ +# Temporary files that may be generated when loading Paddle2ONNX using an IDE +.idea +.vscode -*.pyc -.pydevproject -.eggs/* -dist/* -.setuptools* -paddle2onnx.egg-info/* -*.pdmodel -*_*.onnx -*.log +# Files that change dynamically when compiling Paddle2ONNX version.py paddle2onnx/mappers_registry.h -# CMD -build/* -paddle2onnx-* - -# Clion +# Temporary files that may be generated when building Paddle2ONNX +protobuf-* +installed_protobuf +build cmake-build-* -.idea +paddle2onnx.egg-info/* +dist/* -# VSCode -.vscode \ No newline at end of file +# Temporary files automatically generated when executing Paddle2ONNX unit tests +*.pdmodel +*.onnx \ No newline at end of file diff --git a/VERSION_NUMBER b/VERSION_NUMBER index e2cac26c1..b966e81a4 100644 --- a/VERSION_NUMBER +++ b/VERSION_NUMBER @@ -1 +1 @@ -1.2.3 \ No newline at end of file +1.2.4 \ No newline at end of file diff --git a/docs/zh/compile_docker.md b/docs/zh/compile_docker.md index c89fab965..3eeda0b1e 100644 --- a/docs/zh/compile_docker.md +++ b/docs/zh/compile_docker.md @@ -1,26 +1,70 @@ # Docker编译安装Paddle2ONNX -Paddle2ONNX编译安装需要确保环境满足以下需求 -- cmake >= 3.18.0 -- protobuf == 3.16.0 +## 1 拉取manylinux镜像 -注意:Paddle2ONNX产出的模型,在使用ONNX Runtime推理时,要求使用最新版本(1.10.0版本以及上),如若需要使用低版本(1.6~1.10之间),则需要将ONNX版本降至1.8.2,在执行完`git submodule update`后,执行如下命令,然后再进行编译 +根据系统架构拉取不同的manylinux镜像 + +```bash +# Pull manylinux2014_x86_64 +docker pull quay.io/pypa/manylinux2014_x86_64 +docker create --name p2o_build -it quay.io/pypa/manylinux2014_x86_64 /bin/bash +# Pull manylinux2014_x86_64 +docker pull quay.io/pypa/manylinux2014_aarch64 +docker create --name p2o_build -it quay.io/pypa/manylinux2014_aarch64 /bin/bash ``` -cd Paddle2ONNX/third/onnx -git checkout v1.8.1 + +## 2 创建并进入容器 + +创建并进入 Docker 容器 + +```bash +docker start p2o_build +docker exec -it p2o_build /bin/bash ``` -拉取manylinux镜像并创建容器 +## 3 拉取 Paddle2ONNX 仓库 + +执行以下命令来拉取并初始化 Paddle2ONNX 仓库 ```bash -docker pull quay.io/pypa/manylinux2014_x86_64 -docker run --name p2o_build -d quay.io/pypa/manylinux2014_x86_64 +git clone https://github.com/PaddlePaddle/Paddle2ONNX.git +cd Paddle2ONNX +git submodule init +git submodule update +``` + +## 4 获取 protobuf 依赖库 + +### 4.1 使用 protobuf 预编译库 + +执行以下命令来下载 protobuf 依赖库 + +```bash +source .github/workflows/scripts/download_protobuf.sh ``` -创建容器并运行 +### 4.2 下载并编译 protobuf 预编译库 + +执行以下命令来下载并编译 protobuf 预编译库 ```bash -docker start p2o_build -docker exec -it p2o_build bash +git clone https://github.com/protocolbuffers/protobuf.git +cd protobuf +git checkout v3.16.0 +mkdir build_source && cd build_source +cmake ../cmake -DCMAKE_INSTALL_PREFIX=${PWD}/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release +make -j8 +make install + +# 将编译目录加入环境变量 +export PATH=${PWD}/installed_protobuf_lib/bin:${PATH} ``` +## 5 执行编译和安装 + +```bash +/opt/python/cp38-cp38/bin/pip install setuptools wheel auditwheel auditwheel-symbols build +cd /path/to/Paddle2ONNX +/opt/python/cp38-cp38/bin/python -m build +/opt/python/cp38-cp38/bin/pip install dist/*.whl +```