diff --git a/.github/workflows/build-publish-cfg.yaml b/.github/workflows/build-publish-cfg.yaml deleted file mode 100644 index 6b8467984c..0000000000 --- a/.github/workflows/build-publish-cfg.yaml +++ /dev/null @@ -1,167 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Build and Publish Docker Image - -on: - workflow_call: - inputs: - name: - description: Name of the container - type: string - required: true - directory: - description: Directory of service - type: string - required: true - file: - description: Dockerfile to build - type: string - required: true - template-file: - description: Template dockefile to resolve - type: string - required: false - www-name: - description: Name of the www artifact - type: string - required: false - www-build-dir: - description: Directory destination to download the www artifact - type: string - required: false - - -env: - REGISTRY: ghcr.io - -jobs: - build-and-publish-with-cfg: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Validate WWW Inputs - run: | - if [[ -n "${{ inputs.www-name }}" && -z "${{ inputs.www-build-dir }}" ]]; then - echo "inputs.www-build-dir is missing." - exit 1 - fi - if [[ -z "${{ inputs.www-name }}" && -n "${{ inputs.www-build-dir }}" ]]; then - echo "inputs.www-name is missing." - exit 1 - fi - echo "Both inputs are either empty or set. Validation successful." - - - uses: actions/download-artifact@v3 - if: ${{ inputs.www-name && inputs.www-build-dir }} - with: - name: ${{ inputs.www-name }} - path: ${{ inputs.www-build-dir }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for docker image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ inputs.name }} - tags: | - type=raw,enable={{is_default_branch}},value=vnext - type=sha,enable={{is_default_branch}},prefix=vnext-,format=short - type=ref,event=pr,prefix=vnext-pr,suffix=-{{sha}} - type=ref,event=pr,prefix=vnext-pr - type=ref,enable={{is_default_branch}},event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - - name: Resolve template file - if: ${{ inputs.template-file }} - run: | - mkdir -p $(dirname ${{ inputs.file }}) - awk '{gsub("@NAME@","${{ inputs.name }}")} {gsub("@DIRECTORY@","${{ inputs.directory }}")} {print}' ${{ inputs.template-file }} > ${{ inputs.file }} - - - name: Set build args - id: build-args - run: | - echo "commit_date=$(date -u +%FT%TZ --date=@$(git show --format=%ct HEAD --quiet))" >> $GITHUB_OUTPUT - short_commit=$(git show --format=%h HEAD --quiet) - echo "short_commit=${short_commit}" >> $GITHUB_OUTPUT - echo "date=$(date -u +%FT%TZ)" >> $GITHUB_OUTPUT - version=$(git describe --tags --abbrev=0 | sed 's/^v//') - release_url="${{ github.server_url }}/${{ github.repository }}/releases/tag/${{ github.ref_name }}" - if [[ "${{ github.ref }}" != "refs/tags/"* ]]; then - version="${version}-pr${{ github.event.number }}" - elif curl --fail-with-body -s "$release_url" > /dev/null; then - echo "release_url=$release_url" >> $GITHUB_OUTPUT - fi - echo "version=$version" >> $GITHUB_OUTPUT - - - name: First try to build and push docker image - uses: docker/build-push-action@v5 - # This needs to be true to proceed to the next step of failure - continue-on-error: true - # You need to specify an id to be able to tell what the status of this action was - id: BuildAndPush1 - with: - context: ./ - platforms: linux/amd64,linux/arm64 - builder: ${{ steps.buildx.outputs.name }} - build-args: | - DIRECTORY=${{ inputs.directory }} - NAME=${{ inputs.name }} - COMMIT_DATE=${{ steps.build-args.outputs.commit_date }} - SHORT_COMMIT=${{ steps.build-args.outputs.short_commit }} - DATE=${{ steps.build-args.outputs.date }} - VERSION=${{ steps.build-args.outputs.version }} - RELEASE_URL=${{ steps.build-args.outputs.release_url }} - file: ${{ inputs.file }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Second try to build and push docker image when first build fails - # Only run this step if step 1 fails. It knows that step one failed because we specified an `id` for the first step - if: steps.BuildAndPush1.outcome == 'failure' - uses: docker/build-push-action@v5 - with: - context: ./ - platforms: linux/amd64,linux/arm64 - builder: ${{ steps.buildx.outputs.name }} - build-args: | - DIRECTORY=${{ inputs.directory }} - NAME=${{ inputs.name }} - COMMIT_DATE=${{ steps.build-args.outputs.commit-date }} - SHORT_COMMIT=${{ steps.build-args.outputs.short-commit }} - DATE=${{ steps.build-args.outputs.date }} - VERSION=${{ steps.build-args.outputs.version }} - file: ${{ inputs.file }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/build-publish.yaml b/.github/workflows/build-publish.yaml deleted file mode 100644 index 8d8b730d00..0000000000 --- a/.github/workflows/build-publish.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Create and publish a docker images to github - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref_name != 'main' }} - -on: - push: - branches: - - main - tags: - - "*" - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - build-www: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: true - - - name: Build www - run: | - cd http-gateway - make build-www - - - name: Upload www - uses: actions/upload-artifact@v3 - with: - name: www - path: http-gateway/web/build - retention-days: 1 - - build-and-publish-image-www: - needs: build-www - strategy: - fail-fast: false - matrix: - include: - - name: http-gateway - directory: http-gateway - file: http-gateway/Dockerfile - - name: bundle - directory: bundle - file: bundle/Dockerfile - uses: ./.github/workflows/build-publish-cfg.yaml - with: - name: ${{ matrix.name }} - directory: ${{ matrix.directory }} - file: ${{ matrix.file }} - www-name: www - www-build-dir: http-gateway/web/build - - build-and-publish-image: - strategy: - fail-fast: false - matrix: - include: - - name: resource-aggregate - directory: resource-aggregate - file: .tmp/docker/resource-aggregate/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: resource-directory - directory: resource-directory - file: .tmp/docker/resource-directory/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: coap-gateway - directory: coap-gateway - file: .tmp/docker/coap-gateway/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: cloud2cloud-connector - directory: cloud2cloud-connector - file: .tmp/docker/cloud2cloud-connector/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: cloud2cloud-gateway - directory: cloud2cloud-gateway - file: .tmp/docker/cloud2cloud-gateway/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: certificate-authority - directory: certificate-authority - file: .tmp/docker/certificate-authority/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: grpc-gateway - directory: grpc-gateway - file: .tmp/docker/grpc-gateway/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: identity-store - directory: identity-store - file: .tmp/docker/identity-store/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: mock-oauth-server - directory: test/oauth-server - file: .tmp/docker/mock-oauth-server/Dockerfile - template-file: tools/docker/Dockerfile.in - - name: test-cloud-server - directory: test/cloud-server - file: test/cloud-server/Dockerfile - - name: cert-tool - directory: tools/cert-tool - file: tools/cert-tool/Dockerfile - - name: nats-server-config-reloader - directory: tools/nats-server-config-reloader - file: tools/nats-server-config-reloader/Dockerfile - # coap-gateway builded by Golang 1.19.4 has an issue with TLS handshake. - # This issue is reproducible with real devices that connect to AWS. - # This seems to be caused by the device's old mbedtls library: - # https://github.com/Mbed-TLS/mbedtls/tree/d81c11b8ab61fd5b2da8133aa73c5fe33a0633eb - - name: coap-gateway-go1-18 - directory: coap-gateway - file: tools/docker/Dockerfile.go1.18 - uses: ./.github/workflows/build-publish-cfg.yaml - with: - name: ${{ matrix.name }} - directory: ${{ matrix.directory }} - file: ${{ matrix.file }} - template-file: ${{ matrix.template-file }} - diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml deleted file mode 100644 index 7fbc87a422..0000000000 --- a/.github/workflows/builds.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Run special builds that should be supported -name: Builds - -on: - pull_request: - workflow_dispatch: - -jobs: - builds: - strategy: - fail-fast: false - matrix: - include: - # required by coap-gateway-go1-18 in https://github.com/plgd-dev/hub/blob/main/.github/workflows/publishDockerImagesGhcr.yml - - name: go1.18 - go-version: "~1.18" - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version || '^1.18' }} - check-latest: true - - - run: | - go version - go build ./coap-gateway/cmd/service diff --git a/.github/workflows/checkFormat.yml b/.github/workflows/checkFormat.yml deleted file mode 100644 index 17d4ced40c..0000000000 --- a/.github/workflows/checkFormat.yml +++ /dev/null @@ -1,30 +0,0 @@ -# Check that only correctly formatted golang code is merged -name: Check Format - -on: - pull_request: - branches: - - main - -jobs: - format: - runs-on: ubuntu-latest - - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - uses: actions/checkout@v4 - with: - # fetch branches and history so `git merge-base` in check-format-on-diff works correctly - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version: "^1.18" # The Go version to download (if necessary) and use. - check-latest: true - - - name: Check formatting - shell: bash - run: tools/format/check-format-on-diff.sh --simplify --branch origin/${{ github.base_ref }} diff --git a/.github/workflows/ghcr-cleanup-package.yaml b/.github/workflows/ghcr-cleanup-package.yaml new file mode 100644 index 0000000000..2b41cb6641 --- /dev/null +++ b/.github/workflows/ghcr-cleanup-package.yaml @@ -0,0 +1,78 @@ +name: Delete old packages +on: + push: + workflow_dispatch: + inputs: + package: + description: "Package name" + required: true + type: choice + options: + # - test-cloud-server + # - cert-tool + # - grpc-gateway + - coap-gateway + # - resource-directory + - identity-store + - http-gateway + - mock-oauth-server + - bundle + - resource-aggregate + - cloud2cloud-connector + - cloud2cloud-gateway + # - certificate-authority + - coap-gateway-go1-18 + +jobs: + ghcr-cleanup-old-packages: + name: Delete old tagged images + runs-on: ubuntu-latest + steps: + - name: Delete older than a month vnext images + uses: snok/container-retention-policy@v2 + with: + image-names: hub/mock-oauth-server + cut-off: One month ago UTC + account-type: org + org-name: plgd-dev + filter-tags: vnext-* + skip-tags: vnext-pr*,main + token: ${{ secrets.GHCR_CLEANUP_PAT }} + + - name: Delete older than a month vnext-pr images + uses: snok/container-retention-policy@v2 + with: + image-names: hub/mock-oauth-server + cut-off: One week ago UTC + account-type: org + org-name: plgd-dev + filter-tags: vnext-pr* + skip-tags: main + token: ${{ secrets.GHCR_CLEANUP_PAT }} + + # ghcr-cleanup-untagged-packages: + # name: Delete untagged packages + # if: ${{ always() }} + # runs-on: ubuntu-latest + # needs: ghcr-cleanup-old-packages + # steps: + # - name: Set up Docker Buildx for delete untagged images action + # uses: docker/setup-buildx-action@v3 + + # - name: Login to GitHub Container Registry for delete untagged images action + # uses: docker/login-action@v3 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GHCR_CLEANUP_PAT }} + + # - name: Delete untagged images with no dependency + # uses: Chizkiyahu/delete-untagged-ghcr-action@v3 + # with: + # package_name: hub/coap-gateway + # repository: ${{ github.repository }} + # repository_owner: ${{ github.repository_owner }} + # token: ${{ secrets.GHCR_CLEANUP_PAT }} + # owner_type: org + # untagged_only: true + # except_untagged_multiplatform: true diff --git a/.github/workflows/ghcr-cleanup.yaml b/.github/workflows/ghcr-cleanup.yaml deleted file mode 100644 index 56ec965be7..0000000000 --- a/.github/workflows/ghcr-cleanup.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: Delete old ghcr images -on: - schedule: - - cron: "15 1 * * *" # every day at 1:15am - pull_request: - types: [closed] - workflow_dispatch: - -jobs: - pull-request-ghcr-cleanup: - if: ${{ github.event_name == 'pull_request' }} - name: Delete images related to closed PR - runs-on: ubuntu-latest - steps: - - name: Delete images related to closed PR - uses: snok/container-retention-policy@v2 - with: - image-names: hub/* - cut-off: now UTC - account-type: org - org-name: plgd-dev - filter-tags: vnext-pr${{ github.event.pull_request.number }}* - token: ${{ secrets.GHCR_CLEANUP_PAT }} - - nightly-ghcr-cleanup: - if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} - name: Delete old vnext images - runs-on: ubuntu-latest - steps: - - name: Delete older than a month vnext images - uses: snok/container-retention-policy@v2 - with: - image-names: hub/* - cut-off: One month ago UTC - account-type: org - org-name: plgd-dev - filter-tags: vnext-* - skip-tags: vnext-pr*, main - token: ${{ secrets.GHCR_CLEANUP_PAT }} - - - name: Set up Docker Buildx for delete untagged images action - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry for delete untagged images action - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GHCR_CLEANUP_PAT }} - - - name: Delete untagged images with no dependency - uses: Chizkiyahu/delete-untagged-ghcr-action@v3 - with: - repository: ${{ github.repository }} - repository_owner: ${{ github.repository_owner }} - token: ${{ secrets.GHCR_CLEANUP_PAT }} - owner_type: org - untagged_only: true - except_untagged_multiplatform: true diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml deleted file mode 100644 index 5e29874bb6..0000000000 --- a/.github/workflows/golangci-lint.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Golangci-lint - -# golangci-lint is a fast Go linters runner. It runs linters in parallel, -# uses caching, supports yaml config, has integrations with all major IDE and -# has dozens of linters included. -# see: https://github.com/golangci/golangci-lint-action - -on: - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - golangci: - name: lint - runs-on: ubuntu-latest - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "^1.18" # The Go version to download (if necessary) and use. - check-latest: true - cache: false - - - run: go version - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 - with: - version: latest - args: --timeout=5m diff --git a/.github/workflows/measureMemory.yml b/.github/workflows/measureMemory.yml deleted file mode 100644 index 9268c7f91e..0000000000 --- a/.github/workflows/measureMemory.yml +++ /dev/null @@ -1,299 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Measure memory - -# Controls when the action will run. Triggers the workflow push is only on main branch and PR on any branch. -on: - push: - branches: - - main - tags: - - "*" - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref_name != 'main' }} - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - generateSummary: - needs: coapGateway - runs-on: ubuntu-latest - steps: - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - path: outputs - - name: Update summary - run: | - echo "### CoAP Gateway memory usage" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "The following table displays the memory usage of the coap-gateway when connected to varying numbers of devices, resources, and resource sizes. One by one, the devices were connected and the following actions were executed for each device:" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- Sign up" >> $GITHUB_STEP_SUMMARY - echo "- Sign in" >> $GITHUB_STEP_SUMMARY - echo "- Publish resources" >> $GITHUB_STEP_SUMMARY - echo "- Synchronize with twin device" >> $GITHUB_STEP_SUMMARY - echo "- Ping until test ends" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "| Num devices | Num Resources | Resource Data Size(KB) | Current memory usage (MB) | Duration (seconds) |" >> $GITHUB_STEP_SUMMARY - echo "| ----------- | ------------- | ------------------ | ------------------------- | -------- |" >> $GITHUB_STEP_SUMMARY - jq -s '.|=sort_by(.NumDevices,.NumResources,.LogLevel,.LogDumpBody,.ResourceDataSize)' outputs/*/*.json > output.json - for row in $(cat output.json | jq -r '.[] | @base64'); do - _jq() { - echo ${row} | base64 --decode | jq -r ${1} - } - duration=$(echo "scale=2 ; $(_jq '.Duration') / 1000000000" | bc ) - resourceDataSize=$(echo "scale=2 ; $(_jq '.ResourceDataSize') / 1024" | bc ) - echo "| $(_jq '.NumDevices') | $(_jq '.NumResources') | $resourceDataSize | $(_jq '.CurrentMemRSS') | $duration |" >> $GITHUB_STEP_SUMMARY - done - - coapGateway: - # The type of runner that the job will run on - runs-on: ubuntu-latest - strategy: - fail-fast: false - # max number of running jobs is 20 for a single organization, so lets keep at least 4 available to all other repositories - max-parallel: 16 - matrix: - include: - #1 - - name: devices/1/resources/1/size/16KB - numDevices: 1 - numResources: 1 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1/resources/1/size/1KB - numDevices: 1 - numResources: 1 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1/resources/125/size/16KB - numDevices: 1 - numResources: 125 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1/resources/125/size/1KB - numDevices: 1 - numResources: 125 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1/resources/250/size/16KB - numDevices: 1 - numResources: 250 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1/resources/250/size/1KB - numDevices: 1 - numResources: 250 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1/resources/500/size/16KB - numDevices: 1 - numResources: 500 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1/resources/500/size/1KB - numDevices: 1 - numResources: 500 - resourceDataSize: 1024 - logLevel: info - logDumpBody: true - timeout: 120m - #1000 - - name: devices/1000/resources/1/size/16KB - numDevices: 1000 - numResources: 1 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1000/resources/1/size/1KB - numDevices: 1000 - numResources: 1 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1000/resources/125/size/16KB - numDevices: 1000 - numResources: 125 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1000/resources/125/size/1KB - numDevices: 1000 - numResources: 125 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1000/resources/250/size/16KB - numDevices: 1000 - numResources: 250 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1000/resources/250/size/1KB - numDevices: 1000 - numResources: 250 - resourceDataSize: 1024 - timeout: 120m - - name: devices/1000/resources/500/size/16KB - numDevices: 1000 - numResources: 500 - resourceDataSize: 16384 - timeout: 120m - - name: devices/1000/resources/500/size/1KB - numDevices: 1000 - numResources: 500 - resourceDataSize: 1024 - timeout: 120m - #2500 - - name: devices/2500/resources/1/size/16KB - numDevices: 2500 - numResources: 1 - resourceDataSize: 16384 - timeout: 120m - - name: devices/2500/resources/1/size/1KB - numDevices: 2500 - numResources: 1 - resourceDataSize: 1024 - timeout: 120m - - name: devices/2500/resources/125/size/16KB - numDevices: 2500 - numResources: 125 - resourceDataSize: 16384 - timeout: 120m - - name: devices/2500/resources/125/size/1KB - numDevices: 2500 - numResources: 125 - resourceDataSize: 1024 - timeout: 120m - - name: devices/2500/resources/250/size/16KB - numDevices: 2500 - numResources: 250 - resourceDataSize: 16384 - timeout: 120m - - name: devices/2500/resources/250/size/1KB - numDevices: 2500 - numResources: 250 - resourceDataSize: 1024 - timeout: 120m - - name: devices/2500/resources/250/size/16KB - numDevices: 2500 - numResources: 500 - resourceDataSize: 16384 - timeout: 120m - - name: devices/2500/resources/500/size/1KB - numDevices: 2500 - numResources: 500 - resourceDataSize: 1024 - timeout: 120m - #5000 - - name: devices/5000/resources/1/size/16KB - numDevices: 5000 - numResources: 1 - resourceDataSize: 16384 - timeout: 120m - - name: devices/5000/resources/1/size/1KB - numDevices: 5000 - numResources: 1 - resourceDataSize: 1024 - timeout: 120m - - name: devices/5000/resources/125/size/16KB - numDevices: 5000 - numResources: 125 - resourceDataSize: 16384 - timeout: 120m - - name: devices/5000/resources/125/size/1KB - numDevices: 5000 - numResources: 125 - resourceDataSize: 1024 - timeout: 120m - - name: devices/5000/resources/250/size/16KB - numDevices: 5000 - numResources: 250 - resourceDataSize: 16384 - timeout: 120m - - name: devices/5000/resources/250/size/1KB - numDevices: 5000 - numResources: 250 - resourceDataSize: 1024 - timeout: 120m - - name: devices/5000/resources/500/size/16KB - numDevices: 5000 - numResources: 500 - resourceDataSize: 16384 - timeout: 120m - - name: devices/5000/resources/500/size/1KB - numDevices: 5000 - numResources: 500 - resourceDataSize: 1024 - timeout: 120m - #10000 - - name: devices/10000/resources/1/size/16KB - numDevices: 10000 - numResources: 1 - resourceDataSize: 16384 - timeout: 120m - - name: devices/10000/resources/1/size/1KB - numDevices: 10000 - numResources: 1 - resourceDataSize: 1024 - timeout: 120m - - name: devices/10000/resources/125/size/16KB - numDevices: 10000 - numResources: 125 - resourceDataSize: 16384 - timeout: 120m - - name: devices/10000/resources/125/size/1KB - numDevices: 10000 - numResources: 125 - resourceDataSize: 1024 - timeout: 120m - - name: devices/10000/resources/250/size/16KB - numDevices: 10000 - numResources: 250 - resourceDataSize: 16384 - timeout: 120m - - name: devices/10000/resources/250/size/1KB - numDevices: 10000 - numResources: 250 - resourceDataSize: 1024 - timeout: 120m - - name: devices/10000/resources/500/size/4KB - numDevices: 10000 - numResources: 500 - resourceDataSize: 4096 - timeout: 300m - - name: devices/10000/resources/500/size/1KB - numDevices: 10000 - numResources: 500 - resourceDataSize: 1024 - timeout: 120m - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: Run a test - continue-on-error: true - run: | - make test/mem TEST_MEMORY_COAP_GATEWAY_RESOURCE_DATA_SIZE=${{ matrix.resourceDataSize }} TEST_TIMEOUT=${{ matrix.timeout }} TEST_MEMORY_COAP_GATEWAY_NUM_DEVICES=${{ matrix.numDevices }} TEST_MEMORY_COAP_GATEWAY_NUM_RESOURCES=${{ matrix.numResources }} | tee >(grep "TestMemoryWithDevices.result:" | sed -e "s/.*TestMemoryWithDevices.result://g" | jq -r -c > out.json) - - - name: Dump file - if: success() - run: | - jq -rc "." out.json - - - name: Upload file - if: success() - uses: actions/upload-artifact@v3 - with: - name: ${{ hashFiles('out.json') || 'none' }} - path: out.json - retention-days: 1 - if-no-files-found: warn diff --git a/.github/workflows/publishDockerImagesGhcr.yml b/.github/workflows/publishDockerImagesGhcr.yml deleted file mode 100644 index 571a688aa0..0000000000 --- a/.github/workflows/publishDockerImagesGhcr.yml +++ /dev/null @@ -1,155 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Create and publish a docker images to github - -on: - workflow_run: - workflows: - - "Test" - branches: - - main - types: - - completed - push: - tags: - - "*" - workflow_dispatch: - inputs: - push: - description: "Push to container registry" - required: false - type: boolean - default: false -env: - REGISTRY: ghcr.io - VNEXT_BRANCH: main - -jobs: - build-and-publish-image: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - name: resource-aggregate - directory: resource-aggregate - file: .tmp/docker/resource-aggregate/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: resource-directory - directory: resource-directory - file: .tmp/docker/resource-directory/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: coap-gateway - directory: coap-gateway - file: .tmp/docker/coap-gateway/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: cloud2cloud-connector - directory: cloud2cloud-connector - file: .tmp/docker/cloud2cloud-connector/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: cloud2cloud-gateway - directory: cloud2cloud-gateway - file: .tmp/docker/cloud2cloud-gateway/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: certificate-authority - directory: certificate-authority - file: .tmp/docker/certificate-authority/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: grpc-gateway - directory: grpc-gateway - file: .tmp/docker/grpc-gateway/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: http-gateway - directory: http-gateway - file: http-gateway/Dockerfile - - name: identity-store - directory: identity-store - file: .tmp/docker/identity-store/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: mock-oauth-server - directory: test/oauth-server - file: .tmp/docker/mock-oauth-server/Dockerfile - template_file: tools/docker/Dockerfile.in - - name: bundle - directory: bundle - file: bundle/Dockerfile - - name: test-cloud-server - directory: test/cloud-server - file: test/cloud-server/Dockerfile - - name: cert-tool - directory: tools/cert-tool - file: tools/cert-tool/Dockerfile - # coap-gateway builded by Golang 1.19.4 has an issue with TLS handshake. - # This issue is reproducible with real devices that connect to AWS. - # This seems to be caused by the device's old mbedtls library: - # https://github.com/Mbed-TLS/mbedtls/tree/d81c11b8ab61fd5b2da8133aa73c5fe33a0633eb - - name: coap-gateway-go1-18 - directory: coap-gateway - file: tools/docker/Dockerfile.go1.18 - permissions: - contents: read - packages: write - - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set vars.branch - id: vars - run: BRANCH=${GITHUB_REF#refs/*/} && echo ${BRANCH} && echo "branch=${BRANCH}" >> $GITHUB_OUTPUT - - - name: Extract metadata (tags, labels) for docker image - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.name }} - tags: | - type=schedule - type=ref,event=branch - type=raw,enable=${{ startsWith(steps.vars.outputs.branch, env.VNEXT_BRANCH) }},value=vnext - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=sha,enable=${{ startsWith(steps.vars.outputs.branch, env.VNEXT_BRANCH) }},prefix=vnext-,format=short - - - name: Resolve template file - if: ${{ matrix.template_file }} - run: | - mkdir -p $(dirname ${{ matrix.file }}) - awk '{gsub("@NAME@","${{ matrix.name }}")} {gsub("@DIRECTORY@","${{ matrix.directory }}")} {print}' ${{ matrix.template_file }} > ${{ matrix.file }} - - - name: Build and push docker image - uses: docker/build-push-action@v5 - with: - context: ./ - platforms: linux/amd64,linux/arm64 - builder: ${{ steps.buildx.outputs.name }} - build-args: | - DIRECTORY=${{ matrix.directory }} - NAME=${{ matrix.name }} - file: ${{ matrix.file }} - push: ${{ github.event_name != 'workflow_dispatch' || inputs.push }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/releaseHelmCharts.yml b/.github/workflows/releaseHelmCharts.yml deleted file mode 100644 index 9cb6945e0e..0000000000 --- a/.github/workflows/releaseHelmCharts.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Release Helm Charts -on: - release: - types: [published] - -jobs: - release-charts: - runs-on: ubuntu-latest - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Get the version - id: get_version - run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT - - name: Publish Helm chart - uses: stefanprodan/helm-gh-pages@master - with: - token: ${{ secrets.GITHUB_TOKEN }} - charts_url: https://charts.plgd.dev - helm_version: "3.7.1" - app_version: "${{ steps.get_version.outputs.VERSION }}" - chart_version: "${{ steps.get_version.outputs.VERSION }}" diff --git a/.github/workflows/run-bundle.yml b/.github/workflows/run-bundle.yml deleted file mode 100644 index 716554534f..0000000000 --- a/.github/workflows/run-bundle.yml +++ /dev/null @@ -1,196 +0,0 @@ -name: Run bundle on workflow dispatch from github - -on: - workflow_dispatch: - inputs: - bundle_tag: - description: 'Tag of bundle' - required: true - default: 'vnext' - cloud_server_tag: - description: 'Tag of cloud server' - required: true - default: 'vnext' - cloud_server_log_level: - description: 'Log level of cloud server (disabled, trace, debug, info, warning, error)' - required: true - default: 'info' - number_devices: - description: 'Number of devices' - required: true - default: '1' - number_device_resources: - description: 'Number of device resources' - required: true - default: '1' - -env: - cloud_server_image: ghcr.io/iotivity/iotivity-lite/cloud-server-discovery-resource-observable-debug - cloud_server_tag: vnext - cloud_server_log_level: info - number_devices: 1 - number_device_resources: 1 - -jobs: - run-bundle: - runs-on: ubuntu-latest - concurrency: - group: ${{ github.workflow }} - cancel-in-progress: ${{ github.ref_name != 'main' }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Install ngrok and run ngrok for HTTP - uses: luisboto/ngrok-tunnel-action@v0.1.7.2 - with: - timeout: 6h - port: 443 - tunnel_type: http - ngrok_authtoken: ${{ secrets.NGROK_0_HTTP }} - save_url_to_filename: ngrok_http.txt - - - name: Print bundle HTTP URL - run: | - cat ngrok_http.txt - - - name: Parse bundle HTTP URL - id: parse_ngrok_http - run: | - echo "fqdn=$(cat ngrok_http.txt | sed -e 's/[htcp][htcps]*:\/\///g' | cut -d ":" -f 1)" >> "$GITHUB_OUTPUT" - - - name: Setup ngrok for CoAP - run: | - echo 'web_addr: 4045' > ngrok_coap.yml - ./ngrok authtoken ${{ secrets.NGROK_0_COAP }} --config ngrok_coap.yml - - - name: Run ngrok for CoAP - run: | - ( timeout 6h ./ngrok tcp 5684 --config ngrok_coap.yml) & - sleep 1 - curl http://127.0.0.1:4045/api/tunnels | jq -r ".tunnels[0].public_url" > ngrok_coap.txt - - - name: Print bundle CoAP Address - run: | - cat ngrok_coap.txt - - - name: Parse bundle CoAP external address - id: parse_ngrok_coap - run: | - echo "coap_gateway_external_address=$(cat ngrok_coap.txt | sed -e 's/[htcp][htcps]*:\/\///g')" >> "$GITHUB_OUTPUT" - - - name: Extract metadata (tags, labels) for docker image - id: docker_meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/plgd-dev/hub/bundle - tags: | - type=raw,enable={{is_default_branch}},value=vnext - type=ref,event=pr,prefix=vnext-pr - type=ref,enable={{is_default_branch}},event=branch - - - name: Set tag of bundle - id: meta - run: | - if [ -n "${{ inputs.bundle_tag }}" ]; then - echo "tag=${{ inputs.bundle_tag }}" >> "$GITHUB_OUTPUT" - elif [ -n "${{ steps.docker_meta.outputs.version }}" ]; then - echo "tag=${{ steps.docker_meta.outputs.version }}" >> "$GITHUB_OUTPUT" - else - echo "tag=vnext" >> "$GITHUB_OUTPUT" - fi - - - name: Run bundle - run: | - mkdir -p `pwd`/.tmp/bundle - docker run -d \ - --name=bundle \ - --network=host \ - -v `pwd`/.tmp/bundle:/data \ - -e LOG_LEVEL=DEBUG \ - -e COAP_GATEWAY_HUB_ID=$(uuidgen) \ - -e COAP_GATEWAY_EXTERNAL_ADDRESS=${{ steps.parse_ngrok_coap.outputs.coap_gateway_external_address }} \ - -e FQDN=${{ steps.parse_ngrok_http.outputs.fqdn }} \ - -e OWNER_CLAIM=sub \ - ghcr.io/plgd-dev/hub/bundle:${{ steps.meta.outputs.tag }} - - - name: Set cloud server vars - id: cloud_server_vars - run: | - if [ -n "${{ inputs.cloud_server_tag }}" ]; then - echo "tag=${{ inputs.cloud_server_tag }}" >> "$GITHUB_OUTPUT" - else - echo "tag=${{ env.cloud_server_tag }}" >> "$GITHUB_OUTPUT" - fi - if [ -n "${{ inputs.number_devices }}" ]; then - echo "number_devices=${{ inputs.number_devices }}" >> "$GITHUB_OUTPUT" - else - echo "number_devices=${{ env.number_devices }}" >> "$GITHUB_OUTPUT" - fi - if [ -n "${{ inputs.number_device_resources }}" ]; then - echo "number_device_resources=${{ inputs.number_device_resources }}" >> "$GITHUB_OUTPUT" - else - echo "number_device_resources=${{ env.number_device_resources }}" >> "$GITHUB_OUTPUT" - fi - if [ -n "${{ inputs.cloud_server_log_level }}" ]; then - echo "log_level=${{ inputs.cloud_server_log_level }}" >> "$GITHUB_OUTPUT" - else - echo "log_level=${{ env.cloud_server_log_level }}" >> "$GITHUB_OUTPUT" - fi - - - name: Run simulators - run: | - mkdir -p `pwd`/.tmp/devices - docker run --rm -d \ - --network=host \ - --name devices \ - --privileged \ - -v `pwd`/.tmp/devices:/tmp \ - -e NUM_DEVICES=${{ steps.cloud_server_vars.outputs.number_devices }} \ - ${{ env.cloud_server_image }}:${{ steps.cloud_server_vars.outputs.tag }} -l ${{ steps.cloud_server_vars.outputs.log_level }} device-simulator auth coaps+tcp://127.0.0.1:5684 cis apn ${{ steps.cloud_server_vars.outputs.number_device_resources }} - - - name: Onboard simulators - run: | - go build -o onboard ./bundle/client/ob - ./onboard -maxNum ${{ steps.cloud_server_vars.outputs.number_devices }} - - - name: Pause workflow - # wait for 5 hours and generate lines to prevent workflow from being killed - run: | - docker logs bundle - for ((i=0;i<300;i++)); do - sleep 60 - echo "Workflow paused for $((i+1)) minutes from 300 minutes" - done - - - name: Stop simulators - if: cancelled() || failure() || success() - run: | - docker stop devices - - - name : Stop bundle - if: cancelled() || failure() || success() - run: | - docker stop bundle - - - name: Collect bundle logs - if: cancelled() || failure() || success() - uses: actions/upload-artifact@v3 - with: - name: bundle_logs - path: .tmp/bundle/log/*.log - retention-days: 5 - - - name: Collect devices logs - if: cancelled() || failure() || success() - uses: actions/upload-artifact@v3 - with: - name: devices_logs - path: .tmp/devices/*.log - retention-days: 5 - - - diff --git a/.github/workflows/staticAnalysis.yml b/.github/workflows/staticAnalysis.yml deleted file mode 100644 index 3b80062ebb..0000000000 --- a/.github/workflows/staticAnalysis.yml +++ /dev/null @@ -1,47 +0,0 @@ -# Run static analysis checks -name: Static Analysis - -on: - pull_request: - branches: - - main - workflow_dispatch: - -jobs: - analysis: - runs-on: ubuntu-latest - - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 - with: - go-version: "^1.18" # The Go version to download (if necessary) and use. - check-latest: true - - run: go version - - - name: Run go vet - run: go vet ./... - - - name: Install and run gocyclo - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go install github.com/fzipp/gocyclo/cmd/gocyclo@latest - gocyclo -over 15 -ignore ".pb(.gw)?.go$|_test.go$|wsproxy" . || echo "gocyclo detected too complex functions" - - - name: Install and run misspell - run: | - # check misspelling in all files in repository - export PATH=${PATH}:`go env GOPATH`/bin - go install github.com/client9/misspell/cmd/misspell@latest - find . -type f -exec misspell {} \; - - - name: Check yaml field names and yaml tags - run: tools/validate/validateYaml.py || echo "invalid yaml field name(s) detected" - - - name: Check json tags - run: tools/validate/validateJson.py || echo "invalid json tag(s) detected" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 82406149ae..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,146 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Test - -# Controls when the action will run. Triggers the workflow push is only on main branch and PR on any branch. -on: - push: - branches: - - main - tags: - - "*" - pull_request: - workflow_dispatch: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref_name != 'main' }} - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - test: - # The type of runner that the job will run on - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - # test with check race with coverage and sonarcloud - - name: test - cmd: test - checkRace: "true" - coapGateway: - log: - level: "debug" - dumpBody: "true" - - # test without check race - - name: test/norace - cmd: test - coapGateway: - log: - level: "debug" - dumpBody: "true" - - # test without check race and with ECDSA-SHA256 signature and P384 elliptic curve certificates - - name: test/norace-384 - cmd: test - args: CERT_TOOL_SIGN_ALG=ECDSA-SHA384 CERT_TOOL_ELLIPTIC_CURVE=P384 - - # test without check race with logs from all services - - name: test/norace/logs - cmd: test - coapGateway: - log: - level: "debug" - dumpBody: "true" - grpcGateway: - log: - level: "debug" - dumpBody: "true" - resourceAggregate: - log: - level: "debug" - dumpBody: "true" - identityStore: - log: - level: "debug" - dumpBody: "true" - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - name: CPU Info - run: | - cat /proc/cpuinfo - echo "Number of cores: $(nproc)" - echo "Number of threads: $(nproc --all)" - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: Run a test - run: | - make ${{ matrix.cmd }} TEST_CHECK_RACE=${{ matrix.checkRace }} \ - TEST_COAP_GATEWAY_LOG_LEVEL=${{ matrix.coapGateway.log.level }} TEST_COAP_GATEWAY_LOG_DUMP_BODY=${{ matrix.coapGateway.log.dumpBody }} \ - TEST_RESOURCE_AGGREGATE_LOG_LEVEL=${{ matrix.resourceAggregate.log.level }} TEST_RESOURCE_AGGREGATE_LOG_DUMP_BODY=${{ matrix.resourceAggregate.log.dumpBody }} \ - TEST_GRPC_GATEWAY_LOG_LEVEL=${{ matrix.grpcGateway.log.level }} TEST_GRPC_GATEWAY_LOG_DUMP_BODY=${{ matrix.grpcGateway.log.dumpBody }} \ - TEST_IDENTITY_STORE_LOG_LEVEL=${{ matrix.identityStore.log.level }} TEST_IDENTITY_STORE_LOG_DUMP_BODY=${{ matrix.identityStore.log.dumpBody }} \ - ${{ matrix.args }} - - - name: Remove simulators container - if: ${{ always() }} - run: | - make simulators/remove - - - name: Collect cloud server logs when the test fails - if: ${{ failure() }} - run: | - cat .tmp/devsim*/*.log - - - name: Prepare upload files - run: | - mkdir -p ./outputs - cp -r .tmp/coverage ./outputs/ - cp -r .tmp/report ./outputs/ - - - name: Upload coverage and report files - uses: actions/upload-artifact@v3 - with: - name: ${{ hashFiles('./outputs') || 'none' }} - path: ./outputs - retention-days: 1 - if-no-files-found: warn - - coverage-sonar-cloud-scan: - needs: test - # The type of runner that the job will run on - runs-on: ubuntu-latest - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - - name: Download artifacts - uses: actions/download-artifact@v3 - with: - path: ./outputs - - - name: Prepare coverage and report files - run: | - mkdir -p .tmp/coverage - mkdir -p .tmp/report - find ./outputs -name "*.coverage.txt" -exec sh -c 'cp $1 .tmp/coverage/$(echo $1 | sed "s/[\/.]/-/g" ).coverage.txt' _ {} \; - find ./outputs -name "*.report.json" -exec sh -c 'cp $1 .tmp/report/$(echo $1 | sed "s/[\/.]/-/g" ).report.json' _ {} \; - - - name: Publish the coverage for main branch - if: ${{ github.ref == 'refs/heads/main' }} - run: bash <(curl -s https://codecov.io/bash) - - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}