tests(ci): copy and upload failed test logs (#13115) (#9432)

KAG-2563
Kong · Jun 12, 2024 · 24efc15 · 24efc15
1 parent 7f31128
commit 24efc15
Show file tree

Hide file tree

Showing 2 changed files with 62 additions and 0 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -551,6 +551,7 @@ jobs:
       run: |
            echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV
            echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV
+           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV
 
     - name: Download test rerun information
       uses: actions/download-artifact@v4
@@ -597,6 +598,7 @@ jobs:
         DT_TEST: ${{ inputs.dt-test }}
         DT_ENV_ID: ${{ secrets.DT_ENV_ID }}
         DT_API_TOKEN: ${{ secrets.DT_API_TOKEN }}
+        SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
       uses: Kong/gateway-test-scheduler/runner@1ca1120856ffaf9ba87e14564ccabe91e2b4ebf3 # v5
       with:
         tests-to-run-file: test-chunk.${{ matrix.runner }}.json
@@ -606,6 +608,14 @@ jobs:
         build-root: ${{ env.BUILD_ROOT }}
         build-dest-path: "/usr/local"
 
+    - name: Upload error logs
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: busted-test-errlogs-${{ matrix.runner }}
+        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
+        retention-days: 1
+
     - name: Upload test rerun information
       if: always()
       uses: actions/upload-artifact@v4
@@ -727,6 +737,10 @@ jobs:
           $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
           $CPAN_DOWNLOAD/cpanm --notest Test::Nginx
 
+    - name: Generate environment variables
+      run: |
+           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV
+
     - name: Tests
       env:
         TEST_SUITE: pdk
@@ -738,6 +752,14 @@ jobs:
         eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
         TEST_NGINX_RANDOMIZE=1 prove -I. -r t
 
+    - name: Upload error logs
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: PDK-test-errlogs
+        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
+        retention-days: 1
+
     - name: Archive coverage stats file
       uses: actions/upload-artifact@v3
       if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
@@ -900,6 +922,7 @@ jobs:
       run: |
         echo "/usr/local/openresty/bin/" >> $GITHUB_PATH
         echo "/usr/local/openresty/nginx/sbin/" >> $GITHUB_PATH
+        echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/plugins_ee_test/${{ matrix.label }}_${{ matrix.split }} >> $GITHUB_ENV
 
     - name: Checkout Kong source code
       uses: actions/checkout@v3
@@ -947,11 +970,20 @@ jobs:
         TEST_SUITE: plugins-ee
         TEST_SPLIT: ${{ matrix.split }}
         KONG_LICENSE_DATA: ${{ steps.decrypted_license.outputs.out }}
+        SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
       run: |
         mkdir $TEST_RESULTS_XML_OUTPUT
         export XML_OUTPUT=$(realpath $TEST_RESULTS_XML_OUTPUT)
         .ci/run_tests_github.sh
 
+    - name: Upload error logs
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: plugins-ee-test-errlogs-${{ matrix.label }}-${{ matrix.split }}
+        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
+        retention-days: 1
+
     - uses: datadog/junit-upload-github-action@056dec14f19687cd7d1e97f2009df81f8e64d621
       continue-on-error: true
       # upload test results even if the job fails

diff --git a/spec/helpers.lua b/spec/helpers.lua
@@ -90,6 +90,7 @@ local resty_signal = require "resty.signal"
 local lfs = require "lfs"
 local luassert = require "luassert.assert"
 local uuid = require("kong.tools.uuid").uuid
+local colors = require "ansicolors"
 
 -- XXX EE
 local dist_constants = require "kong.enterprise_edition.distributions_constants"
@@ -2587,6 +2588,33 @@ local deep_sort do
 end
 
 
+local function copy_errlog(errlog_path)
+  local file_path = "Unknown path"
+  local line_number = "Unknown line"
+  local errlog_cache_dir = os.getenv("SPEC_ERRLOG_CACHE_DIR") or "/tmp/kong_errlog_cache"
+
+  local ok, err = pl_dir.makepath(errlog_cache_dir)
+  assert(ok, "makepath failed: " .. tostring(err))
+
+  local info = debug.getinfo(4, "Sl")
+  if info then
+    file_path = info.source:gsub("^@", "")
+    line_number = info.currentline
+  end
+
+  if string.find(file_path, '/', nil, true) then
+    file_path = string.gsub(file_path, '/', '_')
+  end
+  file_path = errlog_cache_dir .. "/" .. file_path:gsub("%.lua$", "_") .. "line_" .. line_number .. '.log'
+
+  ok, err = pl_file.copy(errlog_path, file_path)
+  if ok then
+    print(colors("%{yellow}Log saved as: " .. file_path .. "%{reset}"))
+  else
+    print(colors("%{red}Failed to save error log for test " .. file_path .. ": " .. err))
+  end
+end
+
 --- Assertion to check the status-code of a http response.
 -- @function status
 -- @param expected the expected status code
@@ -2619,6 +2647,8 @@ local function res_status(state, args)
     args.n = 3
 
     if res.status == 500 then
+      copy_errlog(conf.nginx_err_logs)
+
       -- on HTTP 500, we can try to read the server's error logs
       -- for debugging purposes (very useful for travis)
       local str = pl_file.read(conf.nginx_err_logs)