tests(CI): copy and upload failed test logs (#13115)

KAG-2563

.github/workflows/build_and_test.yml

@@ -259,6 +259,7 @@ jobs:
       run: |
            echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV
            echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV
+           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV
 
     - name: Build & install dependencies
       run: |
@@ -290,13 +291,22 @@ jobs:
         DD_CIVISIBILITY_AGENTLESS_ENABLED: true
         DD_TRACE_GIT_METADATA_ENABLED: true
         DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+        SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
       uses: Kong/gateway-test-scheduler/runner@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3
       with:
         tests-to-run-file: test-chunk.${{ matrix.runner }}.json
         failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }}
         test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }}
         setup-venv-path: ${{ env.BUILD_ROOT }}
 
+    - name: Upload error logs
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: busted-test-errlogs-${{ matrix.runner }}
+        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
+        retention-days: 1
+
     - name: Upload test rerun information
       if: always()
       uses: actions/upload-artifact@v3
@@ -361,6 +371,10 @@ jobs:
           $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
           $CPAN_DOWNLOAD/cpanm --notest Test::Nginx
 
+    - name: Generate environment variables
+      run: |
+           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV
+
     - name: Tests
       env:
         TEST_SUITE: pdk
@@ -372,6 +386,14 @@ jobs:
           eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
           prove -I. -r t
 
+    - name: Upload error logs
+      if: failure()
+      uses: actions/upload-artifact@v3
+      with:
+        name: PDK-test-errlogs
+        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
+        retention-days: 1
+
     - name: Archive coverage stats file
       uses: actions/upload-artifact@v3
       if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}

spec/helpers.lua

@@ -78,6 +78,7 @@ local resty_signal = require "resty.signal"
 local lfs = require "lfs"
 local luassert = require "luassert.assert"
 local uuid = require("kong.tools.uuid").uuid
+local colors = require "ansicolors"
 
 ffi.cdef [[
   int setenv(const char *name, const char *value, int overwrite);
@@ -2473,6 +2474,33 @@ local deep_sort do
 end
 
 
+local function copy_errlog(errlog_path)
+  local file_path = "Unknown path"
+  local line_number = "Unknown line"
+  local errlog_cache_dir = os.getenv("SPEC_ERRLOG_CACHE_DIR") or "/tmp/kong_errlog_cache"
+
+  local ok, err = pl_dir.makepath(errlog_cache_dir)
+  assert(ok, "makepath failed: " .. tostring(err))
+
+  local info = debug.getinfo(4, "Sl")
+  if info then
+    file_path = info.source:gsub("^@", "")
+    line_number = info.currentline
+  end
+
+  if string.find(file_path, '/', nil, true) then
+    file_path = string.gsub(file_path, '/', '_')
+  end
+  file_path = errlog_cache_dir .. "/" .. file_path:gsub("%.lua$", "_") .. "line_" .. line_number .. '.log'
+
+  ok, err = pl_file.copy(errlog_path, file_path)
+  if ok then
+    print(colors("%{yellow}Log saved as: " .. file_path .. "%{reset}"))
+  else
+    print(colors("%{red}Failed to save error log for test " .. file_path .. ": " .. err))
+  end
+end
+
 --- Assertion to check the status-code of a http response.
 -- @function status
 -- @param expected the expected status code
@@ -2505,6 +2533,8 @@ local function res_status(state, args)
     args.n = 3
 
     if res.status == 500 then
+      copy_errlog(conf.nginx_err_logs)
+
       -- on HTTP 500, we can try to read the server's error logs
       -- for debugging purposes (very useful for travis)
       local str = pl_file.read(conf.nginx_err_logs)