Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests(invalidations): address flakiness #11571

Closed
wants to merge 2 commits into from
Closed

tests(invalidations): address flakiness #11571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a546b1
tests(invalidations): isolate test
samugi Sep 14, 2023
03bd0ea
tests(invalidations): address flakiness
samugi Sep 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 3 additions & 103 deletions .ci/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ function red() {
echo -e "\033[1;31m$*\033[0m"
}

export BUSTED_ARGS="--no-k -o htest -v --exclude-tags=flaky,ipv6"
export BUSTED_ARGS="--no-k -o htest --repeat 200 -v --exclude-tags=flaky,ipv6"

if [ "$KONG_TEST_DATABASE" == "postgres" ]; then
export TEST_CMD="bin/busted $BUSTED_ARGS,off"
Expand All @@ -29,111 +29,11 @@ else
export TEST_CMD="bin/busted $BUSTED_ARGS,postgres,db"
fi

if [[ "$KONG_TEST_COVERAGE" = true ]]; then
export TEST_CMD="$TEST_CMD --keep-going"
fi


if [ "$TEST_SUITE" == "integration" ]; then
if [[ "$TEST_SPLIT" == first* ]]; then
# GitHub Actions, run first batch of integration tests
eval "$TEST_CMD" $(ls -d spec/02-integration/* | sort | grep -v 05-proxy)

elif [[ "$TEST_SPLIT" == second* ]]; then
# GitHub Actions, run second batch of integration tests
# Note that the split here is chosen carefully to result
# in a similar run time between the two batches, and should
# be adjusted if imbalance become significant in the future
eval "$TEST_CMD" $(ls -d spec/02-integration/* | sort | grep 05-proxy)
eval "$TEST_CMD" spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua

else
# Non GitHub Actions
eval "$TEST_CMD" spec/02-integration/
fi
fi

if [ "$TEST_SUITE" == "dbless" ]; then
eval "$TEST_CMD" spec/02-integration/02-cmd \
spec/02-integration/05-proxy \
spec/02-integration/04-admin_api/02-kong_routes_spec.lua \
spec/02-integration/04-admin_api/15-off_spec.lua \
spec/02-integration/08-status_api/01-core_routes_spec.lua \
spec/02-integration/08-status_api/03-readiness_endpoint_spec.lua \
spec/02-integration/11-dbless
fi
if [ "$TEST_SUITE" == "plugins" ]; then
set +ex
rm -f .failed

if [[ "$TEST_SPLIT" == first* ]]; then
# GitHub Actions, run first batch of plugin tests
PLUGINS=$(ls -d spec/03-plugins/* | head -n22)

elif [[ "$TEST_SPLIT" == second* ]]; then
# GitHub Actions, run second batch of plugin tests
# Note that the split here is chosen carefully to result
# in a similar run time between the two batches, and should
# be adjusted if imbalance become significant in the future
PLUGINS=$(ls -d spec/03-plugins/* | tail -n+23)

else
# Non GitHub Actions
PLUGINS=$(ls -d spec/03-plugins/*)
fi

for p in $PLUGINS; do
echo
cyan "--------------------------------------"
cyan $(basename $p)
cyan "--------------------------------------"
echo

$TEST_CMD $p || echo "* $p" >> .failed
done

if [[ "$TEST_SPLIT" == second* ]] || [[ "$TEST_SPLIT" != first* ]]; then
cat kong-*.rockspec | grep kong- | grep -v zipkin | grep -v sidecar | grep "~" | grep -v kong-prometheus-plugin | while read line ; do
REPOSITORY=`echo $line | sed "s/\"/ /g" | awk -F" " '{print $1}'`
VERSION=`luarocks show $REPOSITORY | grep $REPOSITORY | head -1 | awk -F" " '{print $2}' | cut -f1 -d"-"`
REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-prometheus-plugin/kong-plugin-prometheus/g'`
REPOSITORY=`echo $REPOSITORY | sed -e 's/kong-proxy-cache-plugin/kong-plugin-proxy-cache/g'`

echo
cyan "--------------------------------------"
cyan $REPOSITORY $VERSION
cyan "--------------------------------------"
echo

git clone https://github.com/Kong/$REPOSITORY.git --branch $VERSION --single-branch /tmp/test-$REPOSITORY || \
git clone https://github.com/Kong/$REPOSITORY.git --branch v$VERSION --single-branch /tmp/test-$REPOSITORY
sed -i 's/grpcbin:9000/localhost:15002/g' /tmp/test-$REPOSITORY/spec/*.lua
sed -i 's/grpcbin:9001/localhost:15003/g' /tmp/test-$REPOSITORY/spec/*.lua
cp -R /tmp/test-$REPOSITORY/spec/fixtures/* spec/fixtures/ || true
pushd /tmp/test-$REPOSITORY
luarocks make
popd

$TEST_CMD /tmp/test-$REPOSITORY/spec/ || echo "* $REPOSITORY" >> .failed

done
fi

if [ -f .failed ]; then
echo
red "--------------------------------------"
red "Plugin tests failed:"
red "--------------------------------------"
cat .failed
exit 1
else
exit 0
fi
fi
if [ "$TEST_SUITE" == "pdk" ]; then
prove -I. -r t
fi
if [ "$TEST_SUITE" == "unit" ]; then
unset KONG_TEST_NGINX_USER KONG_PG_PASSWORD KONG_TEST_PG_PASSWORD
scripts/autodoc
bin/busted -v -o htest spec/01-unit
make lint
fi
109 changes: 94 additions & 15 deletions spec/02-integration/06-invalidations/02-core_entities_invalidations_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,50 @@ local ssl_fixtures = require "spec.fixtures.ssl"

local POLL_INTERVAL = 0.1

local function get_admin_2_plugins_iterator_id()
local admin_client_2 = helpers.http_client("127.0.0.1", 9001)
local res = assert(admin_client_2:send {
method = "GET",
path = "/cache/plugins_iterator:version",
})
local body = assert.res_status(200, res)
local json = cjson.decode(body)
return json.message
end

local function assert_admin_2_wait(request, res_status, res_not_message)
local ret
helpers.wait_until(function()
local admin_client_2 = helpers.http_client("127.0.0.1", 9001)
finally(function()
admin_client_2:close()
end)

local res = admin_client_2:send(request)
if not res then
return false
end
if res.status ~= res_status then
return false
end
local body = res:read_body()
if not body then
return false
end
local json = cjson.decode(body)
if res_not_message then
if not json.message:match("^[%w-]+$") then
return false
end
if json.message == res_not_message then
return false
end
end
ret = json
return true
end, 10)
return ret
end

local function assert_proxy_2_wait(request, res_status, res_headers)
helpers.wait_until(function()
Expand Down Expand Up @@ -651,9 +695,9 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(201, admin_res)

helpers.wait_for_all_config_update({
forced_admin_port = 8001,
forced_proxy_port = 8000,
})
forced_admin_port = 8001,
forced_proxy_port = 8000,
})

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker
Expand Down Expand Up @@ -697,9 +741,9 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(200, admin_res)

helpers.wait_for_all_config_update({
forced_admin_port = 8001,
forced_proxy_port = 8000,
})
forced_admin_port = 8001,
forced_proxy_port = 8000,
})

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker
Expand Down Expand Up @@ -734,9 +778,9 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(200, admin_res)

helpers.wait_for_all_config_update({
forced_admin_port = 8001,
forced_proxy_port = 8000,
})
forced_admin_port = 8001,
forced_proxy_port = 8000,
})

local cert_1_old = get_cert(8443, "test.wildcard.com")
assert.certificate(cert_1_old).has.cn("localhost")
Expand Down Expand Up @@ -769,9 +813,9 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(200, admin_res)

helpers.wait_for_all_config_update({
forced_admin_port = 8001,
forced_proxy_port = 8000,
})
forced_admin_port = 8001,
forced_proxy_port = 8000,
})

local cert_1_old = get_cert(8443, "test.wildcard_updated.com")
assert.certificate(cert_1_old).has.cn("localhost")
Expand Down Expand Up @@ -803,9 +847,9 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(204, admin_res)

helpers.wait_for_all_config_update({
forced_admin_port = 8001,
forced_proxy_port = 8000,
})
forced_admin_port = 8001,
forced_proxy_port = 8000,
})

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker
Expand Down Expand Up @@ -906,6 +950,7 @@ for _, strategy in helpers.each_strategy() do
}
}, 200, { ["Dummy-Plugin"] = ngx.null })

local iterator_id_before = get_admin_2_plugins_iterator_id()
-- create Plugin

local admin_res_plugin = assert(admin_client_1:send {
Expand All @@ -928,6 +973,12 @@ for _, strategy in helpers.each_strategy() do
forced_proxy_port = 8000,
})

-- wait for new plugins iterator version on node 2
assert_admin_2_wait({
method = "GET",
path = "/cache/plugins_iterator:version",
}, 200, iterator_id_before)

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker

Expand All @@ -951,6 +1002,7 @@ for _, strategy in helpers.each_strategy() do
end)

it("on update", function()
local iterator_id_before = get_admin_2_plugins_iterator_id()
local admin_res_plugin = assert(admin_client_1:send {
method = "PATCH",
path = "/plugins/" .. service_plugin_id,
Expand All @@ -970,6 +1022,12 @@ for _, strategy in helpers.each_strategy() do
forced_proxy_port = 8000,
})

-- wait for new plugins iterator version on node 2
assert_admin_2_wait({
method = "GET",
path = "/cache/plugins_iterator:version",
}, 200, iterator_id_before)

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker

Expand All @@ -993,6 +1051,7 @@ for _, strategy in helpers.each_strategy() do
end)

it("on delete", function()
local iterator_id_before = get_admin_2_plugins_iterator_id()
local admin_res_plugin = assert(admin_client_1:send {
method = "DELETE",
path = "/plugins/" .. service_plugin_id,
Expand All @@ -1004,6 +1063,12 @@ for _, strategy in helpers.each_strategy() do
forced_proxy_port = 8000,
})

-- wait for new plugins iterator version on node 2
assert_admin_2_wait({
method = "GET",
path = "/cache/plugins_iterator:version",
}, 200, iterator_id_before)

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker

Expand Down Expand Up @@ -1057,6 +1122,7 @@ for _, strategy in helpers.each_strategy() do
assert.is_nil(res.headers["Dummy-Plugin"])
end

local iterator_id_before = get_admin_2_plugins_iterator_id()
local admin_res_plugin = assert(admin_client_1:send {
method = "POST",
path = "/plugins",
Expand All @@ -1076,6 +1142,12 @@ for _, strategy in helpers.each_strategy() do
forced_proxy_port = 8000,
})

-- wait for new plugins iterator version on node 2
assert_admin_2_wait({
method = "GET",
path = "/cache/plugins_iterator:version",
}, 200, iterator_id_before)

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker

Expand Down Expand Up @@ -1109,6 +1181,7 @@ for _, strategy in helpers.each_strategy() do
assert.res_status(200, res_1)
assert.equal("1", res_1.headers["Dummy-Plugin"])

local iterator_id_before = get_admin_2_plugins_iterator_id()
local admin_res = assert(admin_client_1:send {
method = "DELETE",
path = "/plugins/" .. global_dummy_plugin_id,
Expand All @@ -1120,6 +1193,12 @@ for _, strategy in helpers.each_strategy() do
forced_proxy_port = 8000,
})

-- wait for new plugins iterator version on node 2
assert_admin_2_wait({
method = "GET",
path = "/cache/plugins_iterator:version",
}, 200, iterator_id_before)

-- no need to wait for workers propagation (lua-resty-events)
-- because our test instance only has 1 worker

Expand Down
Loading