From a662a96bc335b5eafa679047563a5069ac8084bf Mon Sep 17 00:00:00 2001 From: Sascha Grunert Date: Mon, 12 Feb 2024 14:01:54 +0100 Subject: [PATCH] Add user namespaces tests Adding user namespaces tests for covering the `UsernamespaceMode` supported by the CRI. Fixes https://github.com/kubernetes-sigs/cri-tools/issues/1348 Signed-off-by: Sascha Grunert --- .github/workflows/build.yml | 2 +- .github/workflows/containerd.yml | 34 ++++-- .github/workflows/crio.yml | 2 +- .github/workflows/e2e.yml | 2 +- pkg/framework/util.go | 7 ++ pkg/validate/container.go | 16 +++ pkg/validate/security_context_linux.go | 158 ++++++++++++++++++++++++- traverse.go | 45 +++++++ 8 files changed, 255 insertions(+), 11 deletions(-) create mode 100644 traverse.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 596e0b4641..d6a0cf4d0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,7 @@ on: - "*" branches: - master - pull_request: + #pull_request: jobs: images: runs-on: ubuntu-latest diff --git a/.github/workflows/containerd.yml b/.github/workflows/containerd.yml index 861004a61d..0ca1e0a018 100644 --- a/.github/workflows/containerd.yml +++ b/.github/workflows/containerd.yml @@ -12,6 +12,7 @@ jobs: # build-and-critest-containerd: strategy: + fail-fast: false matrix: # ╔══════════════════╤═══════════╤═════════╗ # ║ master / release │ ubuntu │ windows ║ @@ -22,12 +23,23 @@ jobs: # ╟──────────────────┼───────────┼─────────╢ # ║ hcshim │ │ runhcs ║ # ╚══════════════════╧═══════════╧═════════╝ - os: [ubuntu-22.04, windows-2019] + os: + - ubuntu-22.04 + # - windows-2019 # not every command likes the slash in branch name. # So will use format command to replace to either `_` or '/' - version: [main, 'release{0}1.6', 'release{0}1.7'] - runtime: [io.containerd.runtime.v1.linux, io.containerd.runc.v1, io.containerd.runc.v2, containerd-shim-runhcs-v1] - runc: [runc, crun] + version: + - main + #- 'release{0}1.6' + #- 'release{0}1.7' + runtime: + - io.containerd.runc.v2 + #- io.containerd.runtime.v1.linux + #- io.containerd.runc.v1 + # - containerd-shim-runhcs-v1 + runc: + - runc + #- crun exclude: - runtime: io.containerd.runc.v1 os: windows-2019 @@ -298,7 +310,7 @@ jobs: set -o nounset set -o pipefail - BDIR="$(mktemp -d -p $PWD)" + BDIR="$(mktemp -d)" echo "containerd temp dir: ${BDIR}" mkdir -p ${BDIR}/{root,state} @@ -312,16 +324,24 @@ jobs: # Remove possibly existing containerd configuration sudo rm -rf /etc/containerd + # UserNamespaces only work for 1.7 and main + SKIP= + if [[ "${{ format(matrix.version, '/') }}" == "release/1.6" ]]; then + SKIP=--ginkgo.skip=UserNamespaces + fi + sudo PATH=$PATH /usr/local/bin/containerd -a ${BDIR}/c.sock -root ${BDIR}/root -state ${BDIR}/state -log-level debug &> ${BDIR}/containerd-cri.log & sudo /usr/local/bin/ctr -a ${BDIR}/c.sock version sudo /usr/local/sbin/runc --version - sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 - TEST_RC=$? + TEST_RC=0 + sudo -E PATH=$PATH critest --runtime-endpoint=unix:///${BDIR}/c.sock --parallel=8 ${SKIP} --ginkgo.focus=NamespaceMode_POD || TEST_RC=$? test $TEST_RC -ne 0 && cat ${BDIR}/containerd-cri.log + test $TEST_RC -ne 0 && sudo -E PATH=$PATH go run traverse.go $(sudo find $BDIR -type d -name rootfs) sudo pkill containerd echo "CONTD_CRI_DIR=$BDIR" >> $GITHUB_ENV test $TEST_RC -eq 0 || /bin/false + working-directory: ${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools - name: Run critest on Windows if: startsWith(matrix.os, 'windows') diff --git a/.github/workflows/crio.yml b/.github/workflows/crio.yml index 22466dd2c0..9971b8fe6e 100644 --- a/.github/workflows/crio.yml +++ b/.github/workflows/crio.yml @@ -5,7 +5,7 @@ on: - "*" branches: - master - pull_request: + #pull_request: jobs: # # Run CRI tests against CRI-O diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e09d625790..99aaf713d0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,7 +5,7 @@ on: - "*" branches: - master - pull_request: + #pull_request: jobs: test-e2e: name: ${{ matrix.os }} diff --git a/pkg/framework/util.go b/pkg/framework/util.go index ff875db95e..9e7213915f 100644 --- a/pkg/framework/util.go +++ b/pkg/framework/util.go @@ -220,6 +220,13 @@ func RunPodSandbox(c internalapi.RuntimeService, config *runtimeapi.PodSandboxCo return podID } +// RunPodSandboxError runs a PodSandbox and expects an error. +func RunPodSandboxError(c internalapi.RuntimeService, config *runtimeapi.PodSandboxConfig) string { + podID, err := c.RunPodSandbox(context.TODO(), config, TestContext.RuntimeHandler) + Expect(err).To(HaveOccurred()) + return podID +} + // CreatePodSandboxForContainer creates a PodSandbox for creating containers. func CreatePodSandboxForContainer(c internalapi.RuntimeService) (string, *runtimeapi.PodSandboxConfig) { podSandboxName := "create-PodSandbox-for-container-" + NewUUID() diff --git a/pkg/validate/container.go b/pkg/validate/container.go index 4cc323fb8e..871d1b7005 100644 --- a/pkg/validate/container.go +++ b/pkg/validate/container.go @@ -23,6 +23,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strings" "time" @@ -629,6 +630,21 @@ func verifyLogContents(podConfig *runtimeapi.PodSandboxConfig, logPath string, l Expect(found).To(BeTrue(), "expected log %q (stream=%q) not found in logs %+v", log, stream, msgs) } +// verifyLogContentsRe verifies the contents of container log using the provided regular expression pattern. +func verifyLogContentsRe(podConfig *runtimeapi.PodSandboxConfig, logPath string, pattern string, stream streamType) { + By("verify log contents using regex pattern") + msgs := parseLogLine(podConfig, logPath) + + found := false + for _, msg := range msgs { + if matched, _ := regexp.MatchString(pattern, msg.log); matched && msg.stream == stream { + found = true + break + } + } + Expect(found).To(BeTrue(), "expected log pattern %q (stream=%q) to match logs %+v", pattern, stream, msgs) +} + // listContainerStatsForID lists container for containerID. func listContainerStatsForID(c internalapi.RuntimeService, containerID string) *runtimeapi.ContainerStats { By("List container stats for containerID: " + containerID) diff --git a/pkg/validate/security_context_linux.go b/pkg/validate/security_context_linux.go index 8ebe496573..cf80d55a3d 100644 --- a/pkg/validate/security_context_linux.go +++ b/pkg/validate/security_context_linux.go @@ -842,6 +842,100 @@ var _ = framework.KubeDescribe("Security Context", func() { matchContainerOutput(podConfig, containerName, "Effective uid: 0\n") }) }) + + Context("UserNamespaces", func() { + var ( + podName string + defaultMapping = []*runtimeapi.IDMapping{{ + ContainerId: 0, + HostId: 1000, + Length: 100000, + }} + ) + + BeforeEach(func() { + podName = "user-namespaces-pod-" + framework.NewUUID() + }) + + It("runtime should support NamespaceMode_POD", func() { + namespaceOption := &runtimeapi.NamespaceOption{ + UsernsOptions: &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: defaultMapping, + Gids: defaultMapping, + }, + } + + hostLogPath, podLogPath := createLogTempDir(podName) + defer os.RemoveAll(hostLogPath) + podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath) + containerName := runUserNamespaceContainer(rc, ic, podID, podConfig) + + matchContainerOutputRe(podConfig, containerName, `\s+0\s+1000\s+100000\n`) + }) + + It("runtime should support NamespaceMode_NODE", func() { + namespaceOption := &runtimeapi.NamespaceOption{ + UsernsOptions: &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_NODE, + }, + } + + hostLogPath, podLogPath := createLogTempDir(podName) + defer os.RemoveAll(hostLogPath) + podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath) + containerName := runUserNamespaceContainer(rc, ic, podID, podConfig) + + // 4294967295 means that the entire range is available + matchContainerOutputRe(podConfig, containerName, `\s+0\s+0\s+4294967295\n`) + }) + + It("runtime should fail if more than one mapping provided", func() { + wrongMapping := []*runtimeapi.IDMapping{{ + ContainerId: 0, + HostId: 1000, + Length: 100000, + }, { + ContainerId: 0, + HostId: 2000, + Length: 100000, + }} + usernsOptions := &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: wrongMapping, + Gids: wrongMapping, + } + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail if container ID 0 is not mapped", func() { + mapping := []*runtimeapi.IDMapping{{ + ContainerId: 1, + HostId: 1000, + Length: 100000, + }} + usernsOptions := &runtimeapi.UserNamespace{ + Mode: runtimeapi.NamespaceMode_POD, + Uids: mapping, + Gids: mapping, + } + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail with NamespaceMode_CONTAINER", func() { + usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_CONTAINER} + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + + It("runtime should fail with NamespaceMode_TARGET", func() { + usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_TARGET} + + runUserNamespacePodWithError(rc, podName, usernsOptions) + }) + }) }) // matchContainerOutput matches log line in container logs. @@ -850,6 +944,12 @@ func matchContainerOutput(podConfig *runtimeapi.PodSandboxConfig, name, output s verifyLogContents(podConfig, fmt.Sprintf("%s.log", name), output, stdoutType) } +// matchContainerOutputRe matches log line in container logs using the provided regular expression pattern. +func matchContainerOutputRe(podConfig *runtimeapi.PodSandboxConfig, name, pattern string) { + By("check container output") + verifyLogContentsRe(podConfig, fmt.Sprintf("%s.log", name), pattern, stdoutType) +} + // createRunAsUserContainer creates the container with specified RunAsUser in ContainerConfig. func createRunAsUserContainer(rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig, prefix string) (string, string) { By("create RunAsUser container") @@ -946,7 +1046,8 @@ func createNamespacePodSandbox(rc internalapi.RuntimeService, podSandboxNamespac uid := framework.DefaultUIDPrefix + framework.NewUUID() namespace := framework.DefaultNamespacePrefix + framework.NewUUID() config := &runtimeapi.PodSandboxConfig{ - Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), + Metadata: framework.BuildPodSandboxMetadata(podSandboxName, uid, namespace, framework.DefaultAttempt), + DnsConfig: &runtimeapi.DNSConfig{}, Linux: &runtimeapi.LinuxPodSandboxConfig{ SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ NamespaceOptions: podSandboxNamespace, @@ -1281,3 +1382,58 @@ func checkSetHostname(rc internalapi.RuntimeService, containerID string, setable Expect(err).To(HaveOccurred(), msg) } } + +func runUserNamespaceContainer( + rc internalapi.RuntimeService, + ic internalapi.ImageManagerService, + podID string, + podConfig *runtimeapi.PodSandboxConfig, +) string { + By("create user namespaces container") + containerName := "user-namespaces-container-" + framework.NewUUID() + containerConfig := &runtimeapi.ContainerConfig{ + Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt), + Image: &runtimeapi.ImageSpec{ + Image: framework.TestContext.TestImageList.DefaultTestContainerImage, + UserSpecifiedImage: framework.TestContext.TestImageList.DefaultTestContainerImage, + }, + Command: []string{"cat", "/proc/self/uid_map"}, + LogPath: fmt.Sprintf("%s.log", containerName), + Linux: &runtimeapi.LinuxContainerConfig{ + SecurityContext: &runtimeapi.LinuxContainerSecurityContext{ + NamespaceOptions: podConfig.Linux.SecurityContext.NamespaceOptions, + }, + }, + } + + containerID := createContainerWithExpectation(rc, ic, containerConfig, podID, podConfig, true) + startContainer(rc, containerID) + + Eventually(func() runtimeapi.ContainerState { + return getContainerStatus(rc, containerID).State + }, time.Minute, time.Second*4).Should(Equal(runtimeapi.ContainerState_CONTAINER_EXITED)) + + return containerName +} + +func runUserNamespacePodWithError( + rc internalapi.RuntimeService, + podName string, + usernsOptions *runtimeapi.UserNamespace, +) { + uid := framework.DefaultUIDPrefix + framework.NewUUID() + namespace := framework.DefaultNamespacePrefix + framework.NewUUID() + config := &runtimeapi.PodSandboxConfig{ + Metadata: framework.BuildPodSandboxMetadata(podName, uid, namespace, framework.DefaultAttempt), + Linux: &runtimeapi.LinuxPodSandboxConfig{ + SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{ + NamespaceOptions: &runtimeapi.NamespaceOption{ + UsernsOptions: usernsOptions, + }, + }, + }, + Labels: framework.DefaultPodLabels, + } + + framework.RunPodSandboxError(rc, config) +} diff --git a/traverse.go b/traverse.go new file mode 100644 index 0000000000..88a5438834 --- /dev/null +++ b/traverse.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "os" + "strings" +) + +func main() { + path := os.Args[1] + fmt.Printf("Using path: %v\n", path) + if err := traversePath(path); err != nil { + panic(err) + } +} + +func traversePath(tPath string) error { + tempBase := os.TempDir() + if !strings.HasPrefix(tPath, tempBase) { + return fmt.Errorf("traversePath: %q is not a descendant of %q", tPath, tempBase) + } + + var path string + for _, p := range strings.SplitAfter(tPath, "/") { + path = path + p + stats, err := os.Stat(path) + if err != nil { + return err + } + + perm := stats.Mode().Perm() + if perm&0o5 == 0o5 { + continue + } + if strings.HasPrefix(tempBase, path) { + return fmt.Errorf("traversePath: directory %q MUST have read+exec permissions for others", path) + } + + if err := os.Chmod(path, perm|0o755); err != nil { + return err + } + } + + return nil +}