skaffold.yaml

apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: cert-manager
manifests:
  kustomize:
    paths:
      - hack/config/cert-manager
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
    hooks:
      after:
        - host:
            command:
              - /usr/bin/env
              - bash
              - -c
              - |
                for i in $(seq 1 20); do
                  if [ "$(kubectl get validatingwebhookconfiguration cert-manager-webhook -oyaml 2>/dev/null | yq '.webhooks[].clientConfig.caBundle')" != "null" ] ; then
                    exit 0
                  fi
                  echo "Waiting until CA has been injected into cert-manager webhook"
                  sleep 5
                done
                exit 1
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: cert-manager-resources
manifests:
  kustomize:
    paths:
      - hack/config/cert-manager/resources
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: ingress-nginx
manifests:
  kustomize:
    paths:
      - hack/config/ingress-nginx/default
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
    hooks:
      before:
        - host:
            command:
              - /usr/bin/env
              - bash
              - -c
              # job template is immutable, delete the old job to prepare for upgrade
              - kubectl -n ingress-nginx delete job --ignore-not-found ingress-nginx-admission-create ingress-nginx-admission-patch
profiles:
  - name: kind
    activation:
      - kubeContext: kind-.*
    manifests:
      kustomize:
        paths:
          - hack/config/ingress-nginx/kind
  - name: shoot
    activation:
      - kubeContext: .*--sharding.*
    manifests:
      kustomize:
        paths:
          - hack/config/ingress-nginx/shoot
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: kyverno
manifests:
  kustomize:
    paths:
      - hack/config/kyverno
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
    hooks:
      after:
        - host:
            command:
              - /usr/bin/env
              - bash
              - -c
              - |
                for i in $(seq 1 20); do
                  # create dummy policy with dry-run enabled to test availability of webhook
                  if kubectl create --raw "/apis/kyverno.io/v1/clusterpolicies?dryRun=All" -f <(echo '{"apiVersion": "kyverno.io/v1", "kind": "ClusterPolicy", "metadata": {"name": "test"}, "spec": {}}') >/dev/null ; then
                    exit 0
                  fi
                  echo "Waiting until kyverno webhook is ready to handle policy creation"
                  sleep 5
                done
                exit 1
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: policy
requires:
  - configs:
      - kyverno
profiles:
  - name: shoot
    activation:
      - kubeContext: .*--sharding.*
    manifests:
      kustomize:
        paths:
          - hack/config/policy/shoot
    deploy:
      kubectl:
        flags:
          apply:
            - --server-side
            - --force-conflicts
        defaultNamespace: ""
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: sharder
build:
  artifacts:
    - image: ghcr.io/timebertt/kubernetes-controller-sharding/sharder
      ko:
        dependencies:
          paths:
            - go.mod
            - 'cmd/**/*.go'
            - 'pkg/**/*.go'
        main: ./cmd/sharder
  tagPolicy:
    inputDigest: {}
  local:
    concurrency: 0
manifests:
  kustomize:
    paths:
      - config/default
  hooks:
    before:
      - host:
          # ensure deepcopy, CRDs, and RBAC are up-to-date
          command:
            - make
            - generate-fast
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
profiles:
  - name: kind
    activation:
      - kubeContext: kind-.*
    patches:
      - op: replace
        path: /manifests/kustomize/paths/0
        value: hack/config/sharder/local
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: shard
build:
  artifacts:
    - image: ghcr.io/timebertt/kubernetes-controller-sharding/shard
      ko:
        dependencies:
          paths:
            - go.mod
            - 'cmd/**/*.go'
            - 'pkg/**/*.go'
        main: ./cmd/shard
  tagPolicy:
    inputDigest: {}
  local:
    concurrency: 0
manifests:
  kustomize:
    paths:
      - hack/config/shard/clusterring
      - hack/config/shard/shard
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: monitoring-crds
manifests:
  kustomize:
    paths:
      - hack/config/monitoring/crds
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: monitoring
requires:
  - configs:
      - monitoring-crds
manifests:
  kustomize:
    paths:
      - hack/config/monitoring/default
      - config/monitoring
  hooks:
    before:
      - host:
          command:
            - hack/config/monitoring/default/ensure-admin-password.sh
deploy:
  kubectl:
    flags:
      apply:
        - --server-side
        - --force-conflicts
    defaultNamespace: ""
portForward:
  - resourceType: service
    resourceName: grafana
    namespace: monitoring
    port: http
    localPort: 3001
  - resourceType: service
    resourceName: prometheus-k8s
    namespace: monitoring
    port: web
    localPort: 9091
profiles:
  - name: shoot
    activation:
      - kubeContext: .*--sharding.*
    patches:
      - op: replace
        path: /manifests/kustomize/paths/0
        value: hack/config/monitoring/shoot
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: profiling
profiles:
- name: profiling
  manifests:
    kustomize:
      paths:
        - hack/config/profiling
    hooks:
      before:
        - host:
            command:
              - hack/config/profiling/ensure-admin-password.sh
  deploy:
    kubectl:
      flags:
        apply:
          - --server-side
          - --force-conflicts
      defaultNamespace: ""
  portForward:
    - resourceType: service
      resourceName: parca
      namespace: parca
      port: http
      localPort: 7071
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: webhosting-operator
requires:
- configs:
  - monitoring-crds
  - kyverno
build:
  artifacts:
  - image: ghcr.io/timebertt/kubernetes-controller-sharding/webhosting-operator
    ko:
      dependencies:
        paths:
        - go.mod
        - webhosting-operator/go.mod
        - 'pkg/**/*.go'
        - 'webhosting-operator/**/*.go'
      main: ./webhosting-operator/cmd/webhosting-operator
  tagPolicy:
    inputDigest: {}
  local:
    concurrency: 0
manifests:
  kustomize:
    paths:
    # default configuration: only run operator shards and use external sharding implementation via ClusterRing
    - webhosting-operator/config/manager/overlays/default
    - webhosting-operator/config/monitoring/default
deploy:
  kubectl:
    flags:
      apply:
      - --server-side
      - --force-conflicts
    defaultNamespace: ""
    hooks:
      before:
      - host:
          # ensure CRDs and RBAC are up-to-date
          command:
          - make
          - generate-fast-webhosting
profiles:
- name: debug
  activation:
  - command: debug
  patches:
  - op: replace
    path: /manifests/kustomize/paths/0
    value: webhosting-operator/config/manager/overlays/debug
- name: non-sharded
  patches:
  - op: replace
    path: /manifests/kustomize/paths/0
    # singleton controller without sharding for comparison
    value: webhosting-operator/config/manager/overlays/non-sharded
# The following profiles are variants of the default and non-sharded profiles for running on the shoot cluster
# with dns for websites enabled.
- name: shoot
  activation:
  - kubeContext: .*--sharding.*
  patches:
  - op: replace
    path: /manifests/kustomize/paths/0
    # default configuration: only run operator shards and use external sharding implementation via ClusterRing
    value: webhosting-operator/config/manager/overlays/shoot/default
  - op: add
    path: /manifests/kustomize/paths/-
    value: webhosting-operator/config/policy
- name: shoot-non-sharded
  patches:
  - op: replace
    path: /manifests/kustomize/paths/0
    # singleton controller without sharding for comparison
    value: webhosting-operator/config/manager/overlays/shoot/non-sharded
---
apiVersion: skaffold/v4beta10
kind: Config
metadata:
  name: experiment
requires:
- configs:
  - monitoring-crds
profiles:
- name: expirement
  activation:
  - env: EXPERIMENT_SCENARIO=.+
  build:
    artifacts:
    - image: ghcr.io/timebertt/kubernetes-controller-sharding/experiment
      ko:
        dependencies:
          paths:
          - go.mod
          - webhosting-operator/go.mod
          - 'pkg/**/*.go'
          - 'webhosting-operator/**/*.go'
        main: ./webhosting-operator/cmd/experiment
    tagPolicy:
      inputDigest: {}
    local:
      concurrency: 0
  manifests:
    kustomize:
      paths:
      - webhosting-operator/config/experiment/{{ .EXPERIMENT_SCENARIO }}
  deploy:
    kubectl:
      flags:
        apply:
        - --server-side
        - --force-conflicts
      defaultNamespace: ""
      hooks:
        before:
        - host:
            command:
            - /usr/bin/env
            - bash
            - -c
            - |
              active_pods="$(kubectl -n experiment get job experiment -ojsonpath='{.status.active}' 2>/dev/null)"
              if [ "${active_pods:-0}" -gt 0 ] && [ -z "$EXPERIMENT_DELETE_FORCE" ] ; then
                echo "Experiment is running currently, refusing to delete the job. Set EXPERIMENT_DELETE_FORCE to override."
                exit 1
              fi
              kubectl -n experiment delete job experiment --ignore-not-found --wait=true
        - host:
            command:
            - /usr/bin/env
            - bash
            - -c
            - |
              if kubectl get clusterring example &>/dev/null || kubectl -n default get deploy shard &>/dev/null ; then
                echo "Example shard is still running, refusing to run a load test experiment."
                echo "Ensure a clean load test environment, i.e., run 'make down SKAFFOLD_MODULE=shard'."
                exit 1
              fi