feat(ws): initial Workspace and WorkspaceKind controller loops (#22)

* feat(ws): implement a reconciliation loop for the workspace Signed-off-by: Adem Baccara <[email protected]> * remove comments Signed-off-by: Adem Baccara <[email protected]> * add correct rbac permission for controller Signed-off-by: Adem Baccara <[email protected]> * implemented collision handling using ownerReferences Signed-off-by: Adem Baccara <[email protected]> * update the status field during workspace reconciliation Signed-off-by: Adem Baccara <[email protected]> * add watcher to workspace kind Signed-off-by: Adem Baccara <[email protected]> * handle the case that multiple ports are specified for an image Signed-off-by: Adem Baccara <[email protected]> * generate correctly the StatefulSet spec Signed-off-by: Adem Baccara <[email protected]> * set status.state of the Workspace Signed-off-by: Adem Baccara <[email protected]> * add rbac permission for configmap Signed-off-by: Adem Baccara <[email protected]> * update dockerfile Signed-off-by: Adem Baccara <[email protected]> * mathew updates Signed-off-by: Mathew Wicks <[email protected]> * mathew updates 2 Signed-off-by: Mathew Wicks <[email protected]> * mathew updates 3 Signed-off-by: Mathew Wicks <[email protected]> * fix todos Signed-off-by: Adem Baccara <[email protected]> * mathew updates 4 Signed-off-by: Mathew Wicks <[email protected]> * handle extraEnv value replacement Signed-off-by: Adem Baccara <[email protected]> * mathew updates 5 Signed-off-by: Mathew Wicks <[email protected]> * mathew updates 6 Signed-off-by: Mathew Wicks <[email protected]> --------- Signed-off-by: Adem Baccara <[email protected]> Signed-off-by: Mathew Wicks <[email protected]> Co-authored-by: Mathew Wicks <[email protected]>
kubeflow · Jul 15, 2024 · 0cacff7 · 0cacff7
1 parent ca78327
commit 0cacff7
Show file tree

Hide file tree

Showing 15 changed files with 4,715 additions and 516 deletions.
diff --git a/workspaces/controller/Dockerfile b/workspaces/controller/Dockerfile
@@ -14,7 +14,7 @@ RUN go mod download
 # Copy the go source
 COPY cmd/main.go cmd/main.go
 COPY api/ api/
-COPY internal/controller/ internal/controller/
+COPY internal/ internal/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

diff --git a/workspaces/controller/api/v1beta1/workspace_types.go b/workspaces/controller/api/v1beta1/workspace_types.go
@@ -36,6 +36,12 @@ type WorkspaceSpec struct {
 	//+kubebuilder:default=false
 	Paused *bool `json:"paused,omitempty"`
 
+	// if true, pending updates are NOT applied when the Workspace is paused
+	// if false, pending updates are applied when the Workspace is paused
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=false
+	DeferUpdates *bool `json:"deferUpdates,omitempty"`
+
 	// the WorkspaceKind to use
 	//+kubebuilder:validation:MinLength:=2
 	//+kubebuilder:validation:MaxLength:=63
@@ -76,18 +82,21 @@ type WorkspacePodVolumes struct {
 	//  - this PVC must be RWX (ReadWriteMany, ReadWriteOnce)
 	//  - the mount path is defined in the WorkspaceKind under
 	//    `spec.podTemplate.volumeMounts.home`
+	//+kubebuilder:validation:Optional
 	//+kubebuilder:validation:MinLength:=2
 	//+kubebuilder:validation:MaxLength:=63
 	//+kubebuilder:validation:Pattern:=^[a-z0-9][-a-z0-9]*[a-z0-9]$
 	//+kubebuilder:example="my-home-pvc"
-	Home string `json:"home"`
+	Home *string `json:"home,omitempty"`
 
 	// additional PVCs to mount
-	//  - these PVCs must already exist in the Namespace
-	//  - these PVCs must be RWX (ReadWriteMany, ReadWriteOnce)
+	//  - these PVC must already exist in the Namespace
+	//  - the same PVC can be mounted multiple times with different `mountPaths`
+	//  - if `readOnly` is false, the PVC must be RWX (ReadWriteMany, ReadWriteOnce)
+	//  - if `readOnly` is true, the PVC must be ReadOnlyMany
 	//+kubebuilder:validation:Optional
 	//+listType:="map"
-	//+listMapKey:="name"
+	//+listMapKey:="mountPath"
 	Data []PodVolumeMount `json:"data,omitempty"`
 }
 
@@ -97,26 +106,35 @@ type PodVolumeMount struct {
 	//+kubebuilder:validation:MaxLength:=63
 	//+kubebuilder:validation:Pattern:=^[a-z0-9][-a-z0-9]*[a-z0-9]$
 	//+kubebuilder:example="my-data-pvc"
-	Name string `json:"name"`
+	PVCName string `json:"pvcName"`
 
 	// the mount path for the PVC
 	//+kubebuilder:validation:MinLength:=2
 	//+kubebuilder:validation:MaxLength:=4096
 	//+kubebuilder:validation:Pattern:=^/[^/].*$
 	//+kubebuilder:example="/data/my-data"
 	MountPath string `json:"mountPath"`
+
+	// if the PVC should be mounted as ReadOnly
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:default=false
+	ReadOnly *bool `json:"readOnly,omitempty"`
 }
 
 type WorkspacePodOptions struct {
 	// the id of an imageConfig option
 	//  - options are defined in WorkspaceKind under
 	//    `spec.podTemplate.options.imageConfig.values[]`
-	//+kubebuilder:example="jupyter_scipy_170"
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	//+kubebuilder:example="jupyterlab_scipy_190"
 	ImageConfig string `json:"imageConfig"`
 
 	// the id of a podConfig option
 	//  - options are defined in WorkspaceKind under
 	//    `spec.podTemplate.options.podConfig.values[]`
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
 	//+kubebuilder:example="big_gpu"
 	PodConfig string `json:"podConfig"`
 }
@@ -129,11 +147,12 @@ type WorkspacePodOptions struct {
 
 // WorkspaceStatus defines the observed state of Workspace
 type WorkspaceStatus struct {
-
 	// activity information for the Workspace, used to determine when to cull
 	Activity WorkspaceActivity `json:"activity"`
 
-	// the time when the Workspace was paused, 0 if the Workspace is not paused
+	// the time when the Workspace was paused (UNIX epoch)
+	//  - set to 0 when the Workspace is NOT paused
+	//+kubebuilder:default=0
 	//+kubebuilder:example=1704067200
 	PauseTime int64 `json:"pauseTime"`
 
@@ -142,32 +161,66 @@ type WorkspaceStatus struct {
 	//    and so will be patched on the next restart
 	//  - true if the WorkspaceKind has changed one of its common `podTemplate` fields
 	//    like `podMetadata`, `probes`, `extraEnv`, or `containerSecurityContext`
-	//+kubebuilder:example=false
+	//+kubebuilder:default=false
 	PendingRestart bool `json:"pendingRestart"`
 
-	// the `spec.podTemplate.options` which will take effect after the next restart
-	PodTemplateOptions WorkspacePodOptions `json:"podTemplateOptions"`
+	// information about the current podTemplate options
+	PodTemplateOptions WorkspacePodOptionsStatus `json:"podTemplateOptions"`
 
 	// the current state of the Workspace
-	//+kubebuilder:example="Running"
+	//+kubebuilder:default="Unknown"
 	State WorkspaceState `json:"state"`
 
 	// a human-readable message about the state of the Workspace
 	//  - WARNING: this field is NOT FOR MACHINE USE, subject to change without notice
-	//+kubebuilder:example="Pod is not ready"
+	//+kubebuilder:default=""
 	StateMessage string `json:"stateMessage"`
 }
 
 type WorkspaceActivity struct {
 	// the last time activity was observed on the Workspace (UNIX epoch)
+	//+kubebuilder:default=0
 	//+kubebuilder:example=1704067200
 	LastActivity int64 `json:"lastActivity"`
 
 	// the last time we checked for activity on the Workspace (UNIX epoch)
+	//+kubebuilder:default=0
 	//+kubebuilder:example=1704067200
 	LastUpdate int64 `json:"lastUpdate"`
 }
 
+type WorkspacePodOptionsStatus struct {
+	// info about the current imageConfig option
+	ImageConfig WorkspacePodOptionInfo `json:"imageConfig"`
+
+	// info about the current podConfig option
+	PodConfig WorkspacePodOptionInfo `json:"podConfig"`
+}
+
+type WorkspacePodOptionInfo struct {
+	// the option id which will take effect after the next restart
+	//+kubebuilder:validation:Optional
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	Desired string `json:"desired,omitempty"`
+
+	// the chain from the current option to the desired option
+	//+kubebuilder:validation:Optional
+	RedirectChain []WorkspacePodOptionRedirectStep `json:"redirectChain,omitempty"`
+}
+
+type WorkspacePodOptionRedirectStep struct {
+	// the source option id
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	Source string `json:"source"`
+
+	// the target option id
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	Target string `json:"target"`
+}
+
 // +kubebuilder:validation:Enum:={"Running","Terminating","Paused","Pending","Error","Unknown"}
 type WorkspaceState string
 

diff --git a/workspaces/controller/api/v1beta1/workspacekind_types.go b/workspaces/controller/api/v1beta1/workspacekind_types.go
@@ -98,7 +98,6 @@ type WorkspaceKindConfigMap struct {
 
 type WorkspaceKindPodTemplate struct {
 	// metadata for Workspace Pods (MUTABLE)
-	//  - changes are applied the NEXT time each Workspace is PAUSED
 	//+kubebuilder:validation:Optional
 	PodMetadata *WorkspaceKindPodMetadata `json:"podMetadata,omitempty"`
 
@@ -110,7 +109,6 @@ type WorkspaceKindPodTemplate struct {
 	Culling *WorkspaceKindCullingConfig `json:"culling,omitempty"`
 
 	// standard probes to determine Container health (MUTABLE)
-	//  - changes are applied the NEXT time each Workspace is PAUSED
 	//+kubebuilder:validation:Optional
 	Probes *WorkspaceKindProbes `json:"probes,omitempty"`
 
@@ -122,16 +120,30 @@ type WorkspaceKindPodTemplate struct {
 	HTTPProxy *HTTPProxy `json:"httpProxy,omitempty"`
 
 	// environment variables for Workspace Pods (MUTABLE)
-	//  - changes are applied the NEXT time each Workspace is PAUSED
-	//  - the following string templates are available:
-	//     - `.PathPrefix`: the path prefix of the Workspace (e.g. '/workspace/{profile_name}/{workspace_name}/')
+	//  - the following go template functions are available:
+	//     - `httpPathPrefix(portId string)`: returns the HTTP path prefix of the specified port
 	//+kubebuilder:validation:Optional
 	//+listType:="map"
 	//+listMapKey:="name"
 	ExtraEnv []v1.EnvVar `json:"extraEnv,omitempty"`
 
-	// container SecurityContext for Workspace Pods (MUTABLE)
-	//  - changes are applied the NEXT time each Workspace is PAUSED
+	// extra volume mounts for Workspace Pods (MUTABLE)
+	//+kubebuilder:validation:Optional
+	//+listType:="map"
+	//+listMapKey:="mountPath"
+	ExtraVolumeMounts []v1.VolumeMount `json:"extraVolumeMounts,omitempty"`
+
+	// extra volumes for Workspace Pods (MUTABLE)
+	//+kubebuilder:validation:Optional
+	//+listType:="map"
+	//+listMapKey:="name"
+	ExtraVolumes []v1.Volume `json:"extraVolumes,omitempty"`
+
+	// security context for Workspace Pods (MUTABLE)
+	//+kubebuilder:validation:Optional
+	SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"`
+
+	// container security context for Workspace Pods (MUTABLE)
 	//+kubebuilder:validation:Optional
 	ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"`
 
@@ -170,7 +182,7 @@ type WorkspaceKindCullingConfig struct {
 	//+kubebuilder:validation:Optional
 	//+kubebuilder:validation:Minimum:=60
 	//+kubebuilder:default=86400
-	MaxInactiveSeconds *int64 `json:"maxInactiveSeconds,omitempty"`
+	MaxInactiveSeconds *int32 `json:"maxInactiveSeconds,omitempty"`
 
 	// the probe used to determine if the Workspace is active
 	ActivityProbe ActivityProbe `json:"activityProbe"`
@@ -275,9 +287,8 @@ type WorkspaceKindPodOptions struct {
 }
 
 type ImageConfig struct {
-	// the id of the default image config
-	//+kubebuilder:example:="jupyter_scipy_171"
-	Default string `json:"default"`
+	// spawner ui configs
+	Spawner OptionsSpawnerConfig `json:"spawner"`
 
 	// the list of image configs that are available
 	//+kubebuilder:validation:MinItems:=1
@@ -288,7 +299,9 @@ type ImageConfig struct {
 
 type ImageConfigValue struct {
 	// the id of this image config
-	//+kubebuilder:example:="jupyter_scipy_171"
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	//+kubebuilder:example:="jupyterlab_scipy_190"
 	Id string `json:"id"`
 
 	// information for the spawner ui
@@ -319,22 +332,32 @@ type ImageConfigSpec struct {
 	//   - if multiple ports are defined, the user will see multiple "Connect" buttons
 	//     in a dropdown menu on the Workspace overview page
 	//+kubebuilder:validation:MinItems:=1
+	//+listType:="map"
+	//+listMapKey:="id"
 	Ports []ImagePort `json:"ports"`
 }
 
 type ImagePort struct {
-	// the display name of the port
-	//+kubebuilder:validation:MinLength:=2
-	//+kubebuilder:validation:MaxLength:=64
-	//+kubebuilder:example:="JupyterLab"
-	DisplayName string `json:"displayName"`
+	// the id of the port
+	//  - this is NOT used as the Container or Service port name, but as part of the HTTP path
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=32
+	//+kubebuilder:validation:Pattern:=^[a-z0-9][a-z0-9_-]*[a-z0-9]$
+	//+kubebuilder:example="jupyterlab"
+	Id string `json:"id"`
 
 	// the port number
 	//+kubebuilder:validation:Minimum:=1
 	//+kubebuilder:validation:Maximum:=65535
 	//+kubebuilder:example:=8888
 	Port int32 `json:"port"`
 
+	// the display name of the port
+	//+kubebuilder:validation:MinLength:=2
+	//+kubebuilder:validation:MaxLength:=64
+	//+kubebuilder:example:="JupyterLab"
+	DisplayName string `json:"displayName"`
+
 	// the protocol of the port
 	//+kubebuilder:example:="HTTP"
 	Protocol ImagePortProtocol `json:"protocol"`
@@ -348,9 +371,8 @@ const (
 )
 
 type PodConfig struct {
-	// the id of the default pod config
-	//+kubebuilder:example="big_gpu"
-	Default string `json:"default"`
+	// spawner ui configs
+	Spawner OptionsSpawnerConfig `json:"spawner"`
 
 	// the list of pod configs that are available
 	//+kubebuilder:validation:MinItems:=1
@@ -361,6 +383,8 @@ type PodConfig struct {
 
 type PodConfigValue struct {
 	// the id of this pod config
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
 	//+kubebuilder:example="big_gpu"
 	Id string `json:"id"`
 
@@ -394,6 +418,15 @@ type PodConfigSpec struct {
 	Resources *v1.ResourceRequirements `json:"resources,omitempty"`
 }
 
+type OptionsSpawnerConfig struct {
+	// the id of the default option
+	//  - this will be selected by default in the spawner ui
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	//+kubebuilder:example="jupyterlab_scipy_190"
+	Default string `json:"default"`
+}
+
 type OptionSpawnerInfo struct {
 	// the display name of the option
 	//+kubebuilder:validation:MinLength:=2
@@ -426,20 +459,18 @@ type OptionSpawnerLabel struct {
 	Key string `json:"key"`
 
 	// the value of the label
-	//+kubebuilder:validation:MinLength:=2
+	//+kubebuilder:validation:MinLength:=1
 	//+kubebuilder:validation:MaxLength:=64
 	Value string `json:"value"`
 }
 
 type OptionRedirect struct {
 	// the id of the option to redirect to
-	//+kubebuilder:example:="jupyter_scipy_171"
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
+	//+kubebuilder:example:="jupyterlab_scipy_190"
 	To string `json:"to"`
 
-	// if the redirect will be applied after the next restart of the Workspace
-	//+kubebuilder:example:=true
-	WaitForRestart bool `json:"waitForRestart"`
-
 	// information about the redirect
 	//+kubebuilder:validation:Optional
 	Message *RedirectMessage `json:"message,omitempty"`
@@ -476,8 +507,8 @@ const (
 type WorkspaceKindStatus struct {
 
 	// the number of Workspaces that are using this WorkspaceKind
-	//+kubebuilder:example=3
-	Workspaces int64 `json:"workspaces"`
+	//+kubebuilder:default=0
+	Workspaces int32 `json:"workspaces"`
 
 	// metrics for podTemplate options
 	PodTemplateOptions PodTemplateOptionsMetrics `json:"podTemplateOptions"`
@@ -497,12 +528,14 @@ type PodTemplateOptionsMetrics struct {
 
 type OptionMetric struct {
 	// the id of the option
+	//+kubebuilder:validation:MinLength:=1
+	//+kubebuilder:validation:MaxLength:=256
 	//+kubebuilder:example="big_gpu"
 	Id string `json:"id"`
 
 	// the number of Workspaces currently using the option
 	//+kubebuilder:example=3
-	Workspaces int64 `json:"workspaces"`
+	Workspaces int32 `json:"workspaces"`
 }
 
 /*