From 73e67894ca325f3b1a6b1225ba6f25ea50e1b5a2 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 11 Jan 2024 22:51:05 +0100 Subject: [PATCH 01/83] Splitter CRD Add new features enum and field for operator The old enum cannot be extended as it would break old clients. So we define a new extensible enum and a new field for it. We make all of those fields private so that users of that struct only access them via a getter that handles objects deserialized from either old or new versions. use copy target when splitting --- mirrord/cli/src/error.rs | 2 +- mirrord/cli/src/operator/session.rs | 101 ++++--- mirrord/config/src/feature.rs | 10 + mirrord/config/src/feature/split_queues.rs | 94 ++++++ mirrord/config/src/lib.rs | 1 + mirrord/config/src/target.rs | 10 + mirrord/kube/src/api/kubernetes/rollout.rs | 4 +- mirrord/operator/Cargo.toml | 40 +-- mirrord/operator/src/client.rs | 77 ++--- mirrord/operator/src/crd.rs | 335 ++++++++++++++++++++- mirrord/operator/src/setup.rs | 75 ++++- tests/Cargo.toml | 4 +- tests/configs/sqs_queue_splitting_a.json | 13 + tests/src/operator.rs | 1 + tests/src/operator/queue_splitting.rs | 191 ++++++++++++ tests/src/utils.rs | 92 +++++- 16 files changed, 919 insertions(+), 131 deletions(-) create mode 100644 mirrord/config/src/feature/split_queues.rs create mode 100644 tests/configs/sqs_queue_splitting_a.json create mode 100644 tests/src/operator/queue_splitting.rs diff --git a/mirrord/cli/src/error.rs b/mirrord/cli/src/error.rs index 06a165d8e38..1e0a48322b7 100644 --- a/mirrord/cli/src/error.rs +++ b/mirrord/cli/src/error.rs @@ -333,7 +333,7 @@ impl From for CliError { feature, operator_version, } => Self::FeatureNotSupportedInOperatorError { - feature, + feature: feature.to_string(), operator_version, }, OperatorApiError::CreateApiError(e) => Self::KubernetesApiFailed(e), diff --git a/mirrord/cli/src/operator/session.rs b/mirrord/cli/src/operator/session.rs index 02cb12227f2..90c46313251 100644 --- a/mirrord/cli/src/operator/session.rs +++ b/mirrord/cli/src/operator/session.rs @@ -3,6 +3,7 @@ use mirrord_operator::{ client::{session_api, OperatorApiError, OperatorOperation}, crd::{MirrordOperatorCrd, SessionCrd, OPERATOR_STATUS_NAME}, }; +use mirrord_operator::crd::NewOperatorFeature; use mirrord_progress::{Progress, ProgressTracker}; use super::get_status_api; @@ -90,58 +91,58 @@ impl SessionCommandHandler { .await .map(|either| either.right()), } - .map_err(|kube_fail| match kube_fail { - // The random `reason` we get when the operator returns from a "missing route". - kube::Error::Api(ErrorResponse { code, reason, .. }) + .map_err(|kube_fail| match kube_fail { + // The random `reason` we get when the operator returns from a "missing route". + kube::Error::Api(ErrorResponse { code, reason, .. }) if code == 404 && reason.contains("parse") => - { - OperatorApiError::UnsupportedFeature { - feature: "session management".to_string(), - operator_version, + { + OperatorApiError::UnsupportedFeature { + feature: NewOperatorFeature::SessionManagement, + operator_version, + } + } + // Something actually went wrong. + other => OperatorApiError::KubeError { + error: other, + operation: OperatorOperation::SessionManagement, + }, + }) + // Finish the progress report here if we have an error response. + .inspect_err(|fail| { + sub_progress.failure(Some(&fail.to_string())); + progress.failure(Some("Session management operation failed!")); + })? + // The kube api interaction was successful, but we might still fail the operation + // itself, so let's check the `Status` and report. + .map(|status| { + if status.is_failure() { + sub_progress.failure(Some(&format!( + "`{command}` failed due to `{}` with code `{}`!", + status.message, status.code + ))); + progress.failure(Some("Session operation failed!")); + + Err(OperatorApiError::StatusFailure { + operation: command.to_string(), + status: Box::new(status), + }) + } else { + sub_progress.success(Some(&format!( + "`{command}` finished successfully with `{}` with code `{}`.", + status.message, status.code + ))); + progress.success(Some("Session operation is completed.")); + + Ok(()) } - } - // Something actually went wrong. - other => OperatorApiError::KubeError { - error: other, - operation: OperatorOperation::SessionManagement, - }, - }) - // Finish the progress report here if we have an error response. - .inspect_err(|fail| { - sub_progress.failure(Some(&fail.to_string())); - progress.failure(Some("Session management operation failed!")); - })? - // The kube api interaction was successful, but we might still fail the operation - // itself, so let's check the `Status` and report. - .map(|status| { - if status.is_failure() { - sub_progress.failure(Some(&format!( - "`{command}` failed due to `{}` with code `{}`!", - status.message, status.code - ))); - progress.failure(Some("Session operation failed!")); - - Err(OperatorApiError::StatusFailure { - operation: command.to_string(), - status: Box::new(status), - }) - } else { - sub_progress.success(Some(&format!( - "`{command}` finished successfully with `{}` with code `{}`.", - status.message, status.code - ))); - progress.success(Some("Session operation is completed.")); - - Ok(()) - } - }) - .transpose()? - // We might've gotten a `SessionCrd` instead of a `Status` (we have a `Left(T)`), - // meaning that the operation has started, but it might not be finished yet. - .unwrap_or_else(|| { - sub_progress.success(Some(&format!("No issues found when executing `{command}`, but the operation status could not be determined at this time."))); - progress.success(Some(&format!("`{command}` is done, but the operation might be pending."))); - }); + }) + .transpose()? + // We might've gotten a `SessionCrd` instead of a `Status` (we have a `Left(T)`), + // meaning that the operation has started, but it might not be finished yet. + .unwrap_or_else(|| { + sub_progress.success(Some(&format!("No issues found when executing `{command}`, but the operation status could not be determined at this time."))); + progress.success(Some(&format!("`{command}` is done, but the operation might be pending."))); + }); Ok(()) } diff --git a/mirrord/config/src/feature.rs b/mirrord/config/src/feature.rs index 766719b44e2..f16e6c29758 100644 --- a/mirrord/config/src/feature.rs +++ b/mirrord/config/src/feature.rs @@ -4,11 +4,13 @@ use schemars::JsonSchema; use self::{copy_target::CopyTargetConfig, env::EnvConfig, fs::FsConfig, network::NetworkConfig}; use crate::config::source::MirrordConfigSource; +use crate::feature::split_queues::SplitQueuesConfig; pub mod copy_target; pub mod env; pub mod fs; pub mod network; +pub mod split_queues; /// Controls mirrord features. /// @@ -95,6 +97,13 @@ pub struct FeatureConfig { /// Should mirrord return the hostname of the target pod when calling `gethostname` #[config(default = true)] pub hostname: bool, + + /// ## feature.split_queues {#feature-split_queues} + /// + /// Define filters to split queues by, and make your local application consume only messages + /// that match those filters. + #[config(nested)] + pub split_queues: SplitQueuesConfig, } impl CollectAnalytics for &FeatureConfig { @@ -104,5 +113,6 @@ impl CollectAnalytics for &FeatureConfig { analytics.add("network", &self.network); analytics.add("copy_target", &self.copy_target); analytics.add("hostname", self.hostname); + analytics.add("split_queues", &self.split_queues); } } diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs new file mode 100644 index 00000000000..aa1cd60fb5b --- /dev/null +++ b/mirrord/config/src/feature/split_queues.rs @@ -0,0 +1,94 @@ +use std::collections::HashMap; + +use mirrord_analytics::{Analytics, CollectAnalytics}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::config::{ConfigContext, FromMirrordConfig, MirrordConfig}; + +pub type QueueId = String; + +/// ```json +/// { +/// "feature": { +/// "split_queues": { +/// "first-queue": { +/// "queue_type": "SQS", +/// "message_filter": { +/// "wows": "so wows", +/// "coolz": "^very .*" +/// } +/// }, +/// "second-queue": { +/// "queue_type": "SomeFutureQueueType", +/// "message_filter": { +/// "wows": "so wows", +/// "coolz": "^very .*" +/// } +/// }, +/// } +/// } +/// } +/// ``` +#[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Deserialize, Default)] +pub struct SplitQueuesConfig(Option>); + +impl SplitQueuesConfig { + pub fn is_set(&self) -> bool { + self.0.is_some() + } + + /// Out of the whole queue splitting config, get only the sqs queues. + pub fn get_sqs_filter(&self) -> Option>> { + self.0.as_ref().map(|queue_id2queue_filter| { + queue_id2queue_filter + .iter() + .filter_map(|(queue_id, queue_filter)| match queue_filter { + QueueFilter::Sqs(filter_mapping) => { + Some((queue_id.clone(), filter_mapping.clone())) + } + }) + .collect() + }) + } +} + +impl MirrordConfig for SplitQueuesConfig { + type Generated = Self; + + fn generate_config( + self, + _context: &mut ConfigContext, + ) -> crate::config::Result { + Ok(self) + } +} + +impl FromMirrordConfig for SplitQueuesConfig { + type Generator = Self; +} + +pub type MessageAttributeName = String; +pub type AttributeValuePattern = String; + +pub type SqsMessageFilter = HashMap; + +/// More queue types might be added in the future. +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, JsonSchema)] +#[serde(tag = "queue_type", content = "message_filter")] +pub enum QueueFilter { + #[serde(rename = "SQS")] + Sqs(SqsMessageFilter), +} + +impl CollectAnalytics for &SplitQueuesConfig { + fn collect_analytics(&self, analytics: &mut Analytics) { + analytics.add( + "queue_count", + self.0 + .as_ref() + .map(|mapping| mapping.len()) + .unwrap_or_default(), + ) + } +} diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index e7cf9b9aee0..c18478c4d2a 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -739,6 +739,7 @@ mod tests { })), copy_target: None, hostname: None, + split_queues: None, }), connect_tcp: None, operator: None, diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs index 8ed7129ddb6..438b1de99eb 100644 --- a/mirrord/config/src/target.rs +++ b/mirrord/config/src/target.rs @@ -251,6 +251,16 @@ impl Target { } } } + + /// Get the target type - "pod", "deployment", "rollout" or "targetless" + pub fn get_target_type(&self) -> &str { + match self { + Target::Targetless => "targetless", + Target::Pod(pod) => pod.target_type(), + Target::Deployment(dep) => dep.target_type(), + Target::Rollout(roll) => roll.target_type(), + } + } } trait TargetDisplay { diff --git a/mirrord/kube/src/api/kubernetes/rollout.rs b/mirrord/kube/src/api/kubernetes/rollout.rs index 5e29894756e..34f610fa0d9 100644 --- a/mirrord/kube/src/api/kubernetes/rollout.rs +++ b/mirrord/kube/src/api/kubernetes/rollout.rs @@ -4,9 +4,9 @@ use k8s_openapi::{ apimachinery::pkg::apis::meta::v1::ObjectMeta, ListableResource, Metadata, NamespaceResourceScope, Resource, }; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct Rollout { metadata: ObjectMeta, pub spec: serde_json::Value, diff --git a/mirrord/operator/Cargo.toml b/mirrord/operator/Cargo.toml index f6fd689e9ae..0b873cb8bb1 100644 --- a/mirrord/operator/Cargo.toml +++ b/mirrord/operator/Cargo.toml @@ -14,29 +14,29 @@ publish.workspace = true edition.workspace = true [features] -default = [] +default = ["crd"] license-fetch = ["dep:reqwest"] client = [ - "crd", - "dep:base64", - "dep:bincode", - "dep:http", - "dep:futures", - "dep:mirrord-analytics", - "dep:mirrord-auth", - "dep:mirrord-kube", - "dep:mirrord-progress", - "dep:mirrord-protocol", - "dep:rand", - "dep:tokio-tungstenite", - "dep:tracing", + "crd", + "dep:base64", + "dep:bincode", + "dep:http", + "dep:futures", + "dep:mirrord-analytics", + "dep:mirrord-auth", + "dep:mirrord-kube", + "dep:mirrord-progress", + "dep:mirrord-protocol", + "dep:rand", + "dep:tokio-tungstenite", + "dep:tracing", ] crd = [ - "dep:k8s-openapi", - "dep:kube", - "dep:mirrord-config", - "dep:tokio", - "dep:serde_json" + "dep:k8s-openapi", + "dep:kube", + "dep:mirrord-config", + "dep:tokio", + "dep:serde_json" ] setup = ["crd", "dep:serde_yaml"] @@ -52,7 +52,7 @@ mirrord-protocol = { path = "../protocol", optional = true } async-trait = "0.1" actix-codec = { workspace = true, optional = true } base64 = { version = "0.21", optional = true } -bincode = { version = "2.0.0-rc.2", features = ["serde"], optional = true } +bincode = { version = "2.0.0-rc.2", features = ["serde"], optional = true } bytes = { workspace = true, optional = true } chrono = { version = "0.4", features = ["clock", "serde"] } http = { version = "0.2", optional = true } diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 677b1731333..4452bb10d3b 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, fmt::{self, Display}, io, }; @@ -34,7 +35,7 @@ use tokio_tungstenite::tungstenite::{Error as TungsteniteError, Message}; use tracing::{debug, error, info, warn}; use crate::crd::{ - CopyTargetCrd, CopyTargetSpec, MirrordOperatorCrd, OperatorFeatures, SessionCrd, TargetCrd, + CopyTargetCrd, CopyTargetSpec, MirrordOperatorCrd, NewOperatorFeature, SessionCrd, TargetCrd, OPERATOR_STATUS_NAME, }; @@ -90,13 +91,13 @@ pub enum OperatorApiError { #[error("mirrord operator {operator_version} does not support feature {feature}")] UnsupportedFeature { - feature: String, + feature: NewOperatorFeature, operator_version: String, }, #[error( - "Tried executing {operation}, but operator returned with `{}` and code `{}``!", - status.reason, status.code + "Tried executing {operation}, but operator returned with `{}` and code `{}``!", + status.reason, status.code )] StatusFailure { operation: String, @@ -114,18 +115,16 @@ pub struct OperatorSessionMetadata { client_certificate: Option, session_id: u64, fingerprint: Option, - operator_features: Vec, + operator_features: Vec, protocol_version: Option, - copy_pod_enabled: Option, } impl OperatorSessionMetadata { fn new( client_certificate: Option, fingerprint: Option, - operator_features: Vec, + operator_features: Vec, protocol_version: Option, - copy_pod_enabled: Option, ) -> Self { Self { client_certificate, @@ -133,7 +132,6 @@ impl OperatorSessionMetadata { fingerprint, operator_features, protocol_version, - copy_pod_enabled, } } @@ -162,7 +160,8 @@ impl OperatorSessionMetadata { } fn proxy_feature_enabled(&self) -> bool { - self.operator_features.contains(&OperatorFeatures::ProxyApi) + self.operator_features + .contains(&NewOperatorFeature::ProxyApi) } } @@ -212,12 +211,10 @@ impl OperatorApi { /// Checks used config against operator specification. fn check_config(config: &LayerConfig, operator: &MirrordOperatorCrd) -> Result<()> { - if config.feature.copy_target.enabled && !operator.spec.copy_target_enabled.unwrap_or(false) - { - return Err(OperatorApiError::UnsupportedFeature { - feature: "copy target".into(), - operator_version: operator.spec.operator_version.clone(), - }); + if config.feature.copy_target.enabled { + operator + .spec + .require_feature(NewOperatorFeature::CopyTarget)?; } Ok(()) @@ -252,8 +249,8 @@ impl OperatorApi { progress: &P, analytics: &mut R, ) -> Result - where - P: Progress + Send + Sync, + where + P: Progress + Send + Sync, { let operator_api = OperatorApi::new(config).await?; @@ -274,7 +271,7 @@ impl OperatorApi { }) .unwrap_or_else(|| "today".to_string()); - let expiring_message = format!("Operator license will expire {expiring_soon}!",); + let expiring_message = format!("Operator license will expire {expiring_soon}!", ); progress.warning(&expiring_message); warn!(expiring_message); @@ -300,15 +297,15 @@ impl OperatorApi { .await .ok() .flatten(); + let features = operator.spec.supported_features(); let metadata = OperatorSessionMetadata::new( client_certificate, operator.spec.license.fingerprint, - operator.spec.features.unwrap_or_default(), + features, operator .spec .protocol_version .and_then(|str_version| str_version.parse().ok()), - operator.spec.copy_target_enabled, ); metadata.set_operator_properties(analytics); @@ -321,10 +318,10 @@ impl OperatorApi { if operator_version > mirrord_version { // we make two sub tasks since it looks best this way version_progress.warning( - &format!( - "Your mirrord plugin/CLI version {} does not match the operator version {}. This can lead to unforeseen issues.", - mirrord_version, - operator_version)); + &format!( + "Your mirrord plugin/CLI version {} does not match the operator version {}. This can lead to unforeseen issues.", + mirrord_version, + operator_version)); version_progress.success(None); version_progress = progress.subtask("comparing versions"); version_progress.warning( @@ -333,13 +330,17 @@ impl OperatorApi { } version_progress.success(None); - let target_to_connect = if config.feature.copy_target.enabled { + let target_to_connect = if config.feature.copy_target.enabled + // use copy_target for splitting queues + || config.feature.split_queues.is_set() + { let mut copy_progress = progress.subtask("copying target"); let copied = operator_api .copy_target( &metadata, config.target.path.clone().unwrap_or(Target::Targetless), config.feature.copy_target.scale_down, + config.feature.split_queues.get_sqs_filter(), ) .await?; copy_progress.success(None); @@ -383,8 +384,8 @@ impl OperatorApi { config.kubeconfig.clone(), config.kube_context.clone(), ) - .await - .map_err(OperatorApiError::CreateApiError)?; + .await + .map_err(OperatorApiError::CreateApiError)?; let target_namespace = if target_config.path.is_some() { target_config.namespace.clone() @@ -500,9 +501,9 @@ impl OperatorApi { .target_api .get_subresource("port-locks", &target.name()) .await - else { - return Ok(()); - }; + else { + return Ok(()); + }; let no_port_locks = lock_target .spec @@ -593,6 +594,7 @@ impl OperatorApi { session_metadata: &OperatorSessionMetadata, target: Target, scale_down: bool, + sqs_filter: Option>>, ) -> Result { let name = TargetCrd::target_name(&target); @@ -602,6 +604,7 @@ impl OperatorApi { target, idle_ttl: Some(Self::COPIED_POD_IDLE_TTL), scale_down, + sqs_filter, }, ); @@ -637,12 +640,12 @@ pub struct ConnectionWrapper { } impl ConnectionWrapper -where - for<'stream> T: StreamExt> - + SinkExt - + Send - + Unpin - + 'stream, + where + for<'stream> T: StreamExt> + + SinkExt + + Send + + Unpin + + 'stream, { fn wrap( connection: T, diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 6e731e15891..b7d39995121 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -1,7 +1,19 @@ +use std::{ + collections::HashMap, + fmt::{Display, Formatter}, +}; + +use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; use kube::CustomResource; -use mirrord_config::target::{Target, TargetConfig}; +pub use mirrord_config::feature::split_queues::QueueId; +use mirrord_config::{ + feature::split_queues::SqsMessageFilter, + target::{Target, TargetConfig}, +}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +#[cfg(feature = "client")] +use crate::client::OperatorApiError; use self::label_selector::LabelSelector; use crate::types::LicenseInfoOwned; @@ -89,10 +101,100 @@ pub static OPERATOR_STATUS_NAME: &str = "operator"; pub struct MirrordOperatorSpec { pub operator_version: String, pub default_namespace: String, - pub features: Option>, + /// Should be removed when we can stop supporting compatibility with versions from before the + /// `supported_features` field was added. + /// "Breaking" that compatibility by removing this field and then running with one old (from + /// before the `supported_features` field) side (client or operator) would make the client + /// think `ProxyApi` is not supported even if it is. + #[deprecated(note = "use supported_features instead")] + features: Option>, + /// Replaces both `features` and `copy_target_enabled`. Operator versions that use a version + /// of this code that has both this and the old fields are expected to populate this field with + /// the full set of features they support, and the old fields with their limited info they + /// support, for old clients. + /// + /// Access this info only via `supported_features()`. + /// Optional for backwards compatibility (new clients can talk to old operators that don't send + /// this field). + supported_features: Option>, pub license: LicenseInfoOwned, pub protocol_version: Option, - pub copy_target_enabled: Option, + /// Should be removed when we can stop supporting compatibility with versions from before the + /// `supported_features` field was added. + /// "Breaking" that compatibility by removing this field and then running with one old (from + /// before the `supported_features` field) side (client or operator) would make the client + /// think copy target is not enabled even if it is. + /// Optional for backwards compatibility (new clients can talk to old operators that don't send + /// this field). + #[deprecated(note = "use supported_features instead")] + copy_target_enabled: Option, +} + +impl MirrordOperatorSpec { + pub fn new( + operator_version: String, + default_namespace: String, + supported_features: Vec, + license: LicenseInfoOwned, + protocol_version: Option, + ) -> Self { + let features = supported_features + .contains(&NewOperatorFeature::ProxyApi) + .then(|| vec![OperatorFeatures::ProxyApi]); + let copy_target_enabled = + Some(supported_features.contains(&NewOperatorFeature::CopyTarget)); + #[allow(deprecated)] // deprecated objects must still be included in construction. + Self { + operator_version, + default_namespace, + supported_features: Some(supported_features), + license, + protocol_version, + features, + copy_target_enabled, + } + } + + /// Get a vector with the features the operator supports. + /// Handles objects sent from old and new operators. + // When the deprecated fields are removed, this can be changed to just return + // `self.supported_features.unwrap_or_default()`. + pub fn supported_features(&self) -> Vec { + self.supported_features + .clone() + // if supported_features was sent, just use that. If not we are dealing with an older + // operator, so we build a vector of new features from the old fields. + .or_else(|| { + // object was sent by an old operator that still uses fields that are now deprecated + #[allow(deprecated)] + self.features.as_ref().map(|features| { + features + .iter() + .map(From::from) + .chain( + self.copy_target_enabled.and_then(|enabled| { + enabled.then_some(NewOperatorFeature::CopyTarget) + }), + ) + .collect() + }) + }) + // Convert `None` to empty vector since we don't expect this to often be + // `None` (although it's ok if it is) and that way the return type is simpler. + .unwrap_or_default() + } + + #[cfg(feature = "client")] + pub fn require_feature(&self, feature: NewOperatorFeature) -> Result<(), OperatorApiError> { + if self.supported_features().contains(&feature) { + Ok(()) + } else { + Err(OperatorApiError::UnsupportedFeature { + feature, + operator_version: self.operator_version.clone(), + }) + } + } } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] @@ -140,9 +242,48 @@ pub struct Session { )] pub struct SessionSpec; +/// Since this enum does not have a variant marked with `#[serde(other)]`, and is present like that +/// in released clients, existing clients would fail to parse any new variant. This means the +/// operator can never send anything but the one existing variant, otherwise the client will error +/// out. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] pub enum OperatorFeatures { ProxyApi, + // DON'T ADD VARIANTS - old clients won't be able to deserialize them. + // Add new features in NewOperatorFeature +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +pub enum NewOperatorFeature { + ProxyApi, + CopyTarget, + Sqs, + SessionManagement, + /// This variant is what a client sees when the operator includes a feature the client is not + /// yet aware of, because it was introduced in a version newer than the client's. + #[serde(other)] + FeatureFromTheFuture, +} + +impl Display for NewOperatorFeature { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let name = match self { + NewOperatorFeature::ProxyApi => "proxy API", + NewOperatorFeature::CopyTarget => "copy target", + NewOperatorFeature::Sqs => "SQS queue splitting", + NewOperatorFeature::FeatureFromTheFuture => "unknown feature", + NewOperatorFeature::SessionManagement => "session management", + }; + f.write_str(name) + } +} + +impl From<&OperatorFeatures> for NewOperatorFeature { + fn from(old_feature: &OperatorFeatures) -> Self { + match old_feature { + OperatorFeatures::ProxyApi => NewOperatorFeature::ProxyApi, + } + } } /// This [`Resource`](kube::Resource) represents a copy pod created from an existing [`Target`] @@ -164,6 +305,8 @@ pub struct CopyTargetSpec { /// Should the operator scale down target deployment to 0 while this pod is alive. /// Ignored if [`Target`] is not [`Target::Deployment`]. pub scale_down: bool, + /// queue id -> (attribute name -> regex) + pub sqs_filter: Option>, } /// Features and operations that can be blocked by a `MirrordPolicy`. @@ -202,3 +345,189 @@ pub struct MirrordPolicySpec { /// List of features and operations blocked by this policy. pub block: Vec, } + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] // queue_name_key -> queueNameKey in yaml. +pub struct ConfigMapQueueNameSource { + /// The name of the config map that holds the name of the queue we want to split. + pub name: String, + + /// The name of the key in the config map that holds the name of the queue we want to + /// split. + pub queue_name_key: String, +} + +/// Set where the application reads the name of the queue from, so that mirrord can find that queue, +/// split it, and temporarily change the name there to the name of the branch queue when splitting. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] // ConfigMap -> configMap in yaml. +pub enum QueueNameSource { + ConfigMap(ConfigMapQueueNameSource), + EnvVar(String), +} + +pub type OutputQueueName = String; + +/// The details of a queue that should be split. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(tag = "queueType")] +// So that controllers that only handle 1 type of queue don't have to adapt when we add more queue types. +#[non_exhaustive] +pub enum SplitQueue { + /// Amazon SQS + /// + /// Where the application gets the queue name from. Will be used to read messages from that + /// queue and distribute them to the output queues. When running with mirrord and splitting + /// this queue, applications will get a modified name from that source. + #[serde(rename = "SQS")] + Sqs(QueueNameSource), +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] // Deployment -> deployment in yaml. +pub enum QueueConsumer { + Deployment(String), + Rollout(String), +} + +impl QueueConsumer { + pub fn get_type_and_name(&self) -> (&str, &str) { + match self { + QueueConsumer::Deployment(dep) => ("deployment", dep), + QueueConsumer::Rollout(roll) => ("rollout", roll) + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +pub struct Filtering { + pub original_spec: PodTemplateSpec, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +pub struct QueueNameUpdate { + pub original_name: String, + pub output_name: String, +} + + +/// Details retrieved from K8s resources once the splitter is active, used on filter session +/// creation to create the altered config maps and pod specs that make the application use the +/// output queues instead of the original. +// This status struct is not optimal in that it contains redundant information. This makes the +// controller's code a bit simpler. +// The information on config map updates and on env vars is present in the resource itself, but it +// is organized differently. +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +pub struct QueueDetails { + /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config map. + pub queue_names: HashMap, + + /// Names of env vars that contain the queue name directly in the pod spec, without config + /// map refs, mapped to their queue id. + pub direct_env_vars: HashMap, + + /// For each config map name, a mapping from queue id to key name in the map that holds the name + /// of that queue. + pub config_map_updates: HashMap>, + // ^ ^ ^ + // | | ---- name of key that points to Q name + // | ---- queue id + // ---- ConfigMap name + + // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with + // `kubectl apply` due to a length limit. + /// The original PodSpec of the queue consumer. + pub original_spec: PodSpec, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] // active_filters -> activeFilters +pub struct QueueSplitterStatus { + pub active_filters: Option, +} + +impl QueueSplitterStatus { + pub fn is_active(&self) -> bool { + self.active_filters.is_some() + } + + pub fn is_inactive(&self) -> bool { + !self.is_active() + } +} + +/// Defines a Custom Resource that holds a central configuration for splitting a queue. mirrord +/// users specify a splitter by name in their configuration. mirrord then starts splitting according +/// to the spec and the user's filter. +#[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[kube( +group = "splitters.mirrord.metalbear.co", +version = "v1alpha", +kind = "MirrordQueueSplitter", +shortname = "qs", +status = "QueueSplitterStatus", +namespaced +)] +pub struct MirrordQueueSplitterSpec { + /// A map of the queues that should be split. + /// The key is used by users to associate filters to the right queues. + pub queues: HashMap, + + /// The resource (deployment or Argo rollout) that reads from the queues. + pub consumer: QueueConsumer, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename = "SQSSessionDetails", rename_all = "camelCase")] +pub struct SqsSessionDetails { + // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with + // `kubectl apply` due to a length limit. + pub spec: PodSpec, + pub queue_names: HashMap, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] +pub struct SqsSessionStatus { + pub details: Option, +} + +impl SqsSessionStatus { + pub fn is_ready(&self) -> bool { + self.details.is_some() + } +} + + +/// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. +/// To be used in [`kube::runtime::wait::await_condition`]. +pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { + session + .and_then(|session| session.status.as_ref()) + .map(|status| status.is_ready()) + .unwrap_or_default() +} + +// TODO: docs +// TODO: Clients actually never use this resource in any way directly, so maybe we could define it +// in the operator on startup? The operator would probably need permissions to create CRDs and to +// give itself permissions for those CRDs? Is that possible? Right now it's the user that installs +// the operator that defines this CRD and also give the operator permissions to do stuff with it. +#[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[kube( +group = "splitters.mirrord.metalbear.co", +version = "v1alpha", +kind = "MirrordSQSSession", +root = "MirrordSqsSession", // for Rust naming conventions (Sqs, not SQS) +status = "SqsSessionStatus", +namespaced +)] +#[serde(rename_all = "camelCase")] // queue_filters -> queueFilters +pub struct MirrordSqsSessionSpec { + pub queue_filters: HashMap, + pub queue_consumer: QueueConsumer, + // The Kubernetes API can't deal with 64 bit numbers (with most significant bit set) + // so we save that field as a string even though its source is a u64 + pub session_id: String, +} diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 1fbed12d840..0081ca2fc89 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -23,7 +23,7 @@ use k8s_openapi::{ use kube::{CustomResourceExt, Resource}; use thiserror::Error; -use crate::crd::{MirrordPolicy, TargetCrd}; +use crate::crd::{MirrordPolicy, MirrordQueueSplitter, MirrordSqsSession, TargetCrd}; static OPERATOR_NAME: &str = "mirrord-operator"; static OPERATOR_PORT: i32 = 3000; @@ -177,6 +177,12 @@ impl OperatorSetup for Operator { writer.write_all(b"---\n")?; MirrordPolicy::crd().to_writer(&mut writer)?; + writer.write_all(b"---\n")?; + MirrordQueueSplitter::crd().to_writer(&mut writer)?; + + writer.write_all(b"---\n")?; + MirrordSqsSession::crd().to_writer(&mut writer)?; + Ok(()) } } @@ -412,6 +418,28 @@ impl OperatorRole { verbs: vec!["get".to_owned(), "list".to_owned(), "watch".to_owned()], ..Default::default() }, + // For SQS controller to temporarily change deployments to use changed queues. + PolicyRule { + api_groups: Some(vec![ + "apps".to_owned(), + ]), + resources: Some(vec![ + "deployments".to_owned(), + ]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, + // For SQS controller to temporarily change Argo Rollouts to use changed queues. + PolicyRule { + api_groups: Some(vec![ + "argoproj.io".to_owned(), + ]), + resources: Some(vec![ + "rollouts".to_owned(), + ]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, PolicyRule { api_groups: Some(vec!["apps".to_owned(), "argoproj.io".to_owned()]), resources: Some(vec![ @@ -452,13 +480,56 @@ impl OperatorRole { verbs: vec!["impersonate".to_owned()], ..Default::default() }, - // Allow the operator to list mirrord policies. + // Allow the operator to list+get mirrord policies. PolicyRule { api_groups: Some(vec!["policies.mirrord.metalbear.co".to_owned()]), resources: Some(vec![MirrordPolicy::plural(&()).to_string()]), verbs: vec!["list".to_owned(), "get".to_owned()], ..Default::default() }, + // Allow the operator to list mirrord queue splitters. + PolicyRule { + api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + resources: Some(vec![MirrordQueueSplitter::plural(&()).to_string()]), + verbs: vec![ + "list".to_owned(), + ], + ..Default::default() + }, + // Allow the SQS controller to update queue splitter status. + PolicyRule { + api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + resources: Some(vec!["mirrordqueuesplitters/status".to_string()]), + verbs: vec![ + // For setting the status in the SQS controller. + "update".to_owned(), + ], + ..Default::default() + }, + // Allow the operator to control mirrord queue filters. + PolicyRule { + api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + resources: Some(vec![MirrordSqsSession::plural(&()).to_string()]), + verbs: vec![ + "create".to_owned(), + "watch".to_owned(), + "list".to_owned(), + "get".to_owned(), + "delete".to_owned(), + "patch".to_owned(), + ], + ..Default::default() + }, + // Allow the SQS controller to update queue splitter status. + PolicyRule { + api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + resources: Some(vec!["mirrordsqssessions/status".to_string()]), + verbs: vec![ + // For setting the status in the SQS controller. + "update".to_owned(), + ], + ..Default::default() + }, ]), ..Default::default() }; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b8b63b9b653..06751641ab8 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" doctest = false [dependencies] +aws-config = { version = "1.1.2", optional = true } +aws-sdk-sqs = { version = "1.10.0", features = ["behavior-version-latest"], optional = true } k8s-openapi.workspace = true kube.workspace = true reqwest.workspace = true @@ -36,7 +38,7 @@ const_format = "0.2" default = ["ephemeral", "job", "cli", "targetless"] ephemeral = [] job = [] -operator = [] +operator = ["dep:aws-config", "dep:aws-sdk-sqs"] docker = [] cli = [] targetless = [] diff --git a/tests/configs/sqs_queue_splitting_a.json b/tests/configs/sqs_queue_splitting_a.json new file mode 100644 index 00000000000..40658ddaf3e --- /dev/null +++ b/tests/configs/sqs_queue_splitting_a.json @@ -0,0 +1,13 @@ +{ + "feature": { + "split_queues": { + "e2e-test-queue": { + "queue_type": "SQS", + "message_filter": { + "wows": "so wows", + "coolz": "^very .*" + } + } + } + } +} \ No newline at end of file diff --git a/tests/src/operator.rs b/tests/src/operator.rs index bcccea48431..fbae776ec9f 100644 --- a/tests/src/operator.rs +++ b/tests/src/operator.rs @@ -2,4 +2,5 @@ mod concurrent_steal; mod policies; +mod queue_splitting; mod setup; diff --git a/tests/src/operator/queue_splitting.rs b/tests/src/operator/queue_splitting.rs new file mode 100644 index 00000000000..8baf8182f69 --- /dev/null +++ b/tests/src/operator/queue_splitting.rs @@ -0,0 +1,191 @@ +#![cfg(test)] +#![cfg(feature = "operator")] +//! Test queue splitting features with an operator. + +use std::collections::{BTreeMap, HashMap}; + +use aws_sdk_sqs::types::QueueAttributeName::{FifoQueue, MessageRetentionPeriod}; +use k8s_openapi::api::core::v1::ConfigMap; +use kube::{Api, Client}; +use mirrord_operator::crd::{ + MirrordQueueSplitter, MirrordQueueSplitterSpec, QueueConsumer, QueueNameSource, SplitQueue, +}; +use rstest::*; +use tokio::join; + +use crate::utils::{ + config_dir, kube_client, service, service_with_env, Application, KubeService, ResourceGuard, +}; + +const SQS_CONFIG_MAP_NAME: &str = "mirrord-e2e-test-sqs-splitting"; +const SQS_CONFIG_MAP_KEY_NAME: &str = "queue_name"; +const QUEUE_SPLITTER_RESOURCE_NAME: &str = "mirrord-e2e-test-queue-splitter"; + +#[fixture] +fn sqs_queue_name() -> String { + format!( + "MirrordE2ESplitterTests-{}.fifo", + crate::utils::random_string() + ) +} + +/// Create a new SQS fifo queue with a randomized name, return queue name and url. +#[fixture] +async fn sqs_queue(sqs_queue_name: String) -> (String, String) { + let shared_config = aws_config::load_from_env().await; + let client = aws_sdk_sqs::Client::new(&shared_config); + let queue = client + .create_queue() + .queue_name(sqs_queue_name.clone()) + .attributes(MessageRetentionPeriod, "3600") // delete messages after an hour. + .attributes(FifoQueue, "true") // Fifo for predictable test scenarios + .send() + .await + .unwrap(); + (sqs_queue_name, queue.queue_url.unwrap()) +} + +pub async fn create_config_map( + kube_client: Client, + namespace: &str, + sqs_queue_name: String, +) -> ResourceGuard { + let config_map_api: Api = Api::namespaced(kube_client.clone(), namespace); + let config_map = ConfigMap { + binary_data: None, + data: Some(BTreeMap::from([( + SQS_CONFIG_MAP_KEY_NAME.to_string(), + sqs_queue_name, + )])), + immutable: None, + metadata: Default::default(), + }; + ResourceGuard::create( + config_map_api, + SQS_CONFIG_MAP_NAME.to_string(), + &config_map, + true, + ) + .await + .expect("Could not create config map in E2E test.") +} + +/// Create the `MirrordQueueSplitter` K8s resource and a resource guard to delete it when done. +pub async fn create_splitter_resource( + kube_client: Client, + namespace: &str, + deployment_name: &str, +) -> ResourceGuard { + let qs_api: Api = Api::namespaced(kube_client.clone(), namespace); + let queue_splitter = MirrordQueueSplitter::new( + QUEUE_SPLITTER_RESOURCE_NAME, + MirrordQueueSplitterSpec { + queues: HashMap::from([( + "e2e-test-queue".to_string(), + SplitQueue::Sqs(QueueNameSource::ConfigMap { + name: SQS_CONFIG_MAP_NAME.to_string(), + queue_name_key: SQS_CONFIG_MAP_KEY_NAME.to_string(), + sub_key: None, + }), + )]), + consumer: QueueConsumer::Deployment(deployment_name.to_string()), + }, + ); + ResourceGuard::create( + qs_api, + QUEUE_SPLITTER_RESOURCE_NAME.to_string(), + &queue_splitter, + true, + ) + .await + .expect("Could not create queue splitter in E2E test.") +} + +#[fixture] +pub async fn sqs_consumer_service(#[future] kube_client: Client) -> KubeService { + // By using a different namespace for each test run, we're able to use a constant ConfigMap + // name, and don't have to pass that information to the test service. + let namespace = format!("e2e-tests-sqs-splitting-{}", crate::utils::random_string()); + service_with_env( + &namespace, + "ClusterIP", + "ghcr.io/metalbear-co/mirrord-node-udp-logger:latest", // TODO + "queue-forwarder", + false, + kube_client, + serde_json::json!([ + // TODO + ]), + ) + .await +} + +/// Send 6 messages to the original queue, such that 2 will reach each mirrord run and 2 the +/// deployed app. +async fn write_sqs_messages(queue_url: &str) { + // TODO +} + +/// This test creates a new sqs_queue with a random name and credentials from env. +/// +/// Define a queue splitter for a deployment. Start two services that both consume from an SQS +/// queue, send some messages to the queue, verify that each of the applications running with +/// mirrord get exactly the messages they are supposed to, and that the deployed application gets +/// the rest. +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +pub async fn two_users_one_queue( + config_dir: &std::path::PathBuf, + #[future] kube_client: Client, + #[future] sqs_consumer_service: KubeService, + #[future] sqs_queue: (String, String), +) { + let application = Application::Go21HTTP; // TODO + let kube_client = kube_client.await; + let (sqs_queue_name, sqs_queue_url) = sqs_queue.await; + + let deployed = sqs_consumer_service.await; + + // Create the config map that the remote deployment uses as the source of the queue's name. + let _config_map_guard = + create_config_map(kube_client.clone(), &deployed.namespace, sqs_queue_name); + + // Create the `MirrordQueueSplitter` to be used in the client configurations. + let _splitter_resource_guard = + create_splitter_resource(kube_client.clone(), &deployed.namespace, &deployed.name).await; + + let mut config_path = config_dir.clone(); + config_path.push("sqs_queue_splitting_a.json"); + + let mut client_a = application + .run( + &deployed.target, // TODO: target the deployment maybe? + Some(&deployed.namespace), + None, + Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), + ) + .await; + + let mut config_path = config_dir.clone(); + config_path.push("sqs_queue_splitting_b.json"); + + let mut client_b = application + .run( + &deployed.target, // TODO: target the deployment maybe? + Some(&deployed.namespace), + None, + Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), + ) + .await; + + write_sqs_messages(&sqs_queue_url).await; + + // The test application consumes messages and verifies exact expected messages. + join!( + client_a.wait_assert_success(), + client_b.wait_assert_success() + ); + + // TODO: read output queue and verify that exactly the expected messages were + // consumed and forwarded. +} diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 0ef37e2e801..dc91aeabc74 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -30,7 +30,7 @@ use rand::{distributions::Alphanumeric, Rng}; use reqwest::{RequestBuilder, StatusCode}; use rstest::*; use serde::{de::DeserializeOwned, Serialize}; -use serde_json::json; +use serde_json::{json, Value}; use tempfile::{tempdir, TempDir}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt, BufReader}, @@ -656,6 +656,25 @@ impl Drop for KubeService { } } +fn default_env() -> Value { + json!( + [ + { + "name": "MIRRORD_FAKE_VAR_FIRST", + "value": "mirrord.is.running" + }, + { + "name": "MIRRORD_FAKE_VAR_SECOND", + "value": "7777" + }, + { + "name": "MIRRORD_FAKE_VAR_THIRD", + "value": "foo=bar" + } + ] + ) +} + /// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted /// when the returned service is dropped, unless it is dropped during panic. /// This behavior can be changed, see [`FORCE_CLEANUP_ENV_NAME`]. @@ -668,6 +687,62 @@ pub async fn service( #[default("http-echo")] service_name: &str, #[default(true)] randomize_name: bool, #[future] kube_client: Client, +) -> KubeService { + internal_service( + namespace, + service_type, + image, + service_name, + randomize_name, + kube_client.await, + default_env(), + ) +} + +/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted +/// when the returned service is dropped, unless it is dropped during panic. +/// This behavior can be changed, see [`FORCE_CLEANUP_ENV_NAME`]. +/// * `randomize_name` - whether a random suffix should be added to the end of the resource names +/// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. +pub async fn service_with_env( + namespace: &str, + service_type: &str, + image: &str, + service_name: &str, + randomize_name: bool, + kube_client: Client, + env: Value, +) -> KubeService { + internal_service( + namespace, + service_type, + image, + service_name, + randomize_name, + kube_client.await, + env, + ) +} + +/// Internal function to create a custom [`KubeService`]. +/// We keep this private so that whenever we need more customization of test resources, we can +/// change this function and how the public ones use it, and add a new public function that exposes +/// more customization, and we don't need to change all existing usages of public functions/fixtures +/// in tests. +/// +/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be +/// deleted when the returned service is dropped, unless it is dropped during panic. +/// This behavior can be changed, see [`FORCE_CLEANUP_ENV_NAME`]. +/// * `randomize_name` - whether a random suffix should be added to the end of the resource names +/// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. +async fn internal_service( + namespace: &str, + service_type: &str, + image: &str, + service_name: &str, + randomize_name: bool, + kube_client: Client, + env: Value, ) -> KubeService { let delete_after_fail = std::env::var_os(PRESERVE_FAILED_ENV_NAME).is_none(); @@ -758,20 +833,7 @@ pub async fn service( "containerPort": 80 } ], - "env": [ - { - "name": "MIRRORD_FAKE_VAR_FIRST", - "value": "mirrord.is.running" - }, - { - "name": "MIRRORD_FAKE_VAR_SECOND", - "value": "7777" - }, - { - "name": "MIRRORD_FAKE_VAR_THIRD", - "value": "foo=bar" - } - ], + "env": env, } ] } From b3fcaddce0437bd960a19190d6221472b500d84e Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 7 May 2024 21:45:08 +0200 Subject: [PATCH 02/83] working env --- Cargo.lock | 367 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 365 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ded64b9bb6..11396e6b820 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,6 +574,323 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "aws-config" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297b64446175a73987cedc3c438d79b2a654d0fff96f65ff530fbe039347644c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "hyper 0.14.28", + "ring 0.17.8", + "time", + "tokio", + "tracing 0.1.40", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4963ac9ff2d33a4231b3806c1c69f578f221a9cabb89ad2bde62ce2b442c8a7" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing 0.1.40", + "uuid", +] + +[[package]] +name = "aws-sdk-sqs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaacad4846908c0ff49edf57136ab10b58b177739897a502fe007db7a2025af2" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing 0.1.40", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "019a07902c43b03167ea5df0182f0cb63fae89f9a9682c44d18cf2e4a042cb34" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing 0.1.40", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c46ee08a48a7f4eaa4ad201dcc1dd537b49c50859d14d4510e00ad9d3f9af2" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing 0.1.40", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f752ac730125ca6017f72f9db5ec1772c9ecc664f87aa7507a7d81b023c23713" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing 0.1.40", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58b56f1cbe6fd4d0c2573df72868f20ab1c125ca9c9dbce17927a463433a2e57" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "percent-encoding", + "sha2", + "time", + "tracing 0.1.40", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing 0.1.40", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec81002d883e5a7fd2bb063d6fb51c4999eb55d404f4fff3dd878bf4733b9f01" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.10", + "tokio", + "tracing 0.1.40", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c19fdae6e3d5ac9cd01f2d6e6c359c5f5a3e028c2d148a8f5b90bf3399a18a7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing 0.1.40", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe14dceea1e70101d38fbf2a99e6a34159477c0fb95e68e05c66bd7ae4c3729" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a43b56df2c529fe44cb4d92bd64d0479883fb9608ff62daede4df5405381814" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.12", + "rustc_version", + "tracing 0.1.40", +] + [[package]] name = "axum" version = "0.6.20" @@ -685,6 +1002,16 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -929,6 +1256,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.3.0" @@ -3542,7 +3879,7 @@ dependencies = [ "mirrord-protocol", "rand", "regex", - "rstest 0.17.0", + "rstest 0.18.2", "serde", "serde_json", "shellexpand", @@ -3591,7 +3928,7 @@ dependencies = [ "os_info", "rand", "regex", - "rstest 0.17.0", + "rstest 0.18.2", "serde", "serde_json", "socket2 0.5.6", @@ -4031,6 +4368,12 @@ dependencies = [ name = "outgoing" version = "3.100.1" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -4843,6 +5186,12 @@ dependencies = [ "regex-syntax 0.8.3", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -5947,6 +6296,8 @@ checksum = "8e7a7de15468c6e65dd7db81cf3822c1ec94c71b2a3c1a976ea8e4696c91115c" name = "tests" version = "0.1.0" dependencies = [ + "aws-config", + "aws-sdk-sqs", "bytes", "chrono", "const_format", @@ -6822,6 +7173,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -7264,6 +7621,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xz" version = "0.1.0" From df32b32bb04db870398760bc086481bfd64e7133 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 14 May 2024 15:19:31 +0200 Subject: [PATCH 03/83] deletecollection for sqs sessions --- mirrord/operator/src/setup.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 0081ca2fc89..645b96118c6 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -516,6 +516,7 @@ impl OperatorRole { "list".to_owned(), "get".to_owned(), "delete".to_owned(), + "deletecollection".to_owned(), "patch".to_owned(), ], ..Default::default() From d07bc0840f65e8fba4a8f79d3a056d7f303cf4c1 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 15 May 2024 10:03:23 +0200 Subject: [PATCH 04/83] change SqsMessageFilter to BTreeMap for iterating. --- mirrord/config/src/feature/split_queues.rs | 8 ++++---- mirrord/operator/src/client.rs | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index aa1cd60fb5b..1583938a0d7 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use mirrord_analytics::{Analytics, CollectAnalytics}; use schemars::JsonSchema; @@ -31,7 +31,7 @@ pub type QueueId = String; /// } /// ``` #[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Deserialize, Default)] -pub struct SplitQueuesConfig(Option>); +pub struct SplitQueuesConfig(Option>); impl SplitQueuesConfig { pub fn is_set(&self) -> bool { @@ -39,7 +39,7 @@ impl SplitQueuesConfig { } /// Out of the whole queue splitting config, get only the sqs queues. - pub fn get_sqs_filter(&self) -> Option>> { + pub fn get_sqs_filter(&self) -> Option> { self.0.as_ref().map(|queue_id2queue_filter| { queue_id2queue_filter .iter() @@ -71,7 +71,7 @@ impl FromMirrordConfig for SplitQueuesConfig { pub type MessageAttributeName = String; pub type AttributeValuePattern = String; -pub type SqsMessageFilter = HashMap; +pub type SqsMessageFilter = BTreeMap; /// More queue types might be added in the future. #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, JsonSchema)] diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 4452bb10d3b..c2be04376c0 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -3,6 +3,7 @@ use std::{ fmt::{self, Display}, io, }; +use std::collections::BTreeMap; use base64::{engine::general_purpose, Engine as _}; use chrono::{DateTime, Utc}; @@ -594,7 +595,7 @@ impl OperatorApi { session_metadata: &OperatorSessionMetadata, target: Target, scale_down: bool, - sqs_filter: Option>>, + sqs_filter: Option>>, ) -> Result { let name = TargetCrd::target_name(&target); From cb91e2d80400b42a4e1d5abb3dbd3a5cf46fe505 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 15 May 2024 10:18:17 +0200 Subject: [PATCH 05/83] change queue_names to BTreeMap for iterating --- mirrord/operator/src/crd.rs | 51 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index b7d39995121..f6a07c4f5c0 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -2,6 +2,7 @@ use std::{ collections::HashMap, fmt::{Display, Formatter}, }; +use std::collections::BTreeMap; use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; use kube::CustomResource; @@ -24,11 +25,11 @@ pub const TARGETLESS_TARGET_NAME: &str = "targetless"; #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( - group = "operator.metalbear.co", - version = "v1", - kind = "Target", - root = "TargetCrd", - namespaced +group = "operator.metalbear.co", +version = "v1", +kind = "Target", +root = "TargetCrd", +namespaced )] pub struct TargetSpec { /// None when targetless. @@ -92,11 +93,11 @@ pub static OPERATOR_STATUS_NAME: &str = "operator"; #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( - group = "operator.metalbear.co", - version = "v1", - kind = "MirrordOperator", - root = "MirrordOperatorCrd", - status = "MirrordOperatorStatus" +group = "operator.metalbear.co", +version = "v1", +kind = "MirrordOperator", +root = "MirrordOperatorCrd", +status = "MirrordOperatorStatus" )] pub struct MirrordOperatorSpec { pub operator_version: String, @@ -235,10 +236,10 @@ pub struct Session { /// the operator. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( - group = "operator.metalbear.co", - version = "v1", - kind = "Session", - root = "SessionCrd" +group = "operator.metalbear.co", +version = "v1", +kind = "Session", +root = "SessionCrd" )] pub struct SessionSpec; @@ -290,11 +291,11 @@ impl From<&OperatorFeatures> for NewOperatorFeature { /// (operator's copy pod feature). #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( - group = "operator.metalbear.co", - version = "v1", - kind = "CopyTarget", - root = "CopyTargetCrd", - namespaced +group = "operator.metalbear.co", +version = "v1", +kind = "CopyTarget", +root = "CopyTargetCrd", +namespaced )] pub struct CopyTargetSpec { /// Original target. Only [`Target::Pod`] and [`Target::Deployment`] are accepted. @@ -323,11 +324,11 @@ pub enum BlockedFeature { /// Custom resource for policies that limit what mirrord features users can use. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( - // The operator group is handled by the operator, we want policies to be handled by k8s. - group = "policies.mirrord.metalbear.co", - version = "v1alpha", - kind = "MirrordPolicy", - namespaced +// The operator group is handled by the operator, we want policies to be handled by k8s. +group = "policies.mirrord.metalbear.co", +version = "v1alpha", +kind = "MirrordPolicy", +namespaced )] #[serde(rename_all = "camelCase")] // target_path -> targetPath in yaml. pub struct MirrordPolicySpec { @@ -421,7 +422,7 @@ pub struct QueueNameUpdate { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct QueueDetails { /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config map. - pub queue_names: HashMap, + pub queue_names: BTreeMap, /// Names of env vars that contain the queue name directly in the pod spec, without config /// map refs, mapped to their queue id. From 4147d227524d0342f94223175de0b1bbc90b3833 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 15 May 2024 10:23:34 +0200 Subject: [PATCH 06/83] change config_map_updates to BTreeMap for iterating --- mirrord/operator/src/crd.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index f6a07c4f5c0..16e6a747199 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -430,11 +430,11 @@ pub struct QueueDetails { /// For each config map name, a mapping from queue id to key name in the map that holds the name /// of that queue. - pub config_map_updates: HashMap>, - // ^ ^ ^ - // | | ---- name of key that points to Q name - // | ---- queue id - // ---- ConfigMap name + pub config_map_updates: BTreeMap>, + // ^ ^ ^ + // | | ---- name of key that points to Q name + // | ---- queue id + // ---- ConfigMap name // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with // `kubectl apply` due to a length limit. From d17d15ae8984c6dc816f1004df6390eaf95fb33f Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 15 May 2024 10:30:25 +0200 Subject: [PATCH 07/83] change queue_names to BTreeMap for iterating --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 16e6a747199..6f7a57f998e 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -485,7 +485,7 @@ pub struct SqsSessionDetails { // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with // `kubectl apply` due to a length limit. pub spec: PodSpec, - pub queue_names: HashMap, + pub queue_names: BTreeMap, } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] From 186867a44f006d9c281118e721eda303bfdb8f69 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 15 May 2024 10:34:56 +0200 Subject: [PATCH 08/83] change queues to BTreeMap for iterating --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 6f7a57f998e..f90bcb4164d 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -473,7 +473,7 @@ namespaced pub struct MirrordQueueSplitterSpec { /// A map of the queues that should be split. /// The key is used by users to associate filters to the right queues. - pub queues: HashMap, + pub queues: BTreeMap, /// The resource (deployment or Argo rollout) that reads from the queues. pub consumer: QueueConsumer, From 0cc6d96e87c68b0db2058e3f1ade1b808081ec61 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 21 May 2024 22:09:23 +0200 Subject: [PATCH 09/83] get output queue names from splitter status --- mirrord/operator/src/crd.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index f90bcb4164d..79a948ca75c 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -445,6 +445,7 @@ pub struct QueueDetails { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // active_filters -> activeFilters pub struct QueueSplitterStatus { + // TODO: that option is unnecessary, right? pub active_filters: Option, } @@ -456,6 +457,16 @@ impl QueueSplitterStatus { pub fn is_inactive(&self) -> bool { !self.is_active() } + + pub fn output_queue_names(&self) -> Vec<&str> { + self.active_filters + .as_ref() + .map(|QueueDetails { queue_names, .. }| queue_names + .values() + .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) + .collect()) + .unwrap_or_default() + } } /// Defines a Custom Resource that holds a central configuration for splitting a queue. mirrord From fdc7ced7695d92114d4bc05ff7a3d8c87d735c2e Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 21 May 2024 22:33:42 +0200 Subject: [PATCH 10/83] splitter status --- mirrord/operator/src/crd.rs | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 79a948ca75c..cbd7788779e 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, fmt::{Display, Formatter}, }; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; use kube::CustomResource; @@ -421,7 +421,8 @@ pub struct QueueNameUpdate { // is organized differently. #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct QueueDetails { - /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config map. + /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config + /// map, together with the name of its temporary output queue. pub queue_names: BTreeMap, /// Names of env vars that contain the queue name directly in the pod spec, without config @@ -445,27 +446,18 @@ pub struct QueueDetails { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // active_filters -> activeFilters pub struct QueueSplitterStatus { - // TODO: that option is unnecessary, right? - pub active_filters: Option, + pub queue_details: QueueDetails, + + /// Resource names of active sessions of this splitter. + pub active_sessions: BTreeSet, } impl QueueSplitterStatus { - pub fn is_active(&self) -> bool { - self.active_filters.is_some() - } - - pub fn is_inactive(&self) -> bool { - !self.is_active() - } - pub fn output_queue_names(&self) -> Vec<&str> { - self.active_filters - .as_ref() - .map(|QueueDetails { queue_names, .. }| queue_names - .values() - .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) - .collect()) - .unwrap_or_default() + self.queue_details.queue_names + .values() + .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) + .collect() } } From 2578dddfbeb32f9f35f6fc9c8050162e64cddda1 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 22 May 2024 13:36:18 +0200 Subject: [PATCH 11/83] todo --- mirrord/operator/src/crd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index cbd7788779e..27043f09887 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -494,6 +494,7 @@ pub struct SqsSessionDetails { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] pub struct SqsSessionStatus { + // TODO: is this option unnecessary? pub details: Option, } From 37795f54709d1cc3936e4dd608a4c27d53b87f6c Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 22 May 2024 14:13:46 +0200 Subject: [PATCH 12/83] set -> vec --- mirrord/operator/src/crd.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 27043f09887..48162d08f67 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -448,8 +448,9 @@ pub struct QueueDetails { pub struct QueueSplitterStatus { pub queue_details: QueueDetails, + // can't be a set: "uniqueItems cannot be set to true since the runtime complexity becomes quadratic" /// Resource names of active sessions of this splitter. - pub active_sessions: BTreeSet, + pub active_sessions: Vec, } impl QueueSplitterStatus { From b1524f7705194b52aab626e1c2a021566d44fc8e Mon Sep 17 00:00:00 2001 From: t4lz Date: Sat, 25 May 2024 10:08:34 +0200 Subject: [PATCH 13/83] tags --- mirrord/operator/src/crd.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 48162d08f67..07a8400fdc2 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, fmt::{Display, Formatter}, }; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; use kube::CustomResource; @@ -481,6 +481,10 @@ pub struct MirrordQueueSplitterSpec { /// The resource (deployment or Argo rollout) that reads from the queues. pub consumer: QueueConsumer, + + /// These tags will be added to all temporary SQS queues created by mirrord for queues defined + /// in this MirrordQueueSplitter. + pub tags: Option>, } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] From ba17e1c6041a23c8fd0f3c3eadc0d499a127a925 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 27 May 2024 19:36:03 +0200 Subject: [PATCH 14/83] docs --- mirrord/operator/src/crd.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 07a8400fdc2..1d57ef17f11 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -482,8 +482,10 @@ pub struct MirrordQueueSplitterSpec { /// The resource (deployment or Argo rollout) that reads from the queues. pub consumer: QueueConsumer, - /// These tags will be added to all temporary SQS queues created by mirrord for queues defined - /// in this MirrordQueueSplitter. + /// These tags will be set for all temporary SQS queues created by mirrord for queues defined + /// in this MirrordQueueSplitter, alongside with the original tags of the respective original + /// queue. In case of a collision, the temporary queue will get the value from the tag passed + /// in here. pub tags: Option>, } From 0a52ca57a17dfeefe7404a54bf6393145af0ce1c Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 31 May 2024 11:50:53 +0200 Subject: [PATCH 15/83] remove active_sessions from splitter status --- mirrord/operator/src/crd.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 1d57ef17f11..ed8fb654571 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -447,10 +447,6 @@ pub struct QueueDetails { #[serde(rename_all = "camelCase")] // active_filters -> activeFilters pub struct QueueSplitterStatus { pub queue_details: QueueDetails, - - // can't be a set: "uniqueItems cannot be set to true since the runtime complexity becomes quadratic" - /// Resource names of active sessions of this splitter. - pub active_sessions: Vec, } impl QueueSplitterStatus { From d213e4f467a18de057ea7bde2a0774a275ec5c46 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 31 May 2024 12:20:36 +0200 Subject: [PATCH 16/83] MirrordQueueSplitter -> MirrordWorkloadQueueRegistry, splitters. -> queues. --- mirrord/operator/src/crd.rs | 16 ++++++++-------- mirrord/operator/src/setup.rs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index ed8fb654571..5f90a333121 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -445,11 +445,11 @@ pub struct QueueDetails { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // active_filters -> activeFilters -pub struct QueueSplitterStatus { +pub struct WorkloadQueueRegistryStatus { pub queue_details: QueueDetails, } -impl QueueSplitterStatus { +impl WorkloadQueueRegistryStatus { pub fn output_queue_names(&self) -> Vec<&str> { self.queue_details.queue_names .values() @@ -463,14 +463,14 @@ impl QueueSplitterStatus { /// to the spec and the user's filter. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "splitters.mirrord.metalbear.co", +group = "queues.mirrord.metalbear.co", version = "v1alpha", -kind = "MirrordQueueSplitter", +kind = "MirrordWorkloadQueueRegistry", shortname = "qs", -status = "QueueSplitterStatus", +status = "WorkloadQueueRegistryStatus", namespaced )] -pub struct MirrordQueueSplitterSpec { +pub struct MirrordWorkloadQueueRegistrySpec { /// A map of the queues that should be split. /// The key is used by users to associate filters to the right queues. pub queues: BTreeMap, @@ -479,7 +479,7 @@ pub struct MirrordQueueSplitterSpec { pub consumer: QueueConsumer, /// These tags will be set for all temporary SQS queues created by mirrord for queues defined - /// in this MirrordQueueSplitter, alongside with the original tags of the respective original + /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective original /// queue. In case of a collision, the temporary queue will get the value from the tag passed /// in here. pub tags: Option>, @@ -524,7 +524,7 @@ pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { // the operator that defines this CRD and also give the operator permissions to do stuff with it. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "splitters.mirrord.metalbear.co", +group = "queues.mirrord.metalbear.co", version = "v1alpha", kind = "MirrordSQSSession", root = "MirrordSqsSession", // for Rust naming conventions (Sqs, not SQS) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 645b96118c6..bd881abc415 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -23,7 +23,7 @@ use k8s_openapi::{ use kube::{CustomResourceExt, Resource}; use thiserror::Error; -use crate::crd::{MirrordPolicy, MirrordQueueSplitter, MirrordSqsSession, TargetCrd}; +use crate::crd::{MirrordPolicy, MirrordWorkloadQueueRegistry, MirrordSqsSession, TargetCrd}; static OPERATOR_NAME: &str = "mirrord-operator"; static OPERATOR_PORT: i32 = 3000; @@ -178,7 +178,7 @@ impl OperatorSetup for Operator { MirrordPolicy::crd().to_writer(&mut writer)?; writer.write_all(b"---\n")?; - MirrordQueueSplitter::crd().to_writer(&mut writer)?; + MirrordWorkloadQueueRegistry::crd().to_writer(&mut writer)?; writer.write_all(b"---\n")?; MirrordSqsSession::crd().to_writer(&mut writer)?; @@ -489,8 +489,8 @@ impl OperatorRole { }, // Allow the operator to list mirrord queue splitters. PolicyRule { - api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), - resources: Some(vec![MirrordQueueSplitter::plural(&()).to_string()]), + api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), + resources: Some(vec![MirrordWorkloadQueueRegistry::plural(&()).to_string()]), verbs: vec![ "list".to_owned(), ], @@ -498,8 +498,8 @@ impl OperatorRole { }, // Allow the SQS controller to update queue splitter status. PolicyRule { - api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), - resources: Some(vec!["mirrordqueuesplitters/status".to_string()]), + api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), + resources: Some(vec!["mirrordworkloadqueueregistries/status".to_string()]), verbs: vec![ // For setting the status in the SQS controller. "update".to_owned(), @@ -508,7 +508,7 @@ impl OperatorRole { }, // Allow the operator to control mirrord queue filters. PolicyRule { - api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec![MirrordSqsSession::plural(&()).to_string()]), verbs: vec![ "create".to_owned(), @@ -523,7 +523,7 @@ impl OperatorRole { }, // Allow the SQS controller to update queue splitter status. PolicyRule { - api_groups: Some(vec!["splitters.mirrord.metalbear.co".to_owned()]), + api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec!["mirrordsqssessions/status".to_string()]), verbs: vec![ // For setting the status in the SQS controller. From 44fd364fc88c1065e5fb4e95313956114ffacd21 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 4 Jun 2024 13:48:14 +0200 Subject: [PATCH 17/83] merge fix --- Cargo.lock | 2 +- mirrord/config/src/target.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 69193852d43..e511e07c305 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -823,7 +823,7 @@ dependencies = [ "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.10", + "rustls 0.21.12", "tokio", "tracing 0.1.40", ] diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs index 68f39836a13..86ee1ca5f86 100644 --- a/mirrord/config/src/target.rs +++ b/mirrord/config/src/target.rs @@ -276,6 +276,7 @@ impl Target { Target::Pod(pod) => pod.target_type(), Target::Deployment(dep) => dep.target_type(), Target::Rollout(roll) => roll.target_type(), + Target::Job(job) => job.target_type(), } } } From da74d6dd4bc5b6b293ca2782980432ee97a60484 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 12 Jun 2024 18:36:31 -0400 Subject: [PATCH 18/83] lock --- Cargo.lock | 364 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 364 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 9c4151c4d98..8de35d259a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,6 +617,49 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-config" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac9889352d632214df943e26740c46a0f3da6e329fbd28164fe7ae1b061da7b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "hyper 0.14.28", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.7.2" @@ -644,6 +687,285 @@ dependencies = [ "paste", ] +[[package]] +name = "aws-runtime" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75588e7ee5e8496eed939adac2035a6dbab9f7eb2acdd9ab2d31856dab6f3955" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sqs" +version = "1.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d789fc494a949e70bd0dd42c636ae04ff78cd71cba58b647b2df074438312b4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb97e44983752cf7e12968c5f569a5d7562dbbc67006755c331d9d9c99580ae" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad061d977235898e4a97ecbd5d882786cca41b4828943584dc792dcc35eb3d3c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300ce43d1f7f4eb023e57d38b0921d964e8e62bed7f82f6b7849e7eab7a14575" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31eed8d45759b2c5fe7fd304dd70739060e9e0de509209036eabea14d0720cce" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db83b08939838d18e33b5dbaf1a0f048f28c10bd28071ab7ce6f245451855414" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.0", + "httparse", + "hyper 0.14.28", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b570ea39eb95bd32543f6e4032bce172cb6209b9bc8c83c770d08169e875afc" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.0", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f734808d43702a67e57d478a12e227d4d038d0b90c9005a78c87890d3805922" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.12", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.6.20" @@ -800,6 +1122,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -1050,6 +1382,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bytesize" version = "1.3.0" @@ -2802,7 +3144,9 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.28", + "log", "rustls 0.21.12", + "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -4426,6 +4770,12 @@ dependencies = [ name = "outgoing" version = "3.105.0" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -6360,6 +6710,8 @@ checksum = "8e7a7de15468c6e65dd7db81cf3822c1ec94c71b2a3c1a976ea8e4696c91115c" name = "tests" version = "0.1.0" dependencies = [ + "aws-config", + "aws-sdk-sqs", "bytes", "chrono", "const_format", @@ -7129,6 +7481,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -7599,6 +7957,12 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xz" version = "0.1.0" From ab38356f1036a12f61200d2759fbfff5fee34349 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 20 Jun 2024 07:20:26 +0200 Subject: [PATCH 19/83] mergefix --- mirrord/config/src/target.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs index 613398fe699..92a8f303030 100644 --- a/mirrord/config/src/target.rs +++ b/mirrord/config/src/target.rs @@ -303,6 +303,8 @@ impl Target { Target::Deployment(dep) => dep.target_type(), Target::Rollout(roll) => roll.target_type(), Target::Job(job) => job.target_type(), + Target::CronJob(cron_job) => cron_job.target_type(), + Target::StatefulSet(stateful_set) => stateful_set.target_type(), } } } From b58bb857cb438a02fa7371cf4f27e4750d925165 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 24 Jun 2024 23:57:54 +0200 Subject: [PATCH 20/83] remove pod spec from session status --- mirrord/operator/src/crd.rs | 102 ++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 58 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 663328c04ff..4e9d66a4bde 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -1,8 +1,7 @@ use std::{ - collections::HashMap, + collections::{BTreeMap, HashMap}, fmt::{Display, Formatter}, }; -use std::collections::BTreeMap; use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; use kube::CustomResource; @@ -13,10 +12,10 @@ use mirrord_config::{ }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[cfg(feature = "client")] -use crate::client::OperatorApiError; use self::label_selector::LabelSelector; +#[cfg(feature = "client")] +use crate::client::OperatorApiError; use crate::types::LicenseInfoOwned; pub mod label_selector; @@ -25,11 +24,11 @@ pub const TARGETLESS_TARGET_NAME: &str = "targetless"; #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "operator.metalbear.co", -version = "v1", -kind = "Target", -root = "TargetCrd", -namespaced + group = "operator.metalbear.co", + version = "v1", + kind = "Target", + root = "TargetCrd", + namespaced )] pub struct TargetSpec { /// None when targetless. @@ -95,11 +94,11 @@ pub static OPERATOR_STATUS_NAME: &str = "operator"; #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "operator.metalbear.co", -version = "v1", -kind = "MirrordOperator", -root = "MirrordOperatorCrd", -status = "MirrordOperatorStatus" + group = "operator.metalbear.co", + version = "v1", + kind = "MirrordOperator", + root = "MirrordOperatorCrd", + status = "MirrordOperatorStatus" )] pub struct MirrordOperatorSpec { pub operator_version: String, @@ -238,10 +237,10 @@ pub struct Session { /// the operator. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "operator.metalbear.co", -version = "v1", -kind = "Session", -root = "SessionCrd" + group = "operator.metalbear.co", + version = "v1", + kind = "Session", + root = "SessionCrd" )] pub struct SessionSpec; @@ -293,11 +292,11 @@ impl From<&OperatorFeatures> for NewOperatorFeature { /// (operator's copy pod feature). #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "operator.metalbear.co", -version = "v1", -kind = "CopyTarget", -root = "CopyTargetCrd", -namespaced + group = "operator.metalbear.co", + version = "v1", + kind = "CopyTarget", + root = "CopyTargetCrd", + namespaced )] pub struct CopyTargetSpec { /// Original target. Only [`Target::Pod`] and [`Target::Deployment`] are accepted. @@ -374,7 +373,8 @@ pub type OutputQueueName = String; /// The details of a queue that should be split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(tag = "queueType")] -// So that controllers that only handle 1 type of queue don't have to adapt when we add more queue types. +// So that controllers that only handle 1 type of queue don't have to adapt when we add more queue +// types. #[non_exhaustive] pub enum SplitQueue { /// Amazon SQS @@ -397,7 +397,7 @@ impl QueueConsumer { pub fn get_type_and_name(&self) -> (&str, &str) { match self { QueueConsumer::Deployment(dep) => ("deployment", dep), - QueueConsumer::Rollout(roll) => ("rollout", roll) + QueueConsumer::Rollout(roll) => ("rollout", roll), } } } @@ -413,7 +413,6 @@ pub struct QueueNameUpdate { pub output_name: String, } - /// Details retrieved from K8s resources once the splitter is active, used on filter session /// creation to create the altered config maps and pod specs that make the application use the /// output queues instead of the original. @@ -431,8 +430,8 @@ pub struct QueueDetails { /// map refs, mapped to their queue id. pub direct_env_vars: HashMap, - /// For each config map name, a mapping from queue id to key name in the map that holds the name - /// of that queue. + /// For each config map name, a mapping from queue id to key name in the map that holds the + /// name of that queue. pub config_map_updates: BTreeMap>, // ^ ^ ^ // | | ---- name of key that points to Q name @@ -453,7 +452,8 @@ pub struct WorkloadQueueRegistryStatus { impl WorkloadQueueRegistryStatus { pub fn output_queue_names(&self) -> Vec<&str> { - self.queue_details.queue_names + self.queue_details + .queue_names .values() .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) .collect() @@ -465,12 +465,12 @@ impl WorkloadQueueRegistryStatus { /// to the spec and the user's filter. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( -group = "queues.mirrord.metalbear.co", -version = "v1alpha", -kind = "MirrordWorkloadQueueRegistry", -shortname = "qs", -status = "WorkloadQueueRegistryStatus", -namespaced + group = "queues.mirrord.metalbear.co", + version = "v1alpha", + kind = "MirrordWorkloadQueueRegistry", + shortname = "qs", + status = "WorkloadQueueRegistryStatus", + namespaced )] pub struct MirrordWorkloadQueueRegistrySpec { /// A map of the queues that should be split. @@ -481,41 +481,27 @@ pub struct MirrordWorkloadQueueRegistrySpec { pub consumer: QueueConsumer, /// These tags will be set for all temporary SQS queues created by mirrord for queues defined - /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective original - /// queue. In case of a collision, the temporary queue will get the value from the tag passed - /// in here. + /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective + /// original queue. In case of a collision, the temporary queue will get the value from the + /// tag passed in here. pub tags: Option>, } -#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] -#[serde(rename = "SQSSessionDetails", rename_all = "camelCase")] -pub struct SqsSessionDetails { - // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with - // `kubectl apply` due to a length limit. - pub spec: PodSpec, - pub queue_names: BTreeMap, -} - #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] pub struct SqsSessionStatus { - // TODO: is this option unnecessary? - pub details: Option, -} - -impl SqsSessionStatus { - pub fn is_ready(&self) -> bool { - self.details.is_some() - } + pub queue_names: BTreeMap, + // A bit redundant, because the registry resource status has the mapping from env var name + // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving + // it here in the form that is useful to reader, for simplicity and readability. + pub env_updates: BTreeMap, } - /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. /// To be used in [`kube::runtime::wait::await_condition`]. pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { session - .and_then(|session| session.status.as_ref()) - .map(|status| status.is_ready()) + .map(|session| session.status.is_some()) .unwrap_or_default() } From c2d412f6d755b4be41b4be7f22f0c48c424acb91 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 25 Jun 2024 12:04:32 +0200 Subject: [PATCH 21/83] remove podspec from registry crd status --- mirrord/operator/src/crd.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 4e9d66a4bde..57a68347a0a 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -3,7 +3,7 @@ use std::{ fmt::{Display, Formatter}, }; -use k8s_openapi::api::core::v1::{PodSpec, PodTemplateSpec}; +use k8s_openapi::api::core::v1::{Container, PodTemplateSpec}; use kube::CustomResource; pub use mirrord_config::feature::split_queues::QueueId; use mirrord_config::{ @@ -441,7 +441,7 @@ pub struct QueueDetails { // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with // `kubectl apply` due to a length limit. /// The original PodSpec of the queue consumer. - pub original_spec: PodSpec, + pub containers: Vec, } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] @@ -495,6 +495,7 @@ pub struct SqsSessionStatus { // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving // it here in the form that is useful to reader, for simplicity and readability. pub env_updates: BTreeMap, + // TODO: add updates to config map references. } /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. @@ -506,10 +507,6 @@ pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { } // TODO: docs -// TODO: Clients actually never use this resource in any way directly, so maybe we could define it -// in the operator on startup? The operator would probably need permissions to create CRDs and to -// give itself permissions for those CRDs? Is that possible? Right now it's the user that installs -// the operator that defines this CRD and also give the operator permissions to do stuff with it. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( group = "queues.mirrord.metalbear.co", From dff91dbd9b32b71ebd00a173aeb3748a1b80dd13 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 25 Jun 2024 16:39:28 +0200 Subject: [PATCH 22/83] save only updates instead of containers --- mirrord/operator/src/crd.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 57a68347a0a..39e7d880892 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -3,7 +3,7 @@ use std::{ fmt::{Display, Formatter}, }; -use k8s_openapi::api::core::v1::{Container, PodTemplateSpec}; +use k8s_openapi::api::core::v1::PodTemplateSpec; use kube::CustomResource; pub use mirrord_config::feature::split_queues::QueueId; use mirrord_config::{ @@ -437,11 +437,7 @@ pub struct QueueDetails { // | | ---- name of key that points to Q name // | ---- queue id // ---- ConfigMap name - - // TODO: Don't save whole PodSpec, because its schema is so long you can't create the CRD with - // `kubectl apply` due to a length limit. - /// The original PodSpec of the queue consumer. - pub containers: Vec, + pub env_updates: BTreeMap, } #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] From 951acc79d864e281d62785fc723045aed3a3dceb Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 26 Jun 2024 20:42:41 +0200 Subject: [PATCH 23/83] docs --- mirrord/operator/src/crd.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 39e7d880892..a709bc0bea3 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -386,6 +386,7 @@ pub enum SplitQueue { Sqs(QueueNameSource), } +/// A workload that is a consumer of a queue that is being split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // Deployment -> deployment in yaml. pub enum QueueConsumer { @@ -402,11 +403,6 @@ impl QueueConsumer { } } -#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] -pub struct Filtering { - pub original_spec: PodTemplateSpec, -} - #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct QueueNameUpdate { pub original_name: String, @@ -414,12 +410,11 @@ pub struct QueueNameUpdate { } /// Details retrieved from K8s resources once the splitter is active, used on filter session -/// creation to create the altered config maps and pod specs that make the application use the +/// creation to determine the required config changes that make the application use the /// output queues instead of the original. // This status struct is not optimal in that it contains redundant information. This makes the // controller's code a bit simpler. -// The information on config map updates and on env vars is present in the resource itself, but it -// is organized differently. +// Some information is present in the spec, but it is organized differently. #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct QueueDetails { /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config @@ -486,10 +481,13 @@ pub struct MirrordWorkloadQueueRegistrySpec { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] pub struct SqsSessionStatus { + /// Queue ID -> old and new queue names. pub queue_names: BTreeMap, + // A bit redundant, because the registry resource status has the mapping from env var name // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving // it here in the form that is useful to reader, for simplicity and readability. + /// Env var name -> old and new queue names. pub env_updates: BTreeMap, // TODO: add updates to config map references. } @@ -502,7 +500,8 @@ pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { .unwrap_or_default() } -// TODO: docs +/// The operator creates this object when a user runs mirrord against a target that is a queue +/// consumer. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( group = "queues.mirrord.metalbear.co", @@ -514,8 +513,15 @@ namespaced )] #[serde(rename_all = "camelCase")] // queue_filters -> queueFilters pub struct MirrordSqsSessionSpec { + /// For each queue_id, a mapping from attribute name, to attribute value regex. + /// The queue_id for a queue is determined at the queue registry. It is not (necessarily) + /// The name of the queue on AWS. pub queue_filters: HashMap, + + /// The target of this session. pub queue_consumer: QueueConsumer, + + /// The id of the mirrord exec session, from the operator. // The Kubernetes API can't deal with 64 bit numbers (with most significant bit set) // so we save that field as a string even though its source is a u64 pub session_id: String, From 6ad3a66224b8c6cfb2524fc923f21d0db5f9bc92 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 26 Jun 2024 20:53:03 +0200 Subject: [PATCH 24/83] changelog --- changelog.d/2066.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2066.added.md diff --git a/changelog.d/2066.added.md b/changelog.d/2066.added.md new file mode 100644 index 00000000000..cb6f7a7f102 --- /dev/null +++ b/changelog.d/2066.added.md @@ -0,0 +1 @@ +Client side support for the upcoming SQS queue splitting support in *mirrord for Teams*. From 4a0a6823c4b02f56d54e37deaaa25644f1543be8 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 26 Jun 2024 21:03:04 +0200 Subject: [PATCH 25/83] remove tests (moved to sqs-tests branch) --- tests/Cargo.toml | 4 +- tests/configs/sqs_queue_splitting_a.json | 13 -- tests/src/operator.rs | 1 - tests/src/operator/queue_splitting.rs | 191 ----------------------- tests/src/utils.rs | 92 ++--------- 5 files changed, 16 insertions(+), 285 deletions(-) delete mode 100644 tests/configs/sqs_queue_splitting_a.json delete mode 100644 tests/src/operator/queue_splitting.rs diff --git a/tests/Cargo.toml b/tests/Cargo.toml index b720eec61f1..3a5a86f1797 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -10,8 +10,6 @@ workspace = true doctest = false [dependencies] -aws-config = { version = "1.1.2", optional = true } -aws-sdk-sqs = { version = "1.10.0", features = ["behavior-version-latest"], optional = true } k8s-openapi.workspace = true kube.workspace = true reqwest.workspace = true @@ -42,7 +40,7 @@ rustls.workspace = true default = ["ephemeral", "job", "cli", "targetless"] ephemeral = [] job = [] -operator = ["dep:aws-config", "dep:aws-sdk-sqs"] +operator = [] docker = [] cli = [] targetless = [] diff --git a/tests/configs/sqs_queue_splitting_a.json b/tests/configs/sqs_queue_splitting_a.json deleted file mode 100644 index 40658ddaf3e..00000000000 --- a/tests/configs/sqs_queue_splitting_a.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "feature": { - "split_queues": { - "e2e-test-queue": { - "queue_type": "SQS", - "message_filter": { - "wows": "so wows", - "coolz": "^very .*" - } - } - } - } -} \ No newline at end of file diff --git a/tests/src/operator.rs b/tests/src/operator.rs index bd6c99b9fd7..a494c46a017 100644 --- a/tests/src/operator.rs +++ b/tests/src/operator.rs @@ -3,5 +3,4 @@ mod concurrent_steal; mod policies; mod sanity; -mod queue_splitting; mod setup; diff --git a/tests/src/operator/queue_splitting.rs b/tests/src/operator/queue_splitting.rs deleted file mode 100644 index 8baf8182f69..00000000000 --- a/tests/src/operator/queue_splitting.rs +++ /dev/null @@ -1,191 +0,0 @@ -#![cfg(test)] -#![cfg(feature = "operator")] -//! Test queue splitting features with an operator. - -use std::collections::{BTreeMap, HashMap}; - -use aws_sdk_sqs::types::QueueAttributeName::{FifoQueue, MessageRetentionPeriod}; -use k8s_openapi::api::core::v1::ConfigMap; -use kube::{Api, Client}; -use mirrord_operator::crd::{ - MirrordQueueSplitter, MirrordQueueSplitterSpec, QueueConsumer, QueueNameSource, SplitQueue, -}; -use rstest::*; -use tokio::join; - -use crate::utils::{ - config_dir, kube_client, service, service_with_env, Application, KubeService, ResourceGuard, -}; - -const SQS_CONFIG_MAP_NAME: &str = "mirrord-e2e-test-sqs-splitting"; -const SQS_CONFIG_MAP_KEY_NAME: &str = "queue_name"; -const QUEUE_SPLITTER_RESOURCE_NAME: &str = "mirrord-e2e-test-queue-splitter"; - -#[fixture] -fn sqs_queue_name() -> String { - format!( - "MirrordE2ESplitterTests-{}.fifo", - crate::utils::random_string() - ) -} - -/// Create a new SQS fifo queue with a randomized name, return queue name and url. -#[fixture] -async fn sqs_queue(sqs_queue_name: String) -> (String, String) { - let shared_config = aws_config::load_from_env().await; - let client = aws_sdk_sqs::Client::new(&shared_config); - let queue = client - .create_queue() - .queue_name(sqs_queue_name.clone()) - .attributes(MessageRetentionPeriod, "3600") // delete messages after an hour. - .attributes(FifoQueue, "true") // Fifo for predictable test scenarios - .send() - .await - .unwrap(); - (sqs_queue_name, queue.queue_url.unwrap()) -} - -pub async fn create_config_map( - kube_client: Client, - namespace: &str, - sqs_queue_name: String, -) -> ResourceGuard { - let config_map_api: Api = Api::namespaced(kube_client.clone(), namespace); - let config_map = ConfigMap { - binary_data: None, - data: Some(BTreeMap::from([( - SQS_CONFIG_MAP_KEY_NAME.to_string(), - sqs_queue_name, - )])), - immutable: None, - metadata: Default::default(), - }; - ResourceGuard::create( - config_map_api, - SQS_CONFIG_MAP_NAME.to_string(), - &config_map, - true, - ) - .await - .expect("Could not create config map in E2E test.") -} - -/// Create the `MirrordQueueSplitter` K8s resource and a resource guard to delete it when done. -pub async fn create_splitter_resource( - kube_client: Client, - namespace: &str, - deployment_name: &str, -) -> ResourceGuard { - let qs_api: Api = Api::namespaced(kube_client.clone(), namespace); - let queue_splitter = MirrordQueueSplitter::new( - QUEUE_SPLITTER_RESOURCE_NAME, - MirrordQueueSplitterSpec { - queues: HashMap::from([( - "e2e-test-queue".to_string(), - SplitQueue::Sqs(QueueNameSource::ConfigMap { - name: SQS_CONFIG_MAP_NAME.to_string(), - queue_name_key: SQS_CONFIG_MAP_KEY_NAME.to_string(), - sub_key: None, - }), - )]), - consumer: QueueConsumer::Deployment(deployment_name.to_string()), - }, - ); - ResourceGuard::create( - qs_api, - QUEUE_SPLITTER_RESOURCE_NAME.to_string(), - &queue_splitter, - true, - ) - .await - .expect("Could not create queue splitter in E2E test.") -} - -#[fixture] -pub async fn sqs_consumer_service(#[future] kube_client: Client) -> KubeService { - // By using a different namespace for each test run, we're able to use a constant ConfigMap - // name, and don't have to pass that information to the test service. - let namespace = format!("e2e-tests-sqs-splitting-{}", crate::utils::random_string()); - service_with_env( - &namespace, - "ClusterIP", - "ghcr.io/metalbear-co/mirrord-node-udp-logger:latest", // TODO - "queue-forwarder", - false, - kube_client, - serde_json::json!([ - // TODO - ]), - ) - .await -} - -/// Send 6 messages to the original queue, such that 2 will reach each mirrord run and 2 the -/// deployed app. -async fn write_sqs_messages(queue_url: &str) { - // TODO -} - -/// This test creates a new sqs_queue with a random name and credentials from env. -/// -/// Define a queue splitter for a deployment. Start two services that both consume from an SQS -/// queue, send some messages to the queue, verify that each of the applications running with -/// mirrord get exactly the messages they are supposed to, and that the deployed application gets -/// the rest. -#[rstest] -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -pub async fn two_users_one_queue( - config_dir: &std::path::PathBuf, - #[future] kube_client: Client, - #[future] sqs_consumer_service: KubeService, - #[future] sqs_queue: (String, String), -) { - let application = Application::Go21HTTP; // TODO - let kube_client = kube_client.await; - let (sqs_queue_name, sqs_queue_url) = sqs_queue.await; - - let deployed = sqs_consumer_service.await; - - // Create the config map that the remote deployment uses as the source of the queue's name. - let _config_map_guard = - create_config_map(kube_client.clone(), &deployed.namespace, sqs_queue_name); - - // Create the `MirrordQueueSplitter` to be used in the client configurations. - let _splitter_resource_guard = - create_splitter_resource(kube_client.clone(), &deployed.namespace, &deployed.name).await; - - let mut config_path = config_dir.clone(); - config_path.push("sqs_queue_splitting_a.json"); - - let mut client_a = application - .run( - &deployed.target, // TODO: target the deployment maybe? - Some(&deployed.namespace), - None, - Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), - ) - .await; - - let mut config_path = config_dir.clone(); - config_path.push("sqs_queue_splitting_b.json"); - - let mut client_b = application - .run( - &deployed.target, // TODO: target the deployment maybe? - Some(&deployed.namespace), - None, - Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), - ) - .await; - - write_sqs_messages(&sqs_queue_url).await; - - // The test application consumes messages and verifies exact expected messages. - join!( - client_a.wait_assert_success(), - client_b.wait_assert_success() - ); - - // TODO: read output queue and verify that exactly the expected messages were - // consumed and forwarded. -} diff --git a/tests/src/utils.rs b/tests/src/utils.rs index eebd701cf07..3fc1228b873 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -30,7 +30,7 @@ use rand::{distributions::Alphanumeric, Rng}; use reqwest::{RequestBuilder, StatusCode}; use rstest::*; use serde::{de::DeserializeOwned, Serialize}; -use serde_json::{json, Value}; +use serde_json::json; use tempfile::{tempdir, TempDir}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt, BufReader}, @@ -672,25 +672,6 @@ impl Drop for KubeService { } } -fn default_env() -> Value { - json!( - [ - { - "name": "MIRRORD_FAKE_VAR_FIRST", - "value": "mirrord.is.running" - }, - { - "name": "MIRRORD_FAKE_VAR_SECOND", - "value": "7777" - }, - { - "name": "MIRRORD_FAKE_VAR_THIRD", - "value": "foo=bar" - } - ] - ) -} - /// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted /// when the returned service is dropped, unless it is dropped during panic. /// This behavior can be changed, see `FORCE_CLEANUP_ENV_NAME`. @@ -704,62 +685,6 @@ pub async fn service( #[default("http-echo")] service_name: &str, #[default(true)] randomize_name: bool, #[future] kube_client: Client, -) -> KubeService { - internal_service( - namespace, - service_type, - image, - service_name, - randomize_name, - kube_client.await, - default_env(), - ) -} - -/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted -/// when the returned service is dropped, unless it is dropped during panic. -/// This behavior can be changed, see [`FORCE_CLEANUP_ENV_NAME`]. -/// * `randomize_name` - whether a random suffix should be added to the end of the resource names -/// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. -pub async fn service_with_env( - namespace: &str, - service_type: &str, - image: &str, - service_name: &str, - randomize_name: bool, - kube_client: Client, - env: Value, -) -> KubeService { - internal_service( - namespace, - service_type, - image, - service_name, - randomize_name, - kube_client.await, - env, - ) -} - -/// Internal function to create a custom [`KubeService`]. -/// We keep this private so that whenever we need more customization of test resources, we can -/// change this function and how the public ones use it, and add a new public function that exposes -/// more customization, and we don't need to change all existing usages of public functions/fixtures -/// in tests. -/// -/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be -/// deleted when the returned service is dropped, unless it is dropped during panic. -/// This behavior can be changed, see [`FORCE_CLEANUP_ENV_NAME`]. -/// * `randomize_name` - whether a random suffix should be added to the end of the resource names -/// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. -async fn internal_service( - namespace: &str, - service_type: &str, - image: &str, - service_name: &str, - randomize_name: bool, - kube_client: Client, - env: Value, ) -> KubeService { let delete_after_fail = std::env::var_os(PRESERVE_FAILED_ENV_NAME).is_none(); @@ -850,7 +775,20 @@ async fn internal_service( "containerPort": 80 } ], - "env": env, + "env": [ + { + "name": "MIRRORD_FAKE_VAR_FIRST", + "value": "mirrord.is.running" + }, + { + "name": "MIRRORD_FAKE_VAR_SECOND", + "value": "7777" + }, + { + "name": "MIRRORD_FAKE_VAR_THIRD", + "value": "foo=bar" + } + ], } ] } From 192a749bece06f66a2df7a697ea4317e6753f743 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 26 Jun 2024 21:36:31 +0200 Subject: [PATCH 26/83] docs and improvements --- Cargo.lock | 363 --------------------- mirrord/config/src/feature.rs | 6 +- mirrord/config/src/feature/split_queues.rs | 8 +- mirrord/operator/src/client.rs | 6 +- mirrord/operator/src/crd.rs | 1 - 5 files changed, 13 insertions(+), 371 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef573ad233d..a371188366a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,49 +617,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "aws-config" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac9889352d632214df943e26740c46a0f3da6e329fbd28164fe7ae1b061da7b" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-sdk-sso", - "aws-sdk-ssooidc", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "hex", - "http 0.2.12", - "hyper 0.14.29", - "ring 0.17.8", - "time", - "tokio", - "tracing", - "url", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" -dependencies = [ - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "zeroize", -] - [[package]] name = "aws-lc-rs" version = "1.7.3" @@ -687,284 +644,6 @@ dependencies = [ "paste", ] -[[package]] -name = "aws-runtime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4a5e448145999d7de17bf44a886900ecb834953408dae8aaf90465ce91c1dd" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand", - "http 0.2.12", - "http-body 0.4.6", - "percent-encoding", - "pin-project-lite", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-sqs" -version = "1.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f3fb8da46554d08e63272e56495f7c94908c16dc62d3c7cc8a0fb4d7591726a" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sso" -version = "1.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da75cf91cbb46686a27436d639a720a3a198b148efa76dc2467b7e5374a67fc0" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-ssooidc" -version = "1.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2ec8a6687299685ed0a4a3137c129cdb132b5235bc3aa3443f6cffe468b9ff" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "1.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f1031e094b1411b59b49b19e4118f069e1fe13a9c5b8888e933daaf7ffdd6" -dependencies = [ - "aws-credential-types", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "http 0.2.12", - "once_cell", - "regex-lite", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31eed8d45759b2c5fe7fd304dd70739060e9e0de509209036eabea14d0720cce" -dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "form_urlencoded", - "hex", - "hmac", - "http 0.2.12", - "http 1.1.0", - "once_cell", - "percent-encoding", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "aws-smithy-http" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a7de001a1b9a25601016d8057ea16e31a45fdca3751304c8edf4ad72e706c08" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http-body 0.4.6", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-runtime" -version = "1.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d3965f6417a92a6d1009c5958a67042f57e46342afb37ca58f9ad26744ec73" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "http-body 1.0.0", - "hyper 0.14.29", - "hyper-rustls 0.24.2", - "once_cell", - "pin-project-lite", - "pin-utils", - "rustls 0.21.12", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b570ea39eb95bd32543f6e4032bce172cb6209b9bc8c83c770d08169e875afc" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "bytes", - "http 0.2.12", - "http 1.1.0", - "pin-project-lite", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-smithy-types" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5" -dependencies = [ - "base64-simd", - "bytes", - "bytes-utils", - "futures-core", - "http 0.2.12", - "http 1.1.0", - "http-body 0.4.6", - "http-body 1.0.0", - "http-body-util", - "itoa", - "num-integer", - "pin-project-lite", - "pin-utils", - "ryu", - "serde", - "time", - "tokio", - "tokio-util", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.60.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f734808d43702a67e57d478a12e227d4d038d0b90c9005a78c87890d3805922" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-runtime-api", - "aws-smithy-types", - "http 0.2.12", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.6.20" @@ -1121,16 +800,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] - [[package]] name = "base64ct" version = "1.6.0" @@ -1381,16 +1050,6 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "bytesize" version = "1.3.0" @@ -3143,9 +2802,7 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.29", - "log", "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", "tokio-rustls 0.24.1", ] @@ -4887,12 +4544,6 @@ dependencies = [ name = "outgoing" version = "3.107.0" -[[package]] -name = "outref" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" - [[package]] name = "overload" version = "0.1.1" @@ -6906,8 +6557,6 @@ checksum = "8e7a7de15468c6e65dd7db81cf3822c1ec94c71b2a3c1a976ea8e4696c91115c" name = "tests" version = "0.1.0" dependencies = [ - "aws-config", - "aws-sdk-sqs", "bytes", "chrono", "const_format", @@ -7695,12 +7344,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - [[package]] name = "walkdir" version = "2.5.0" @@ -8201,12 +7844,6 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "xz" version = "0.1.0" diff --git a/mirrord/config/src/feature.rs b/mirrord/config/src/feature.rs index 1cd9c172d8a..ba3c097acf7 100644 --- a/mirrord/config/src/feature.rs +++ b/mirrord/config/src/feature.rs @@ -3,8 +3,7 @@ use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; use self::{copy_target::CopyTargetConfig, env::EnvConfig, fs::FsConfig, network::NetworkConfig}; -use crate::config::source::MirrordConfigSource; -use crate::feature::split_queues::SplitQueuesConfig; +use crate::{config::source::MirrordConfigSource, feature::split_queues::SplitQueuesConfig}; pub mod copy_target; pub mod env; @@ -102,6 +101,9 @@ pub struct FeatureConfig { /// /// Define filters to split queues by, and make your local application consume only messages /// that match those filters. + /// If you don't specify any filter for a queue that is however declared in the + /// `MirrordWorkloadQueueRegistry` of the target you're using, a match-nothing filter + /// will be used, and your local application will not receive any messages from that queue. #[config(nested)] pub split_queues: SplitQueuesConfig, } diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 1583938a0d7..30d32416d4d 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -20,10 +20,9 @@ pub type QueueId = String; /// } /// }, /// "second-queue": { -/// "queue_type": "SomeFutureQueueType", +/// "queue_type": "SQS", /// "message_filter": { -/// "wows": "so wows", -/// "coolz": "^very .*" +/// "who": "*you$" /// } /// }, /// } @@ -47,6 +46,8 @@ impl SplitQueuesConfig { QueueFilter::Sqs(filter_mapping) => { Some((queue_id.clone(), filter_mapping.clone())) } + #[allow(unreachable_patterns)] // will become reachable when we add variants + _ => None, }) .collect() }) @@ -76,6 +77,7 @@ pub type SqsMessageFilter = BTreeMap Date: Wed, 26 Jun 2024 21:48:12 +0200 Subject: [PATCH 27/83] cargo fmt --- mirrord/cli/src/operator/session.rs | 3 +-- mirrord/operator/src/setup.rs | 22 ++++++---------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/mirrord/cli/src/operator/session.rs b/mirrord/cli/src/operator/session.rs index e08826d50ad..dce90391c61 100644 --- a/mirrord/cli/src/operator/session.rs +++ b/mirrord/cli/src/operator/session.rs @@ -1,9 +1,8 @@ use kube::{core::ErrorResponse, Api}; use mirrord_operator::{ client::{session_api, OperatorApiError, OperatorOperation}, - crd::{MirrordOperatorCrd, SessionCrd, OPERATOR_STATUS_NAME}, + crd::{MirrordOperatorCrd, NewOperatorFeature, SessionCrd, OPERATOR_STATUS_NAME}, }; -use mirrord_operator::crd::NewOperatorFeature; use mirrord_progress::{Progress, ProgressTracker}; use super::get_status_api; diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 5c27652a6de..c7cac6470fb 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -23,7 +23,7 @@ use k8s_openapi::{ use kube::{CustomResourceExt, Resource}; use thiserror::Error; -use crate::crd::{MirrordPolicy, MirrordWorkloadQueueRegistry, MirrordSqsSession, TargetCrd}; +use crate::crd::{MirrordPolicy, MirrordSqsSession, MirrordWorkloadQueueRegistry, TargetCrd}; static OPERATOR_NAME: &str = "mirrord-operator"; static OPERATOR_PORT: i32 = 3000; @@ -423,23 +423,15 @@ impl OperatorRole { }, // For SQS controller to temporarily change deployments to use changed queues. PolicyRule { - api_groups: Some(vec![ - "apps".to_owned(), - ]), - resources: Some(vec![ - "deployments".to_owned(), - ]), + api_groups: Some(vec!["apps".to_owned()]), + resources: Some(vec!["deployments".to_owned()]), verbs: vec!["patch".to_owned()], ..Default::default() }, // For SQS controller to temporarily change Argo Rollouts to use changed queues. PolicyRule { - api_groups: Some(vec![ - "argoproj.io".to_owned(), - ]), - resources: Some(vec![ - "rollouts".to_owned(), - ]), + api_groups: Some(vec!["argoproj.io".to_owned()]), + resources: Some(vec!["rollouts".to_owned()]), verbs: vec!["patch".to_owned()], ..Default::default() }, @@ -495,9 +487,7 @@ impl OperatorRole { PolicyRule { api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec![MirrordWorkloadQueueRegistry::plural(&()).to_string()]), - verbs: vec![ - "list".to_owned(), - ], + verbs: vec!["list".to_owned()], ..Default::default() }, // Allow the SQS controller to update queue splitter status. From 8e3f08daa5025c022a775b1af279a59dce6a692a Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 26 Jun 2024 22:05:29 +0200 Subject: [PATCH 28/83] mirrord-schema.json --- mirrord-schema.json | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/mirrord-schema.json b/mirrord-schema.json index 5897114ddeb..04d3d2ae436 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -691,6 +691,18 @@ "type": "null" } ] + }, + "split_queues": { + "title": "feature.split_queues {#feature-split_queues}", + "description": "Define filters to split queues by, and make your local application consume only messages that match those filters. If you don't specify any filter for a queue that is however declared in the `MirrordWorkloadQueueRegistry` of the target you're using, a match-nothing filter will be used, and your local application will not receive any messages from that queue.", + "anyOf": [ + { + "$ref": "#/definitions/SplitQueuesConfig" + }, + { + "type": "null" + } + ] } }, "additionalProperties": false @@ -1209,6 +1221,32 @@ } ] }, + "QueueFilter": { + "description": "More queue types might be added in the future.", + "oneOf": [ + { + "type": "object", + "required": [ + "message_filter", + "queue_type" + ], + "properties": { + "message_filter": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "queue_type": { + "type": "string", + "enum": [ + "SQS" + ] + } + } + } + ] + }, "RolloutTarget": { "description": " Mirror the rollout specified by [`RolloutTarget::rollout`].", "type": "object", @@ -1229,6 +1267,16 @@ }, "additionalProperties": false }, + "SplitQueuesConfig": { + "description": "```json { \"feature\": { \"split_queues\": { \"first-queue\": { \"queue_type\": \"SQS\", \"message_filter\": { \"wows\": \"so wows\", \"coolz\": \"^very .*\" } }, \"second-queue\": { \"queue_type\": \"SQS\", \"message_filter\": { \"who\": \"*you$\" } }, } } } ```", + "type": [ + "object", + "null" + ], + "additionalProperties": { + "$ref": "#/definitions/QueueFilter" + } + }, "StatefulSetTarget": { "type": "object", "required": [ From cfd608e86e15a583ab637311f39221aaed0858f4 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 27 Jun 2024 22:20:45 +0200 Subject: [PATCH 29/83] run medschool --- mirrord/config/configuration.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md index 2d818882baa..3d2fb4d6b16 100644 --- a/mirrord/config/configuration.md +++ b/mirrord/config/configuration.md @@ -1022,6 +1022,36 @@ of regexes specified here. If there is a match, mirrord will connect your applic the target unix socket address on the target pod. Otherwise, it will leave the connection to happen locally on your machine. +## feature.split_queues {#feature-split_queues} + +Define filters to split queues by, and make your local application consume only messages +that match those filters. +If you don't specify any filter for a queue that is however declared in the +`MirrordWorkloadQueueRegistry` of the target you're using, a match-nothing filter +will be used, and your local application will not receive any messages from that queue. + +```json +{ + "feature": { + "split_queues": { + "first-queue": { + "queue_type": "SQS", + "message_filter": { + "wows": "so wows", + "coolz": "^very .*" + } + }, + "second-queue": { + "queue_type": "SQS", + "message_filter": { + "who": "*you$" + } + }, + } + } +} +``` + # internal_proxy {#root-internal_proxy} Configuration for the internal proxy mirrord spawns for each local mirrord session From cdd691c307cf5f46bb7a0a9bdc6d486a87e101c3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 4 Jul 2024 17:40:34 +0200 Subject: [PATCH 30/83] fmt --- mirrord/cli/src/operator/session.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/cli/src/operator/session.rs b/mirrord/cli/src/operator/session.rs index ccebc80f699..8f02415bea5 100644 --- a/mirrord/cli/src/operator/session.rs +++ b/mirrord/cli/src/operator/session.rs @@ -6,7 +6,7 @@ use mirrord_operator::{ error::{OperatorApiError, OperatorOperation}, MaybeClientCert, OperatorApi, }, - crd::{SessionCrd, NewOperatorFeature}, + crd::{NewOperatorFeature, SessionCrd}, }; use mirrord_progress::{Progress, ProgressTracker}; From c8884c9aae8d3c7e9b6a529a36b6a0929b9c3dba Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 9 Jul 2024 11:25:45 +0300 Subject: [PATCH 31/83] oops --- mirrord/operator/src/client.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 694c6e17b5c..98721ff1519 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -644,7 +644,10 @@ impl OperatorApi { { self.check_feature_support(config)?; - let target = if config.feature.copy_target.enabled { + let target = if config.feature.copy_target.enabled + // use copy_target for splitting queues + || config.feature.split_queues.is_set() + { let mut copy_subtask = progress.subtask("copying target"); // We do not validate the `target` here, it's up to the operator. From ed3d13f91f5aa8e470cdb3e03ce6b094e52e03d8 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 11 Jul 2024 11:06:45 +0300 Subject: [PATCH 32/83] merge --- mirrord/config/src/target.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs index c3a4f71f8fe..181d832212f 100644 --- a/mirrord/config/src/target.rs +++ b/mirrord/config/src/target.rs @@ -298,12 +298,12 @@ impl Target { pub fn get_target_type(&self) -> &str { match self { Target::Targetless => "targetless", - Target::Pod(pod) => pod.target_type(), - Target::Deployment(dep) => dep.target_type(), - Target::Rollout(roll) => roll.target_type(), - Target::Job(job) => job.target_type(), - Target::CronJob(cron_job) => cron_job.target_type(), - Target::StatefulSet(stateful_set) => stateful_set.target_type(), + Target::Pod(pod) => pod.type_(), + Target::Deployment(dep) => dep.type_(), + Target::Rollout(roll) => roll.type_(), + Target::Job(job) => job.type_(), + Target::CronJob(cron_job) => cron_job.type_(), + Target::StatefulSet(stateful_set) => stateful_set.type_(), } } From 20ff89deff95543e17a2fa151a506e8accdf4c3a Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 15 Jul 2024 11:33:47 +0300 Subject: [PATCH 33/83] display Q consumer --- mirrord/operator/src/crd.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 6a85cfc169c..926ae87cdfe 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -4,12 +4,12 @@ use std::{ }; use kube::CustomResource; +use kube_target::{KubeTarget, UnknownTargetType}; pub use mirrord_config::feature::split_queues::QueueId; use mirrord_config::{ feature::split_queues::SqsMessageFilter, target::{Target, TargetConfig}, }; -use kube_target::{KubeTarget, UnknownTargetType}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -395,6 +395,13 @@ impl QueueConsumer { } } +impl Display for QueueConsumer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let (type_, name) = self.get_type_and_name(); + write!(f, "{}/{}", type_, name) + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] pub struct QueueNameUpdate { pub original_name: String, From 88f87cb8edb9d15dc98031086340e310dd0ea383 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 15 Jul 2024 11:45:59 +0300 Subject: [PATCH 34/83] fix merge --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 926ae87cdfe..d9a0152ccd3 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -14,7 +14,7 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use self::label_selector::LabelSelector; -use crate::types::LicenseInfoOwned; +use crate::{client::error::OperatorApiError, types::LicenseInfoOwned}; pub mod kube_target; pub mod label_selector; From 84c05e47ee4dfc907a584c0f44e17368939a026a Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 15 Jul 2024 13:18:10 +0300 Subject: [PATCH 35/83] fix feature use --- mirrord/operator/src/crd.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index d9a0152ccd3..ecea3a0ecbd 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -14,7 +14,9 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use self::label_selector::LabelSelector; -use crate::{client::error::OperatorApiError, types::LicenseInfoOwned}; +#[cfg(feature = "client")] +use crate::client::error::OperatorApiError; +use crate::types::LicenseInfoOwned; pub mod kube_target; pub mod label_selector; From 4ea228221388b13cd70d2e887263cc52b01119e2 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 18 Jul 2024 16:05:00 +0300 Subject: [PATCH 36/83] rename sqs details and make optional --- mirrord/operator/src/crd.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index ecea3a0ecbd..46b102cca37 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -439,16 +439,23 @@ pub struct QueueDetails { #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // active_filters -> activeFilters pub struct WorkloadQueueRegistryStatus { - pub queue_details: QueueDetails, + /// Optional even though it's currently the only field, because in the future there will be + /// fields for other queue types. + pub sqs_details: Option, } impl WorkloadQueueRegistryStatus { pub fn output_queue_names(&self) -> Vec<&str> { - self.queue_details - .queue_names - .values() - .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) - .collect() + self.sqs_details + .as_ref() + .map(|details| { + details + .queue_names + .values() + .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) + .collect() + }) + .unwrap_or_default() } } From 4f003e5c20ee0936952a09260ae0dc3b6dc6ef0c Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 18 Jul 2024 16:32:36 +0300 Subject: [PATCH 37/83] rename QueueDetails -> ActiveSqsSplits --- mirrord/operator/src/crd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 46b102cca37..86d588c17bd 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -417,7 +417,7 @@ pub struct QueueNameUpdate { // controller's code a bit simpler. // Some information is present in the spec, but it is organized differently. #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] -pub struct QueueDetails { +pub struct ActiveSqsSplits { /// For each queue_id, the actual queue name as retrieved from the target's pod spec or config /// map, together with the name of its temporary output queue. pub queue_names: BTreeMap, @@ -441,7 +441,7 @@ pub struct QueueDetails { pub struct WorkloadQueueRegistryStatus { /// Optional even though it's currently the only field, because in the future there will be /// fields for other queue types. - pub sqs_details: Option, + pub sqs_details: Option, } impl WorkloadQueueRegistryStatus { From 770e22590d6dc71a5266ee9d15075bfd54e8be15 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 18 Jul 2024 16:37:48 +0300 Subject: [PATCH 38/83] move output_q_names to ActiveSqsSplits --- mirrord/operator/src/crd.rs | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 86d588c17bd..8ec29380eb1 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -436,6 +436,15 @@ pub struct ActiveSqsSplits { pub env_updates: BTreeMap, } +impl ActiveSqsSplits { + pub fn output_queue_names(&self) -> Vec<&str> { + self.queue_names + .values() + .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) + .collect() + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // active_filters -> activeFilters pub struct WorkloadQueueRegistryStatus { @@ -444,21 +453,6 @@ pub struct WorkloadQueueRegistryStatus { pub sqs_details: Option, } -impl WorkloadQueueRegistryStatus { - pub fn output_queue_names(&self) -> Vec<&str> { - self.sqs_details - .as_ref() - .map(|details| { - details - .queue_names - .values() - .map(|QueueNameUpdate { output_name, .. }| output_name.as_str()) - .collect() - }) - .unwrap_or_default() - } -} - /// Defines a Custom Resource that holds a central configuration for splitting a queue. mirrord /// users specify a splitter by name in their configuration. mirrord then starts splitting according /// to the spec and the user's filter. From ce09b3aeaf52a5aeb9d6355997db1a05651934b6 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 19 Jul 2024 00:49:35 +0300 Subject: [PATCH 39/83] fmt --- mirrord/operator/src/client/error.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mirrord/operator/src/client/error.rs b/mirrord/operator/src/client/error.rs index 505307fc3b5..1f08661542f 100644 --- a/mirrord/operator/src/client/error.rs +++ b/mirrord/operator/src/client/error.rs @@ -4,9 +4,7 @@ pub use http::Error as HttpError; use mirrord_kube::error::KubeApiError; use thiserror::Error; -use crate::crd::NewOperatorFeature; - -use crate::crd::kube_target::UnknownTargetType; +use crate::crd::{kube_target::UnknownTargetType, NewOperatorFeature}; /// Operations performed on the operator via [`kube`] API. #[derive(Debug)] From 8eaeebddeb2de512fd1d797ce7ad939293748c79 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 26 Jul 2024 18:57:04 +0200 Subject: [PATCH 40/83] CR: lambda var name --- mirrord/config/src/feature/split_queues.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 30d32416d4d..8cf2b6284da 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -39,8 +39,8 @@ impl SplitQueuesConfig { /// Out of the whole queue splitting config, get only the sqs queues. pub fn get_sqs_filter(&self) -> Option> { - self.0.as_ref().map(|queue_id2queue_filter| { - queue_id2queue_filter + self.0.as_ref().map(|filters| { + filters .iter() .filter_map(|(queue_id, queue_filter)| match queue_filter { QueueFilter::Sqs(filter_mapping) => { From b621750f2a16649e063e8307add30f2e32b40579 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 26 Jul 2024 19:01:59 +0200 Subject: [PATCH 41/83] move method call to own .map call --- mirrord/config/src/feature/split_queues.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 8cf2b6284da..3bb813d183b 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -39,9 +39,8 @@ impl SplitQueuesConfig { /// Out of the whole queue splitting config, get only the sqs queues. pub fn get_sqs_filter(&self) -> Option> { - self.0.as_ref().map(|filters| { + self.0.as_ref().map(BTreeMap::iter).map(|filters| { filters - .iter() .filter_map(|(queue_id, queue_filter)| match queue_filter { QueueFilter::Sqs(filter_mapping) => { Some((queue_id.clone(), filter_mapping.clone())) From e88bfc214ee9b121071f69110c0b02eeffd4f974 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 26 Jul 2024 19:10:38 +0200 Subject: [PATCH 42/83] CR: remove match arm for future variants. --- mirrord/config/src/feature/split_queues.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 3bb813d183b..28b2ce5c993 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -41,12 +41,9 @@ impl SplitQueuesConfig { pub fn get_sqs_filter(&self) -> Option> { self.0.as_ref().map(BTreeMap::iter).map(|filters| { filters - .filter_map(|(queue_id, queue_filter)| match queue_filter { - QueueFilter::Sqs(filter_mapping) => { - Some((queue_id.clone(), filter_mapping.clone())) - } - #[allow(unreachable_patterns)] // will become reachable when we add variants - _ => None, + // When there are more variants of QueueFilter, change this to a `filter_map`. + .map(|(queue_id, queue_filter)| match queue_filter { + QueueFilter::Sqs(filter_mapping) => (queue_id.clone(), filter_mapping.clone()), }) .collect() }) From 7551467cddd037c2150b385f0ea6c50b3f009bc3 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 26 Jul 2024 19:37:48 +0200 Subject: [PATCH 43/83] CR: document SqsMessageFilter --- mirrord/config/src/feature/split_queues.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 28b2ce5c993..5c1fc20d2c8 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -68,6 +68,10 @@ impl FromMirrordConfig for SplitQueuesConfig { pub type MessageAttributeName = String; pub type AttributeValuePattern = String; +/// A filter is a mapping between message attribute names and regexes they should match. +/// The local application will only receive messages that match ALL the patterns. +/// This means, only messages that have all the `MessageAttributeName`s in the filter, +/// with values of those attributes matching the respective `AttributeValuePattern`. pub type SqsMessageFilter = BTreeMap; /// More queue types might be added in the future. @@ -75,6 +79,7 @@ pub type SqsMessageFilter = BTreeMap Date: Fri, 26 Jul 2024 19:39:38 +0200 Subject: [PATCH 44/83] CR: remove get_target_type --- mirrord/config/src/target.rs | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs index 181d832212f..be58a7d0775 100644 --- a/mirrord/config/src/target.rs +++ b/mirrord/config/src/target.rs @@ -294,19 +294,6 @@ impl Target { } } - /// Get the target type - "pod", "deployment", "rollout" or "targetless" - pub fn get_target_type(&self) -> &str { - match self { - Target::Targetless => "targetless", - Target::Pod(pod) => pod.type_(), - Target::Deployment(dep) => dep.type_(), - Target::Rollout(roll) => roll.type_(), - Target::Job(job) => job.type_(), - Target::CronJob(cron_job) => cron_job.type_(), - Target::StatefulSet(stateful_set) => stateful_set.type_(), - } - } - /// `true` if this [`Target`] is only supported when the copy target feature is enabled. pub(super) fn requires_copy(&self) -> bool { matches!( From f642417c0b911e8216b8cfac5d84b4c9eff93bd8 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 30 Jul 2024 19:04:48 +0200 Subject: [PATCH 45/83] CR: Operator feature variant names. --- mirrord/operator/src/client.rs | 2 +- mirrord/operator/src/crd.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 571d4b3ef03..587507ad0eb 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -555,7 +555,7 @@ where if config.feature.split_queues.is_set() { self.operator .spec - .require_feature(NewOperatorFeature::Sqs)?; + .require_feature(NewOperatorFeature::SqsQueueSplitting)?; } Ok(()) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 8ec29380eb1..c8cfdb97764 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -253,12 +253,12 @@ pub enum OperatorFeatures { pub enum NewOperatorFeature { ProxyApi, CopyTarget, - Sqs, + SqsQueueSplitting, SessionManagement, /// This variant is what a client sees when the operator includes a feature the client is not /// yet aware of, because it was introduced in a version newer than the client's. #[serde(other)] - FeatureFromTheFuture, + Unknown, } impl Display for NewOperatorFeature { @@ -266,8 +266,8 @@ impl Display for NewOperatorFeature { let name = match self { NewOperatorFeature::ProxyApi => "proxy API", NewOperatorFeature::CopyTarget => "copy target", - NewOperatorFeature::Sqs => "SQS queue splitting", - NewOperatorFeature::FeatureFromTheFuture => "unknown feature", + NewOperatorFeature::SqsQueueSplitting => "SQS queue splitting", + NewOperatorFeature::Unknown => "unknown feature", NewOperatorFeature::SessionManagement => "session management", }; f.write_str(name) From 53a2188bdd290e08d63850f2d50fc72845f0d807 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 30 Jul 2024 19:22:39 +0200 Subject: [PATCH 46/83] CR: unused type --- mirrord/operator/src/crd.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index c8cfdb97764..5fa9a1a24b8 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -362,8 +362,6 @@ pub enum QueueNameSource { EnvVar(String), } -pub type OutputQueueName = String; - /// The details of a queue that should be split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(tag = "queueType")] From 9a56c858e6473323002649ee9719978a7f6de67d Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 31 Jul 2024 22:07:48 +0200 Subject: [PATCH 47/83] CR: docs: "pod spec" -> "pod template" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 5fa9a1a24b8..6d89dc8f70d 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -420,7 +420,7 @@ pub struct ActiveSqsSplits { /// map, together with the name of its temporary output queue. pub queue_names: BTreeMap, - /// Names of env vars that contain the queue name directly in the pod spec, without config + /// Names of env vars that contain the queue name directly in the pod template, without config /// map refs, mapped to their queue id. pub direct_env_vars: HashMap, From 87a7af1ea3e2a2b80b86e5b8abed8c155bd10912 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 2 Aug 2024 15:15:50 +0200 Subject: [PATCH 48/83] CR: remove dead configmaps code --- mirrord/operator/src/crd.rs | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 6d89dc8f70d..6e09467ae8b 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -342,23 +342,11 @@ pub struct MirrordPolicySpec { pub block: Vec, } -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] // queue_name_key -> queueNameKey in yaml. -pub struct ConfigMapQueueNameSource { - /// The name of the config map that holds the name of the queue we want to split. - pub name: String, - - /// The name of the key in the config map that holds the name of the queue we want to - /// split. - pub queue_name_key: String, -} - /// Set where the application reads the name of the queue from, so that mirrord can find that queue, /// split it, and temporarily change the name there to the name of the branch queue when splitting. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] // ConfigMap -> configMap in yaml. +#[serde(rename_all = "camelCase")] // EnvVar -> envVar in yaml. pub enum QueueNameSource { - ConfigMap(ConfigMapQueueNameSource), EnvVar(String), } @@ -424,13 +412,6 @@ pub struct ActiveSqsSplits { /// map refs, mapped to their queue id. pub direct_env_vars: HashMap, - /// For each config map name, a mapping from queue id to key name in the map that holds the - /// name of that queue. - pub config_map_updates: BTreeMap>, - // ^ ^ ^ - // | | ---- name of key that points to Q name - // | ---- queue id - // ---- ConfigMap name pub env_updates: BTreeMap, } From 643bf2357df4d68237a2132c47c9a12882286648 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 5 Aug 2024 22:14:56 +0200 Subject: [PATCH 49/83] CR: make session status enum --- mirrord/operator/src/crd.rs | 47 ++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 6e09467ae8b..bc96bb58e74 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -459,25 +459,50 @@ pub struct MirrordWorkloadQueueRegistrySpec { pub tags: Option>, } +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub enum SqsSessionStartError { + /// SQS Splitter could not split queues with the given config. + SplitterError(String), + #[serde(other)] + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub enum SqsSessionCleanupError { + #[serde(other)] + Unknown, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] -pub struct SqsSessionStatus { - /// Queue ID -> old and new queue names. - pub queue_names: BTreeMap, - - // A bit redundant, because the registry resource status has the mapping from env var name - // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving - // it here in the form that is useful to reader, for simplicity and readability. - /// Env var name -> old and new queue names. - pub env_updates: BTreeMap, - // TODO: add updates to config map references. +pub enum SqsSessionStatus { + #[default] + Starting, + Ready { + /// Queue ID -> old and new queue names. + queue_names: BTreeMap, + + // A bit redundant, because the registry resource status has the mapping from env var name + // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving + // it here in the form that is useful to reader, for simplicity and readability. + /// Env var name -> old and new queue names. + env_updates: BTreeMap, + }, + StartError(SqsSessionStartError), + CleanupError(SqsSessionCleanupError), } /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. /// To be used in [`kube::runtime::wait::await_condition`]. pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { session - .map(|session| session.status.is_some()) + .and_then(|session| session.status.as_ref()) + .map(|status| { + matches!( + status, + SqsSessionStatus::Ready { .. } | SqsSessionStatus::StartError(..) + ) + }) .unwrap_or_default() } From cc330891d5ea7270d8b4b2edeb76d6f065ee5cbb Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 00:19:08 +0200 Subject: [PATCH 50/83] CR: remove non_exauhstive --- mirrord/operator/src/crd.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index bc96bb58e74..963778f97a3 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -353,9 +353,6 @@ pub enum QueueNameSource { /// The details of a queue that should be split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(tag = "queueType")] -// So that controllers that only handle 1 type of queue don't have to adapt when we add more queue -// types. -#[non_exhaustive] pub enum SplitQueue { /// Amazon SQS /// From ec5809e8e7e26442d8d9f74d188d1ac7fd688529 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 02:38:55 +0200 Subject: [PATCH 51/83] status enum stuff --- mirrord/operator/src/crd.rs | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 963778f97a3..182cfc7ae50 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -1,6 +1,7 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::{Display, Formatter}, + ops::Not, }; use kube::CustomResource; @@ -460,6 +461,7 @@ pub struct MirrordWorkloadQueueRegistrySpec { pub enum SqsSessionStartError { /// SQS Splitter could not split queues with the given config. SplitterError(String), + InvalidTarget, #[serde(other)] Unknown, } @@ -470,23 +472,27 @@ pub enum SqsSessionCleanupError { Unknown, } +#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] +#[serde(rename = "SQSSplitDetails", rename_all = "camelCase")] +pub struct SqsSplitDetails { + /// Queue ID -> old and new queue names. + queue_names: BTreeMap, + + // A bit redundant, because the registry resource status has the mapping from env var name + // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving + // it here in the form that is useful to reader, for simplicity and readability. + /// Env var name -> old and new queue names. + env_updates: BTreeMap, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] pub enum SqsSessionStatus { #[default] Starting, - Ready { - /// Queue ID -> old and new queue names. - queue_names: BTreeMap, - - // A bit redundant, because the registry resource status has the mapping from env var name - // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving - // it here in the form that is useful to reader, for simplicity and readability. - /// Env var name -> old and new queue names. - env_updates: BTreeMap, - }, + Ready(SqsSplitDetails), StartError(SqsSessionStartError), - CleanupError(SqsSessionCleanupError), + CleanupError(SqsSessionCleanupError, Option), } /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. @@ -494,12 +500,7 @@ pub enum SqsSessionStatus { pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { session .and_then(|session| session.status.as_ref()) - .map(|status| { - matches!( - status, - SqsSessionStatus::Ready { .. } | SqsSessionStatus::StartError(..) - ) - }) + .map(|status| matches!(status, SqsSessionStatus::Starting).not()) .unwrap_or_default() } From 05a4e2b28889622cc66501763453920a98a17f22 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 22:52:08 +0200 Subject: [PATCH 52/83] CR: sqs session error - code + reason --- mirrord/operator/src/crd.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 182cfc7ae50..088d510ded4 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -457,15 +457,6 @@ pub struct MirrordWorkloadQueueRegistrySpec { pub tags: Option>, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub enum SqsSessionStartError { - /// SQS Splitter could not split queues with the given config. - SplitterError(String), - InvalidTarget, - #[serde(other)] - Unknown, -} - #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub enum SqsSessionCleanupError { #[serde(other)] @@ -485,14 +476,23 @@ pub struct SqsSplitDetails { env_updates: BTreeMap, } -#[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] -#[serde(rename = "SQSSessionStatus", rename_all = "camelCase")] +/// Representation of Sqs errors for the status of SQS session resources. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct SqsSessionError { + status_code: u16, + + /// Human-readable explanation of what went wrong. + reason: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[serde(rename = "SQSSessionStatus")] pub enum SqsSessionStatus { - #[default] - Starting, + Starting(), Ready(SqsSplitDetails), - StartError(SqsSessionStartError), - CleanupError(SqsSessionCleanupError, Option), + StartError(SqsSessionError), + CleanupError(SqsSessionError, Option), } /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. @@ -500,7 +500,7 @@ pub enum SqsSessionStatus { pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { session .and_then(|session| session.status.as_ref()) - .map(|status| matches!(status, SqsSessionStatus::Starting).not()) + .map(|status| matches!(status, SqsSessionStatus::Starting(..)).not()) .unwrap_or_default() } From a0118d1af2bfc0d642ff359613c9dfae41f3e38a Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 23:17:25 +0200 Subject: [PATCH 53/83] just need to push a commit --- mirrord/operator/src/crd.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 088d510ded4..a3bce9bd2d8 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -480,6 +480,7 @@ pub struct SqsSplitDetails { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SqsSessionError { + /// HTTP code for operator response. status_code: u16, /// Human-readable explanation of what went wrong. From b9378d5bbcf27b1466bb346099e2848ccd059697 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 23:20:43 +0200 Subject: [PATCH 54/83] pub sqs details --- mirrord/operator/src/crd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index a3bce9bd2d8..a0d66046e97 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -467,13 +467,13 @@ pub enum SqsSessionCleanupError { #[serde(rename = "SQSSplitDetails", rename_all = "camelCase")] pub struct SqsSplitDetails { /// Queue ID -> old and new queue names. - queue_names: BTreeMap, + pub queue_names: BTreeMap, // A bit redundant, because the registry resource status has the mapping from env var name // to queue id, and `queue_names` has the mapping from queue id to name update, but, saving // it here in the form that is useful to reader, for simplicity and readability. /// Env var name -> old and new queue names. - env_updates: BTreeMap, + pub env_updates: BTreeMap, } /// Representation of Sqs errors for the status of SQS session resources. From 23d4e06cdc67e6c94d6b08e8727576b6ad0dc263 Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 23:22:07 +0200 Subject: [PATCH 55/83] pub sqs details --- mirrord/operator/src/crd.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index a0d66046e97..800eb489542 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -481,10 +481,10 @@ pub struct SqsSplitDetails { #[serde(rename_all = "camelCase")] pub struct SqsSessionError { /// HTTP code for operator response. - status_code: u16, + pub status_code: u16, /// Human-readable explanation of what went wrong. - reason: String, + pub reason: String, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] From d2f7cde7db546be43eab34caf69fb22bcbbda15e Mon Sep 17 00:00:00 2001 From: t4lz Date: Tue, 6 Aug 2024 23:31:16 +0200 Subject: [PATCH 56/83] impl Display for SqsSessionError --- mirrord/operator/src/crd.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 800eb489542..c39e361d3c4 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -487,6 +487,16 @@ pub struct SqsSessionError { pub reason: String, } +impl Display for SqsSessionError { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + // Write strictly the first element into the supplied output + // stream: `f`. Returns `fmt::Result` which indicates whether the + // operation succeeded or failed. Note that `write!` uses syntax which + // is very similar to `println!`. + write!(f, "{}", self.reason) + } +} + #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus")] pub enum SqsSessionStatus { From abe7f07f03fdcefe8126352f2dc993ef1456807f Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 02:42:27 +0200 Subject: [PATCH 57/83] schema --- mirrord-schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrord-schema.json b/mirrord-schema.json index cc2a964cb94..c7b526f1d47 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -1292,6 +1292,7 @@ "description": "More queue types might be added in the future.", "oneOf": [ { + "description": "Amazon Simple Queue Service.", "type": "object", "required": [ "message_filter", From 82d1254995aa90a85c5963d913949114c6057b25 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 10:25:22 +0200 Subject: [PATCH 58/83] CR: add container to queue consumer --- mirrord/operator/src/crd.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index c39e361d3c4..ef737badf9a 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -368,23 +368,27 @@ pub enum SplitQueue { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // Deployment -> deployment in yaml. pub enum QueueConsumer { - Deployment(String), - Rollout(String), + Deployment(String, Option), + Rollout(String, Option), } impl QueueConsumer { - pub fn get_type_and_name(&self) -> (&str, &str) { + pub fn get_type_name_container(&self) -> (&str, &str, Option<&str>) { match self { - QueueConsumer::Deployment(dep) => ("deployment", dep), - QueueConsumer::Rollout(roll) => ("rollout", roll), + QueueConsumer::Deployment(dep, container) => ("deployment", dep, container.as_deref()), + QueueConsumer::Rollout(roll, container) => ("rollout", roll, container.as_deref()), } } } impl Display for QueueConsumer { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let (type_, name) = self.get_type_and_name(); - write!(f, "{}/{}", type_, name) + let (type_, name, container) = self.get_type_name_container(); + if let Some(container) = container { + write!(f, "{}/{}/container/{container}", type_, name) + } else { + write!(f, "{}/{}", type_, name) + } } } From 3e6bbe9e0a873e21ff5e6d52c9e3cec20f9a085d Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 12:07:23 +0200 Subject: [PATCH 59/83] QueueConsumer registry matching --- mirrord/operator/src/crd.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index ef737badf9a..7f3e5687fb0 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -368,7 +368,14 @@ pub enum SplitQueue { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] // Deployment -> deployment in yaml. pub enum QueueConsumer { + /// The name of the deployment and an optional container. + /// If a container is not specified, the workload queue registry will apply to every run that + /// targets any of the workload's containers. Deployment(String, Option), + + /// The name of the rollout and an optional container. + /// If a container is not specified, the workload queue registry will apply to every run that + /// targets any of the workload's containers. Rollout(String, Option), } @@ -379,6 +386,28 @@ impl QueueConsumer { QueueConsumer::Rollout(roll, container) => ("rollout", roll, container.as_deref()), } } + + /// For self that is the queue consumer of a run, test if a given registry object is the correct + /// registry for this run. + pub fn registry_matches(&self, registry: &MirrordWorkloadQueueRegistry) -> bool { + match (self, ®istry.spec.consumer) { + ( + QueueConsumer::Deployment(name, container), + QueueConsumer::Deployment(registry_consumer_name, registry_consumer_container), + ) + | ( + QueueConsumer::Rollout(name, container), + QueueConsumer::Rollout(registry_consumer_name, registry_consumer_container), + ) => { + name == registry_consumer_name + && (container == registry_consumer_container + // If registry does not specify a container, it applies to all runs with + // this target, regardless of what container they are targeting. + || registry_consumer_container.is_none()) + } + _ => false, + } + } } impl Display for QueueConsumer { From 254a47fbe7513670188630f50cc0f2ab4e5df088 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 14:58:11 +0200 Subject: [PATCH 60/83] QueueConsumer container getter --- mirrord/operator/src/crd.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 7f3e5687fb0..4a2e379bef6 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -387,6 +387,14 @@ impl QueueConsumer { } } + pub fn container(&self) -> Option<&str> { + match self { + QueueConsumer::Deployment(.., container) | QueueConsumer::Rollout(.., container) => { + container.as_deref() + } + } + } + /// For self that is the queue consumer of a run, test if a given registry object is the correct /// registry for this run. pub fn registry_matches(&self, registry: &MirrordWorkloadQueueRegistry) -> bool { From 11d388ad4b3528b695caf502c7c5a3931bf51932 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 17:35:15 +0200 Subject: [PATCH 61/83] Add RegisteringFilters Status variant --- mirrord/operator/src/crd.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 4a2e379bef6..3e020e8509c 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -542,6 +542,10 @@ impl Display for SqsSessionError { #[serde(rename = "SQSSessionStatus")] pub enum SqsSessionStatus { Starting(), + /// SQS operator sets this status before it starts registering filters, so that if anything + /// fails during the registration of filters, we have all the queues we need to delete on + /// cleanup. + RegisteringFilters(SqsSplitDetails), Ready(SqsSplitDetails), StartError(SqsSessionError), CleanupError(SqsSessionError, Option), @@ -552,7 +556,14 @@ pub enum SqsSessionStatus { pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { session .and_then(|session| session.status.as_ref()) - .map(|status| matches!(status, SqsSessionStatus::Starting(..)).not()) + .map(|status| { + matches!( + status, + SqsSessionStatus::Ready(..) + | SqsSessionStatus::StartError(..) + | SqsSessionStatus::CleanupError(..) + ) + }) .unwrap_or_default() } From ce5c5e7060d4f87545640808cd7d7a0c5825ae71 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 17:41:48 +0200 Subject: [PATCH 62/83] get_split_details --- mirrord/operator/src/crd.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 3e020e8509c..67e0f8e24dc 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -551,6 +551,18 @@ pub enum SqsSessionStatus { CleanupError(SqsSessionError, Option), } +impl SqsSessionStatus { + pub fn get_split_details(&self) -> Option<&SqsSplitDetails> { + match self { + SqsSessionStatus::RegisteringFilters(details) | SqsSessionStatus::Ready(details) => { + Some(details) + } + SqsSessionStatus::CleanupError(.., details) => details.as_ref(), + _ => None, + } + } +} + /// The [`kube::runtime::wait::Condition`] trait is auto-implemented for this function. /// To be used in [`kube::runtime::wait::await_condition`]. pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { From 0ec511594224c3bfd9d2f2c7a4b5d5d665aa9177 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 18:05:02 +0200 Subject: [PATCH 63/83] CR: registry docs --- mirrord/operator/src/crd.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 67e0f8e24dc..878798bb2ac 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -471,9 +471,10 @@ pub struct WorkloadQueueRegistryStatus { pub sqs_details: Option, } -/// Defines a Custom Resource that holds a central configuration for splitting a queue. mirrord -/// users specify a splitter by name in their configuration. mirrord then starts splitting according -/// to the spec and the user's filter. +/// Defines a Custom Resource that holds a central configuration for splitting queues for a +/// QueueConsumer (a target workload for which queues should be split). +/// +/// This means there should be 1 such resource per queue splitting target. #[derive(CustomResource, Clone, Debug, Deserialize, Serialize, JsonSchema)] #[kube( group = "queues.mirrord.metalbear.co", From ad9b009b7e7eac2f0dd26d5b4524b1b904612c25 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 18:12:42 +0200 Subject: [PATCH 64/83] CR: doc text formatting suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> --- mirrord/config/src/feature/split_queues.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 5c1fc20d2c8..21207033c2a 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -69,8 +69,8 @@ pub type MessageAttributeName = String; pub type AttributeValuePattern = String; /// A filter is a mapping between message attribute names and regexes they should match. -/// The local application will only receive messages that match ALL the patterns. -/// This means, only messages that have all the `MessageAttributeName`s in the filter, +/// The local application will only receive messages that match **all** of the given patterns. +/// This means, only messages that have **all** the `MessageAttributeName`s in the filter, /// with values of those attributes matching the respective `AttributeValuePattern`. pub type SqsMessageFilter = BTreeMap; From 56606f588aa42fbc485f856985d36d2bdfc66a05 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 18:50:28 +0200 Subject: [PATCH 65/83] CR: individual tags per queue --- mirrord/operator/src/crd.rs | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 878798bb2ac..f319fe40a3a 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -1,7 +1,6 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::{Display, Formatter}, - ops::Not, }; use kube::CustomResource; @@ -351,17 +350,28 @@ pub enum QueueNameSource { EnvVar(String), } +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] // name_source -> nameSource in yaml. +pub struct SqsQueueDetails { + /// Where the application gets the queue name from. Will be used to read messages from that + /// queue and distribute them to the output queues. When running with mirrord and splitting + /// this queue, applications will get a modified name from that source. + name_source: QueueNameSource, + + /// These tags will be set for all temporary SQS queues created by mirrord for queues defined + /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective + /// original queue. In case of a collision, the temporary queue will get the value from the + /// tag passed in here. + pub tags: Option>, +} + /// The details of a queue that should be split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] #[serde(tag = "queueType")] pub enum SplitQueue { /// Amazon SQS - /// - /// Where the application gets the queue name from. Will be used to read messages from that - /// queue and distribute them to the output queues. When running with mirrord and splitting - /// this queue, applications will get a modified name from that source. #[serde(rename = "SQS")] - Sqs(QueueNameSource), + Sqs(SqsQueueDetails), } /// A workload that is a consumer of a queue that is being split. @@ -491,12 +501,6 @@ pub struct MirrordWorkloadQueueRegistrySpec { /// The resource (deployment or Argo rollout) that reads from the queues. pub consumer: QueueConsumer, - - /// These tags will be set for all temporary SQS queues created by mirrord for queues defined - /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective - /// original queue. In case of a collision, the temporary queue will get the value from the - /// tag passed in here. - pub tags: Option>, } #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] @@ -603,6 +607,6 @@ pub struct MirrordSqsSessionSpec { /// The id of the mirrord exec session, from the operator. // The Kubernetes API can't deal with 64 bit numbers (with most significant bit set) - // so we save that field as a string even though its source is a u64 + // so we save that field as a (HEX) string even though its source is a u64 pub session_id: String, } From a86fc3ec0dc583f976299839f68360778b6ce061 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 18:51:48 +0200 Subject: [PATCH 66/83] missing pub --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index f319fe40a3a..3ebfa1d8a20 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -356,7 +356,7 @@ pub struct SqsQueueDetails { /// Where the application gets the queue name from. Will be used to read messages from that /// queue and distribute them to the output queues. When running with mirrord and splitting /// this queue, applications will get a modified name from that source. - name_source: QueueNameSource, + pub name_source: QueueNameSource, /// These tags will be set for all temporary SQS queues created by mirrord for queues defined /// in this MirrordWorkloadQueueRegistry, alongside with the original tags of the respective From aae011f2048e58ac4d18af4b6b18448777980acd Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 19:56:46 +0200 Subject: [PATCH 67/83] mark unstable --- mirrord/config/src/feature.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/config/src/feature.rs b/mirrord/config/src/feature.rs index ba3c097acf7..65afd82b737 100644 --- a/mirrord/config/src/feature.rs +++ b/mirrord/config/src/feature.rs @@ -104,7 +104,7 @@ pub struct FeatureConfig { /// If you don't specify any filter for a queue that is however declared in the /// `MirrordWorkloadQueueRegistry` of the target you're using, a match-nothing filter /// will be used, and your local application will not receive any messages from that queue. - #[config(nested)] + #[config(nested, unstable)] pub split_queues: SplitQueuesConfig, } From 8dcccfadddf42575ddac7495fa28b377f7e77a83 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 20:09:50 +0200 Subject: [PATCH 68/83] CR: CopyTargetCrd, send queues config --- mirrord/config/src/feature/split_queues.rs | 2 +- mirrord/operator/src/client.rs | 20 ++++++++++++-------- mirrord/operator/src/crd.rs | 6 +++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 21207033c2a..a145121ade9 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -30,7 +30,7 @@ pub type QueueId = String; /// } /// ``` #[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Deserialize, Default)] -pub struct SplitQueuesConfig(Option>); +pub struct SplitQueuesConfig(pub Option>); impl SplitQueuesConfig { pub fn is_set(&self) -> bool { diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 587507ad0eb..395969cfe12 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BTreeMap, HashMap}, - fmt, -}; +use std::{collections::BTreeMap, fmt}; use base64::{engine::general_purpose, Engine}; use chrono::{DateTime, Utc}; @@ -18,7 +15,14 @@ use mirrord_auth::{ credential_store::{CredentialStoreSync, UserIdentity}, credentials::LicenseValidity, }; -use mirrord_config::{feature::network::incoming::ConcurrentSteal, target::Target, LayerConfig}; +use mirrord_config::{ + feature::{ + network::incoming::ConcurrentSteal, + split_queues::{QueueFilter, QueueId}, + }, + target::Target, + LayerConfig, +}; use mirrord_kube::{api::kubernetes::create_kube_config, error::KubeApiError}; use mirrord_progress::Progress; use mirrord_protocol::{ClientMessage, DaemonMessage}; @@ -663,7 +667,7 @@ impl OperatorApi { target, scale_down, namespace, - config.feature.split_queues.get_sqs_filter(), + config.feature.split_queues.0.clone(), ) .await?; @@ -730,7 +734,7 @@ impl OperatorApi { target: Target, scale_down: bool, namespace: &str, - sqs_filter: Option>>, + split_queues: Option>, ) -> OperatorApiResult { let name = TargetCrd::urlfied_name(&target); @@ -740,7 +744,7 @@ impl OperatorApi { target, idle_ttl: Some(Self::COPIED_POD_IDLE_TTL), scale_down, - sqs_filter, + split_queues, }, ); diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 3ebfa1d8a20..f0fe36f00d8 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -7,7 +7,7 @@ use kube::CustomResource; use kube_target::{KubeTarget, UnknownTargetType}; pub use mirrord_config::feature::split_queues::QueueId; use mirrord_config::{ - feature::split_queues::SqsMessageFilter, + feature::split_queues::{QueueFilter, SqsMessageFilter}, target::{Target, TargetConfig}, }; use schemars::JsonSchema; @@ -301,8 +301,8 @@ pub struct CopyTargetSpec { /// Should the operator scale down target deployment to 0 while this pod is alive. /// Ignored if [`Target`] is not [`Target::Deployment`]. pub scale_down: bool, - /// queue id -> (attribute name -> regex) - pub sqs_filter: Option>, + /// Split queues client side configuration. + pub split_queues: Option>, } /// Features and operations that can be blocked by a `MirrordPolicy`. From 571b5a22251b592cdafafd6eb27f1718d0b8cbfc Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 7 Aug 2024 21:27:11 +0200 Subject: [PATCH 69/83] non non exauhstive --- mirrord/config/src/feature/split_queues.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index a145121ade9..7eff61904ee 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -77,7 +77,6 @@ pub type SqsMessageFilter = BTreeMap Date: Wed, 7 Aug 2024 21:52:16 +0200 Subject: [PATCH 70/83] CR: CopyTargetCrd, send queues config --- mirrord/config/src/feature/split_queues.rs | 31 +++++++++++++++------- mirrord/operator/src/client.rs | 18 ++++--------- mirrord/operator/src/crd.rs | 4 +-- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 7eff61904ee..c28b68048fc 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -1,4 +1,7 @@ -use std::collections::{BTreeMap, HashMap}; +use std::{ + collections::{BTreeMap, HashMap}, + ops::Not, +}; use mirrord_analytics::{Analytics, CollectAnalytics}; use schemars::JsonSchema; @@ -29,7 +32,7 @@ pub type QueueId = String; /// } /// } /// ``` -#[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Deserialize, Default)] +#[derive(Clone, Debug, Eq, PartialEq, JsonSchema, Serialize, Deserialize, Default)] pub struct SplitQueuesConfig(pub Option>); impl SplitQueuesConfig { @@ -39,14 +42,22 @@ impl SplitQueuesConfig { /// Out of the whole queue splitting config, get only the sqs queues. pub fn get_sqs_filter(&self) -> Option> { - self.0.as_ref().map(BTreeMap::iter).map(|filters| { - filters - // When there are more variants of QueueFilter, change this to a `filter_map`. - .map(|(queue_id, queue_filter)| match queue_filter { - QueueFilter::Sqs(filter_mapping) => (queue_id.clone(), filter_mapping.clone()), - }) - .collect() - }) + self.0 + .as_ref() + .map(BTreeMap::iter) + .map(|filters| { + filters + // When there are more variants of QueueFilter, change this to a `filter_map`. + .map(|(queue_id, queue_filter)| match queue_filter { + QueueFilter::Sqs(filter_mapping) => { + (queue_id.clone(), filter_mapping.clone()) + } + }) + .collect() + }) + .and_then(|filters_map: HashMap| { + filters_map.is_empty().not().then_some(filters_map) + }) } } diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index 395969cfe12..faf1d40a728 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt}; +use std::fmt; use base64::{engine::general_purpose, Engine}; use chrono::{DateTime, Utc}; @@ -16,10 +16,7 @@ use mirrord_auth::{ credentials::LicenseValidity, }; use mirrord_config::{ - feature::{ - network::incoming::ConcurrentSteal, - split_queues::{QueueFilter, QueueId}, - }, + feature::{network::incoming::ConcurrentSteal, split_queues::SplitQueuesConfig}, target::Target, LayerConfig, }; @@ -663,12 +660,7 @@ impl OperatorApi { let scale_down = config.feature.copy_target.scale_down; let namespace = self.target_namespace(config); let copied = self - .copy_target( - target, - scale_down, - namespace, - config.feature.split_queues.0.clone(), - ) + .copy_target(target, scale_down, namespace, &config.feature.split_queues) .await?; copy_subtask.success(Some("target copied")); @@ -734,7 +726,7 @@ impl OperatorApi { target: Target, scale_down: bool, namespace: &str, - split_queues: Option>, + split_queues: &SplitQueuesConfig, ) -> OperatorApiResult { let name = TargetCrd::urlfied_name(&target); @@ -744,7 +736,7 @@ impl OperatorApi { target, idle_ttl: Some(Self::COPIED_POD_IDLE_TTL), scale_down, - split_queues, + split_queues: split_queues.clone(), }, ); diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index f0fe36f00d8..c3d605c8625 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -7,7 +7,7 @@ use kube::CustomResource; use kube_target::{KubeTarget, UnknownTargetType}; pub use mirrord_config::feature::split_queues::QueueId; use mirrord_config::{ - feature::split_queues::{QueueFilter, SqsMessageFilter}, + feature::split_queues::{SplitQueuesConfig, SqsMessageFilter}, target::{Target, TargetConfig}, }; use schemars::JsonSchema; @@ -302,7 +302,7 @@ pub struct CopyTargetSpec { /// Ignored if [`Target`] is not [`Target::Deployment`]. pub scale_down: bool, /// Split queues client side configuration. - pub split_queues: Option>, + pub split_queues: SplitQueuesConfig, } /// Features and operations that can be blocked by a `MirrordPolicy`. From 5c36e00989533f7577346a448648309d44800552 Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 8 Aug 2024 13:49:40 +0200 Subject: [PATCH 71/83] Added unknown variant to QueueFilter --- mirrord/config/src/feature/split_queues.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index c28b68048fc..353b38520ea 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -48,10 +48,11 @@ impl SplitQueuesConfig { .map(|filters| { filters // When there are more variants of QueueFilter, change this to a `filter_map`. - .map(|(queue_id, queue_filter)| match queue_filter { + .filter_map(|(queue_id, queue_filter)| match queue_filter { QueueFilter::Sqs(filter_mapping) => { - (queue_id.clone(), filter_mapping.clone()) + Some((queue_id.clone(), filter_mapping.clone())) } + _ => None, }) .collect() }) @@ -92,6 +93,10 @@ pub enum QueueFilter { /// Amazon Simple Queue Service. #[serde(rename = "SQS")] Sqs(SqsMessageFilter), + /// When a newer client sends a new filter kind to an older operator, that does not yet know + /// about that filter type, this is what that filter will be deserialized to. + #[serde(other)] + Unknown, } impl CollectAnalytics for &SplitQueuesConfig { From fca1a87a74e0f672c465da631378aa04cd727f49 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 9 Aug 2024 00:38:46 +0200 Subject: [PATCH 72/83] CR: info log on implicit copy-target --- mirrord/operator/src/client.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/client.rs b/mirrord/operator/src/client.rs index faf1d40a728..dc64cfd84b0 100644 --- a/mirrord/operator/src/client.rs +++ b/mirrord/operator/src/client.rs @@ -1,4 +1,4 @@ -use std::fmt; +use std::{fmt, ops::Not}; use base64::{engine::general_purpose, Engine}; use chrono::{DateTime, Utc}; @@ -653,6 +653,9 @@ impl OperatorApi { // use copy_target for splitting queues || config.feature.split_queues.is_set() { + if config.feature.copy_target.enabled.not() { + tracing::info!("Creating a copy-target for queue-splitting (even thought copy_target was not explicitly set).") + } let mut copy_subtask = progress.subtask("copying target"); // We do not validate the `target` here, it's up to the operator. From 4f2e53d1fefd70a9db792130fd7345dfedb586a0 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 9 Aug 2024 14:25:50 +0200 Subject: [PATCH 73/83] Change QueueConsumer to have a valid k8s schema --- mirrord/operator/src/crd.rs | 78 ++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 41 deletions(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index c3d605c8625..f2c6dcd0b31 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -376,65 +376,61 @@ pub enum SplitQueue { /// A workload that is a consumer of a queue that is being split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(rename_all = "camelCase")] // Deployment -> deployment in yaml. -pub enum QueueConsumer { - /// The name of the deployment and an optional container. +#[serde(tag = "type")] +pub struct QueueConsumer { + pub name: String, /// If a container is not specified, the workload queue registry will apply to every run that /// targets any of the workload's containers. - Deployment(String, Option), - - /// The name of the rollout and an optional container. - /// If a container is not specified, the workload queue registry will apply to every run that - /// targets any of the workload's containers. - Rollout(String, Option), + pub container: Option, + pub workload_type: QueueConsumerType, } -impl QueueConsumer { - pub fn get_type_name_container(&self) -> (&str, &str, Option<&str>) { - match self { - QueueConsumer::Deployment(dep, container) => ("deployment", dep, container.as_deref()), - QueueConsumer::Rollout(roll, container) => ("rollout", roll, container.as_deref()), - } - } +/// A workload that is a consumer of a queue that is being split. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +pub enum QueueConsumerType { + Deployment, + + Rollout, + + // TODO: verify this is not part of the generated schema, should be de-only + #[serde(other)] + Unsupported, +} - pub fn container(&self) -> Option<&str> { +impl Display for QueueConsumerType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - QueueConsumer::Deployment(.., container) | QueueConsumer::Rollout(.., container) => { - container.as_deref() - } + QueueConsumerType::Deployment => write!(f, "deployment"), + QueueConsumerType::Rollout => write!(f, "rollout"), + QueueConsumerType::Unsupported => write!(f, "unsupported"), } } +} +impl QueueConsumer { /// For self that is the queue consumer of a run, test if a given registry object is the correct /// registry for this run. pub fn registry_matches(&self, registry: &MirrordWorkloadQueueRegistry) -> bool { - match (self, ®istry.spec.consumer) { - ( - QueueConsumer::Deployment(name, container), - QueueConsumer::Deployment(registry_consumer_name, registry_consumer_container), - ) - | ( - QueueConsumer::Rollout(name, container), - QueueConsumer::Rollout(registry_consumer_name, registry_consumer_container), - ) => { - name == registry_consumer_name - && (container == registry_consumer_container - // If registry does not specify a container, it applies to all runs with - // this target, regardless of what container they are targeting. - || registry_consumer_container.is_none()) - } - _ => false, - } + let registry_consumer = ®istry.spec.consumer; + self.workload_type == registry_consumer.workload_type + && self.name == registry_consumer.name + && (self.container == registry_consumer.container + // If registry does not specify a container, it applies to all runs with + // this target, regardless of what container they are targeting. + || registry_consumer.container.is_none()) } } impl Display for QueueConsumer { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let (type_, name, container) = self.get_type_name_container(); - if let Some(container) = container { - write!(f, "{}/{}/container/{container}", type_, name) + if let Some(ref container) = self.container { + write!( + f, + "{}/{}/container/{container}", + self.workload_type, self.name + ) } else { - write!(f, "{}/{}", type_, name) + write!(f, "{}/{}", self.workload_type, self.name) } } } From e6e281d3c2a6846701b0839653bb3a8c91faafab Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 9 Aug 2024 22:25:06 +0200 Subject: [PATCH 74/83] CRD fixes: skip unknown variants in schema, change CleanupError status for k8s --- mirrord/config/src/feature/split_queues.rs | 3 ++- mirrord/operator/src/crd.rs | 22 ++++++++++------------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/mirrord/config/src/feature/split_queues.rs b/mirrord/config/src/feature/split_queues.rs index 353b38520ea..1b25c0cf199 100644 --- a/mirrord/config/src/feature/split_queues.rs +++ b/mirrord/config/src/feature/split_queues.rs @@ -95,7 +95,8 @@ pub enum QueueFilter { Sqs(SqsMessageFilter), /// When a newer client sends a new filter kind to an older operator, that does not yet know /// about that filter type, this is what that filter will be deserialized to. - #[serde(other)] + #[schemars(skip)] + #[serde(other, skip_serializing)] Unknown, } diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index f2c6dcd0b31..b44b8c03158 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -257,7 +257,8 @@ pub enum NewOperatorFeature { SessionManagement, /// This variant is what a client sees when the operator includes a feature the client is not /// yet aware of, because it was introduced in a version newer than the client's. - #[serde(other)] + #[schemars(skip)] + #[serde(other, skip_serializing)] Unknown, } @@ -392,8 +393,8 @@ pub enum QueueConsumerType { Rollout, - // TODO: verify this is not part of the generated schema, should be de-only - #[serde(other)] + #[schemars(skip)] + #[serde(other, skip_serializing)] Unsupported, } @@ -499,12 +500,6 @@ pub struct MirrordWorkloadQueueRegistrySpec { pub consumer: QueueConsumer, } -#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] -pub enum SqsSessionCleanupError { - #[serde(other)] - Unknown, -} - #[derive(Clone, Debug, Default, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSplitDetails", rename_all = "camelCase")] pub struct SqsSplitDetails { @@ -549,7 +544,10 @@ pub enum SqsSessionStatus { RegisteringFilters(SqsSplitDetails), Ready(SqsSplitDetails), StartError(SqsSessionError), - CleanupError(SqsSessionError, Option), + CleanupError { + error: SqsSessionError, + details: Option, + }, } impl SqsSessionStatus { @@ -558,7 +556,7 @@ impl SqsSessionStatus { SqsSessionStatus::RegisteringFilters(details) | SqsSessionStatus::Ready(details) => { Some(details) } - SqsSessionStatus::CleanupError(.., details) => details.as_ref(), + SqsSessionStatus::CleanupError { details, .. } => details.as_ref(), _ => None, } } @@ -574,7 +572,7 @@ pub fn is_session_ready(session: Option<&MirrordSqsSession>) -> bool { status, SqsSessionStatus::Ready(..) | SqsSessionStatus::StartError(..) - | SqsSessionStatus::CleanupError(..) + | SqsSessionStatus::CleanupError { .. } ) }) .unwrap_or_default() From 78526f5c724cbdea687f41d37ffad91b8c5987ba Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 9 Aug 2024 23:36:43 +0200 Subject: [PATCH 75/83] starting status has to have items --- mirrord/operator/src/crd.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index b44b8c03158..013bcb1f804 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -537,7 +537,9 @@ impl Display for SqsSessionError { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus")] pub enum SqsSessionStatus { - Starting(), + Starting { + start_time_millis: u128, + }, /// SQS operator sets this status before it starts registering filters, so that if anything /// fails during the registration of filters, we have all the queues we need to delete on /// cleanup. From 208ae7df7ef1408c041ea1f17290cb7d72922809 Mon Sep 17 00:00:00 2001 From: t4lz Date: Sat, 10 Aug 2024 18:10:30 +0200 Subject: [PATCH 76/83] docs --- mirrord/operator/src/crd.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 013bcb1f804..62a15746e39 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -537,6 +537,10 @@ impl Display for SqsSessionError { #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] #[serde(rename = "SQSSessionStatus")] pub enum SqsSessionStatus { + // kube-rs does not allow mixing unit variants with tuple/struct variants, so this variant + // has to be a tuple/struct too. If we leave the tuple empty, k8s complains about an object + // without any items, and kube-rs does not support internally tagged enums, so we actually + // have to put something in there, even if we don't actually care about that info. Starting { start_time_millis: u128, }, From d682427631702cdf6530b3fb626ec37cc992805f Mon Sep 17 00:00:00 2001 From: t4lz Date: Sun, 11 Aug 2024 16:47:19 +0200 Subject: [PATCH 77/83] change start time to str because I remember k8s actually can't deal with unsigned integer of 64 bits and more --- mirrord/operator/src/crd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 62a15746e39..9aeafbbd9e3 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -542,7 +542,7 @@ pub enum SqsSessionStatus { // without any items, and kube-rs does not support internally tagged enums, so we actually // have to put something in there, even if we don't actually care about that info. Starting { - start_time_millis: u128, + start_time_utc: String, }, /// SQS operator sets this status before it starts registering filters, so that if anything /// fails during the registration of filters, we have all the queues we need to delete on From e2d2219a3e6414bd99ef2567946ca556637e056d Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 12 Aug 2024 19:36:46 +0200 Subject: [PATCH 78/83] rule to patch pods --- mirrord/operator/src/setup.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 3ed5d86dc0c..622dcff1e18 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -452,6 +452,14 @@ impl OperatorRole { verbs: vec!["patch".to_owned()], ..Default::default() }, + // For mirrord (not SQS) controller to patch copy pod to use other SQS queue. + // For SQS queue splitting, but needed by the mirrord operator. + PolicyRule { + api_groups: Some(vec!["apps".to_owned()]), + resources: Some(vec!["pods".to_owned()]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, PolicyRule { api_groups: Some(vec!["apps".to_owned(), "argoproj.io".to_owned()]), resources: Some(vec![ From 0095c85eb97a8cf8c3845e8e177b2df90654638c Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 12 Aug 2024 19:47:45 +0200 Subject: [PATCH 79/83] api group of pods is empty string --- mirrord/operator/src/setup.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 622dcff1e18..5dbf3ab809c 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -455,7 +455,7 @@ impl OperatorRole { // For mirrord (not SQS) controller to patch copy pod to use other SQS queue. // For SQS queue splitting, but needed by the mirrord operator. PolicyRule { - api_groups: Some(vec!["apps".to_owned()]), + api_groups: Some(vec!["".to_owned()]), resources: Some(vec!["pods".to_owned()]), verbs: vec!["patch".to_owned()], ..Default::default() From 24cae8aa010c581c04ffe5b65b85f7a368f7afc6 Mon Sep 17 00:00:00 2001 From: t4lz Date: Mon, 12 Aug 2024 20:31:09 +0200 Subject: [PATCH 80/83] Can't patch pod container env :( --- mirrord/operator/src/setup.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index 5dbf3ab809c..3ed5d86dc0c 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -452,14 +452,6 @@ impl OperatorRole { verbs: vec!["patch".to_owned()], ..Default::default() }, - // For mirrord (not SQS) controller to patch copy pod to use other SQS queue. - // For SQS queue splitting, but needed by the mirrord operator. - PolicyRule { - api_groups: Some(vec!["".to_owned()]), - resources: Some(vec!["pods".to_owned()]), - verbs: vec!["patch".to_owned()], - ..Default::default() - }, PolicyRule { api_groups: Some(vec!["apps".to_owned(), "argoproj.io".to_owned()]), resources: Some(vec![ From 7b7f4dd8bebebcf5b86da693c9e80bbd8dda57b8 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 16 Aug 2024 11:23:47 +0200 Subject: [PATCH 81/83] CR: out of place serde tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> --- mirrord/operator/src/crd.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/mirrord/operator/src/crd.rs b/mirrord/operator/src/crd.rs index 9aeafbbd9e3..2a7cbe0b0ff 100644 --- a/mirrord/operator/src/crd.rs +++ b/mirrord/operator/src/crd.rs @@ -377,7 +377,6 @@ pub enum SplitQueue { /// A workload that is a consumer of a queue that is being split. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(tag = "type")] pub struct QueueConsumer { pub name: String, /// If a container is not specified, the workload queue registry will apply to every run that From 08f5b26f2232bb61e4816c00011019a560b3d625 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 16 Aug 2024 13:40:19 +0200 Subject: [PATCH 82/83] Support installing an SQS-enabled operator via `mirrord operator setup` command --- mirrord/cli/src/config.rs | 17 +++ mirrord/cli/src/operator.rs | 20 ++- mirrord/operator/src/setup.rs | 233 +++++++++++++++++++--------------- 3 files changed, 163 insertions(+), 107 deletions(-) diff --git a/mirrord/cli/src/config.rs b/mirrord/cli/src/config.rs index 2d7cbcf3dff..321cb794128 100644 --- a/mirrord/cli/src/config.rs +++ b/mirrord/cli/src/config.rs @@ -336,6 +336,23 @@ pub(super) enum OperatorCommand { /// will be able to access) #[arg(short, long, default_value = "mirrord")] namespace: OperatorNamespace, + + /// AWS role ARN for the operator's service account. + /// Necessary for enabling SQS queue splitting. + /// For successfully running an SQS queue splitting operator the given IAM role must + #[arg(long, visible_alias = "arn")] + aws_role_arn: Option, + + /// Enable SQS queue splitting. + /// When set, some extra CRDs will be installed on the cluster, and the operator will run + /// an SQS splitting component. + #[arg( + long, + visible_alias = "sqs", + default_value_t = false, + requires = "aws_role_arn" + )] + sqs_splitting: bool, }, /// Print operator status Status { diff --git a/mirrord/cli/src/operator.rs b/mirrord/cli/src/operator.rs index f7a255dc5b1..45b11ffeb58 100644 --- a/mirrord/cli/src/operator.rs +++ b/mirrord/cli/src/operator.rs @@ -59,6 +59,8 @@ async fn operator_setup( namespace: OperatorNamespace, license_key: Option, license_path: Option, + aws_role_arn: Option, + sqs_splitting: bool, ) -> Result<(), OperatorSetupError> { if !accept_tos { eprintln!("Please note that mirrord operator installation requires an active subscription for the mirrord Operator provided by MetalBear Tech LTD.\nThe service ToS can be read here - https://metalbear.co/legal/terms\nPass --accept-tos to accept the TOS"); @@ -101,6 +103,8 @@ async fn operator_setup( license, namespace, image, + aws_role_arn, + sqs_splitting, }); match file { @@ -299,9 +303,19 @@ pub(crate) async fn operator_command(args: OperatorArgs) -> Result<()> { namespace, license_key, license_path, - } => operator_setup(accept_tos, file, namespace, license_key, license_path) - .await - .map_err(CliError::from), + aws_role_arn, + sqs_splitting, + } => operator_setup( + accept_tos, + file, + namespace, + license_key, + license_path, + aws_role_arn, + sqs_splitting, + ) + .await + .map_err(CliError::from), OperatorCommand::Status { config_file } => operator_status(config_file.as_deref()).await, OperatorCommand::Session(session_command) => { SessionCommandHandler::new(session_command) diff --git a/mirrord/operator/src/setup.rs b/mirrord/operator/src/setup.rs index b978dad1e54..d9e6a7432c3 100644 --- a/mirrord/operator/src/setup.rs +++ b/mirrord/operator/src/setup.rs @@ -88,6 +88,8 @@ pub struct SetupOptions { pub license: LicenseType, pub namespace: OperatorNamespace, pub image: String, + pub aws_role_arn: Option, + pub sqs_splitting: bool, } #[derive(Debug)] @@ -103,6 +105,7 @@ pub struct Operator { user_cluster_role: OperatorClusterUserRole, client_ca_role: OperatorClientCaRole, client_ca_role_binding: OperatorClientCaRoleBinding, + sqs_splitting: bool, } impl Operator { @@ -111,6 +114,8 @@ impl Operator { license, namespace, image, + aws_role_arn, + sqs_splitting, } = options; let (license_secret, license_key) = match license { @@ -120,9 +125,9 @@ impl Operator { } }; - let service_account = OperatorServiceAccount::new(&namespace); + let service_account = OperatorServiceAccount::new(&namespace, aws_role_arn); - let role = OperatorRole::new(); + let role = OperatorRole::new(sqs_splitting); let role_binding = OperatorRoleBinding::new(&role, &service_account); let user_cluster_role = OperatorClusterUserRole::new(); @@ -136,6 +141,7 @@ impl Operator { license_secret.as_ref(), license_key, image, + sqs_splitting, ); let service = OperatorService::new(&namespace); @@ -154,6 +160,7 @@ impl Operator { user_cluster_role, client_ca_role, client_ca_role_binding, + sqs_splitting, } } } @@ -197,11 +204,13 @@ impl OperatorSetup for Operator { writer.write_all(b"---\n")?; MirrordPolicy::crd().to_writer(&mut writer)?; - writer.write_all(b"---\n")?; - MirrordWorkloadQueueRegistry::crd().to_writer(&mut writer)?; + if self.sqs_splitting { + writer.write_all(b"---\n")?; + MirrordWorkloadQueueRegistry::crd().to_writer(&mut writer)?; - writer.write_all(b"---\n")?; - MirrordSqsSession::crd().to_writer(&mut writer)?; + writer.write_all(b"---\n")?; + MirrordSqsSession::crd().to_writer(&mut writer)?; + } Ok(()) } @@ -242,6 +251,7 @@ impl OperatorDeployment { license_secret: Option<&OperatorLicenseSecret>, license_key: Option, image: String, + sqs_splitting: bool, ) -> Self { let mut envs = vec![ EnvVar { @@ -301,6 +311,14 @@ impl OperatorDeployment { }); } + if sqs_splitting { + envs.push(EnvVar { + name: "OPERATOR_SQS_SPLITTING".to_owned(), + value: Some("true".to_string()), + value_from: None, + }); + } + let health_probe = Probe { http_get: Some(HTTPGetAction { path: Some("/health".to_owned()), @@ -385,11 +403,13 @@ impl OperatorDeployment { pub struct OperatorServiceAccount(ServiceAccount); impl OperatorServiceAccount { - pub fn new(namespace: &OperatorNamespace) -> Self { + pub fn new(namespace: &OperatorNamespace, aws_role_arn: Option) -> Self { let sa = ServiceAccount { metadata: ObjectMeta { name: Some(OPERATOR_SERVICE_ACCOUNT_NAME.to_owned()), namespace: Some(namespace.name().to_owned()), + annotations: aws_role_arn + .map(|arn| BTreeMap::from([("eks.amazonaws.com/role-arn".to_string(), arn)])), labels: Some(APP_LABELS.clone()), ..Default::default() }, @@ -417,107 +437,105 @@ impl OperatorServiceAccount { pub struct OperatorRole(ClusterRole); impl OperatorRole { - pub fn new() -> Self { - let role = ClusterRole { - metadata: ObjectMeta { - name: Some(OPERATOR_ROLE_NAME.to_owned()), + pub fn new(sqs_splitting: bool) -> Self { + let mut rules = vec![ + PolicyRule { + api_groups: Some(vec![ + "".to_owned(), + "apps".to_owned(), + "batch".to_owned(), + "argoproj.io".to_owned(), + ]), + resources: Some(vec![ + "nodes".to_owned(), + "pods".to_owned(), + "pods/log".to_owned(), + "pods/ephemeralcontainers".to_owned(), + "deployments".to_owned(), + "deployments/scale".to_owned(), + "rollouts".to_owned(), + "rollouts/scale".to_owned(), + "jobs".to_owned(), + "cronjobs".to_owned(), + "statefulsets".to_owned(), + "statefulsets/scale".to_owned(), + ]), + verbs: vec!["get".to_owned(), "list".to_owned(), "watch".to_owned()], ..Default::default() }, - rules: Some(vec![ - PolicyRule { - api_groups: Some(vec![ - "".to_owned(), - "apps".to_owned(), - "batch".to_owned(), - "argoproj.io".to_owned(), - ]), - resources: Some(vec![ - "nodes".to_owned(), - "pods".to_owned(), - "pods/log".to_owned(), - "pods/ephemeralcontainers".to_owned(), - "deployments".to_owned(), - "deployments/scale".to_owned(), - "rollouts".to_owned(), - "rollouts/scale".to_owned(), - "jobs".to_owned(), - "cronjobs".to_owned(), - "statefulsets".to_owned(), - "statefulsets/scale".to_owned(), - ]), - verbs: vec!["get".to_owned(), "list".to_owned(), "watch".to_owned()], - ..Default::default() - }, - // For SQS controller to temporarily change deployments to use changed queues. - PolicyRule { - api_groups: Some(vec!["apps".to_owned()]), - resources: Some(vec!["deployments".to_owned()]), - verbs: vec!["patch".to_owned()], - ..Default::default() - }, - // For SQS controller to temporarily change Argo Rollouts to use changed queues. - PolicyRule { - api_groups: Some(vec!["argoproj.io".to_owned()]), - resources: Some(vec!["rollouts".to_owned()]), - verbs: vec!["patch".to_owned()], - ..Default::default() - }, - PolicyRule { - api_groups: Some(vec!["apps".to_owned(), "argoproj.io".to_owned()]), - resources: Some(vec![ - "deployments/scale".to_owned(), - "rollouts/scale".to_owned(), - "statefulsets/scale".to_owned(), - ]), - verbs: vec!["patch".to_owned()], - ..Default::default() - }, - PolicyRule { - api_groups: Some(vec!["".to_owned(), "batch".to_owned()]), - resources: Some(vec!["jobs".to_owned(), "pods".to_owned()]), - verbs: vec!["create".to_owned(), "delete".to_owned()], - ..Default::default() - }, - PolicyRule { - api_groups: Some(vec!["".to_owned()]), - resources: Some(vec!["pods/ephemeralcontainers".to_owned()]), - verbs: vec!["update".to_owned()], - ..Default::default() - }, - PolicyRule { - api_groups: Some(vec!["".to_owned(), "authentication.k8s.io".to_owned()]), - resources: Some(vec![ - "groups".to_owned(), - "users".to_owned(), - "userextras/accesskeyid".to_owned(), - "userextras/arn".to_owned(), - "userextras/canonicalarn".to_owned(), - "userextras/sessionname".to_owned(), - "userextras/iam.gke.io/user-assertion".to_owned(), - "userextras/user-assertion.cloud.google.com".to_owned(), - "userextras/principalid".to_owned(), - "userextras/oid".to_owned(), - "userextras/username".to_owned(), - "userextras/licensekey".to_owned(), - ]), - verbs: vec!["impersonate".to_owned()], - ..Default::default() - }, - // Allow the operator to list+get mirrord policies. - PolicyRule { - api_groups: Some(vec!["policies.mirrord.metalbear.co".to_owned()]), - resources: Some(vec![MirrordPolicy::plural(&()).to_string()]), - verbs: vec!["list".to_owned(), "get".to_owned()], - ..Default::default() - }, - // Allow the operator to list mirrord queue splitters. + // For SQS controller to temporarily change deployments to use changed queues. + PolicyRule { + api_groups: Some(vec!["apps".to_owned()]), + resources: Some(vec!["deployments".to_owned()]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, + // For SQS controller to temporarily change Argo Rollouts to use changed queues. + PolicyRule { + api_groups: Some(vec!["argoproj.io".to_owned()]), + resources: Some(vec!["rollouts".to_owned()]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["apps".to_owned(), "argoproj.io".to_owned()]), + resources: Some(vec![ + "deployments/scale".to_owned(), + "rollouts/scale".to_owned(), + "statefulsets/scale".to_owned(), + ]), + verbs: vec!["patch".to_owned()], + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["".to_owned(), "batch".to_owned()]), + resources: Some(vec!["jobs".to_owned(), "pods".to_owned()]), + verbs: vec!["create".to_owned(), "delete".to_owned()], + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["".to_owned()]), + resources: Some(vec!["pods/ephemeralcontainers".to_owned()]), + verbs: vec!["update".to_owned()], + ..Default::default() + }, + PolicyRule { + api_groups: Some(vec!["".to_owned(), "authentication.k8s.io".to_owned()]), + resources: Some(vec![ + "groups".to_owned(), + "users".to_owned(), + "userextras/accesskeyid".to_owned(), + "userextras/arn".to_owned(), + "userextras/canonicalarn".to_owned(), + "userextras/sessionname".to_owned(), + "userextras/iam.gke.io/user-assertion".to_owned(), + "userextras/user-assertion.cloud.google.com".to_owned(), + "userextras/principalid".to_owned(), + "userextras/oid".to_owned(), + "userextras/username".to_owned(), + "userextras/licensekey".to_owned(), + ]), + verbs: vec!["impersonate".to_owned()], + ..Default::default() + }, + // Allow the operator to list+get mirrord policies. + PolicyRule { + api_groups: Some(vec!["policies.mirrord.metalbear.co".to_owned()]), + resources: Some(vec![MirrordPolicy::plural(&()).to_string()]), + verbs: vec!["list".to_owned(), "get".to_owned()], + ..Default::default() + }, + ]; + if sqs_splitting { + rules.extend([ + // Allow the operator to list mirrord queue registries. PolicyRule { api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec![MirrordWorkloadQueueRegistry::plural(&()).to_string()]), verbs: vec!["list".to_owned()], ..Default::default() }, - // Allow the SQS controller to update queue splitter status. + // Allow the SQS controller to update queue registry status. PolicyRule { api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec!["mirrordworkloadqueueregistries/status".to_string()]), @@ -527,7 +545,7 @@ impl OperatorRole { ], ..Default::default() }, - // Allow the operator to control mirrord queue filters. + // Allow the operator to control mirrord SQS session objects. PolicyRule { api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec![MirrordSqsSession::plural(&()).to_string()]), @@ -542,7 +560,7 @@ impl OperatorRole { ], ..Default::default() }, - // Allow the SQS controller to update queue splitter status. + // Allow the SQS controller to update queue registry status. PolicyRule { api_groups: Some(vec!["queues.mirrord.metalbear.co".to_owned()]), resources: Some(vec!["mirrordsqssessions/status".to_string()]), @@ -552,7 +570,14 @@ impl OperatorRole { ], ..Default::default() }, - ]), + ]); + } + let role = ClusterRole { + metadata: ObjectMeta { + name: Some(OPERATOR_ROLE_NAME.to_owned()), + ..Default::default() + }, + rules: Some(rules), ..Default::default() }; @@ -570,7 +595,7 @@ impl OperatorRole { impl Default for OperatorRole { fn default() -> Self { - Self::new() + Self::new(false) } } From 04487801d2e86ca91ed276ee6ccb41c52fde7f81 Mon Sep 17 00:00:00 2001 From: t4lz Date: Fri, 16 Aug 2024 13:53:49 +0200 Subject: [PATCH 83/83] incomplete flag docs --- mirrord/cli/src/config.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mirrord/cli/src/config.rs b/mirrord/cli/src/config.rs index 321cb794128..d3bfd4a4203 100644 --- a/mirrord/cli/src/config.rs +++ b/mirrord/cli/src/config.rs @@ -339,7 +339,10 @@ pub(super) enum OperatorCommand { /// AWS role ARN for the operator's service account. /// Necessary for enabling SQS queue splitting. - /// For successfully running an SQS queue splitting operator the given IAM role must + /// For successfully running an SQS queue splitting operator the given IAM role must be + /// able to create, read from, write to, and delete SQS queues. + /// If the queue messages are encrypted using KMS, the operator also needs the + /// `kms:Encrypt`, `kms:Decrypt` and `kms:GenerateDataKey` permissions. #[arg(long, visible_alias = "arn")] aws_role_arn: Option,