not-found file filter (#1825)

* File filter and config updated

* Schema udpated

* Changelog entry

* macos fix

* Removed env

* Removed aws paths from default remote-readonly filter

* Test fix

* Tests fix

* Obsolete env reads removed

changelog.d/1694.fixed.md

@@ -0,0 +1 @@
+Added an extra `not-found` file filter to improve experience when using cloud services under mirrord.

mirrord-schema.json

@@ -156,7 +156,7 @@
   "additionalProperties": false,
   "definitions": {
     "AdvancedFsUserConfig": {
-      "description": "Allows the user to specify the default behavior for file operations:\n\n1. `\"read\"` - Read from the remote file system (default) 2. `\"write\"` - Read/Write from the remote file system. 3. `\"local\"` - Read from the local file system. 5. `\"disable\"` - Disable file operations.\n\nBesides the default behavior, the user can specify behavior for specific regex patterns. Case insensitive.\n\n1. `\"read_write\"` - List of patterns that should be read/write remotely. 2. `\"read_only\"` - List of patterns that should be read only remotely. 3. `\"local\"` - List of patterns that should be read locally.\n\nThe logic for choosing the behavior is as follows:\n\n1. Check if one of the patterns match the file path, do the corresponding action. There's no specified order if two lists match the same path, we will use the first one (and we do not guarantee what is first).\n\n**Warning**: Specifying the same path in two lists is unsupported and can lead to undefined behaviour.\n\n2. Check our \"special list\" - we have an internal at compile time list for different behavior based on patterns to provide better UX.\n\n3. If none of the above match, use the default behavior (mode).\n\nFor more information, check the file operations [technical reference](https://mirrord.dev/docs/reference/fileops/).\n\n```json { \"feature\": { \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ] } } } ```",
+      "description": "Allows the user to specify the default behavior for file operations:\n\n1. `\"read\"` - Read from the remote file system (default) 2. `\"write\"` - Read/Write from the remote file system. 3. `\"local\"` - Read from the local file system. 5. `\"disable\"` - Disable file operations.\n\nBesides the default behavior, the user can specify behavior for specific regex patterns. Case insensitive.\n\n1. `\"read_write\"` - List of patterns that should be read/write remotely. 2. `\"read_only\"` - List of patterns that should be read only remotely. 3. `\"local\"` - List of patterns that should be read locally. 4. `\"not_found\"` - List of patters that should never be read nor written. These files should be treated as non-existent.\n\nThe logic for choosing the behavior is as follows:\n\n1. Check if one of the patterns match the file path, do the corresponding action. There's no specified order if two lists match the same path, we will use the first one (and we do not guarantee what is first).\n\n**Warning**: Specifying the same path in two lists is unsupported and can lead to undefined behaviour.\n\n2. Check our \"special list\" - we have an internal at compile time list for different behavior based on patterns to provide better UX.\n\n3. If none of the above match, use the default behavior (mode).\n\nFor more information, check the file operations [technical reference](https://mirrord.dev/docs/reference/fileops/).\n\n```json { \"feature\": { \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\.json\" , \"read_only\": [ \".+\\.yaml\", \".+important-file\\.txt\" ], \"local\": [ \".+\\.js\", \".+\\.mjs\" ], \"not_found\": [ \"\\.config/gcloud\" ] } } } ```",
       "type": "object",
       "properties": {
         "local": {
@@ -182,6 +182,18 @@
             }
           ]
         },
+        "not_found": {
+          "title": "feature.fs.not_found {#feature-fs-not_found}",
+          "description": "Specify file path patterns that if matched will be treated as non-existent.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/VecOrSingle_for_String"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "read_only": {
           "title": "feature.fs.read_only {#feature-fs-read_only}",
           "description": "Specify file path patterns that if matched will be read from the remote. if file matching the pattern is opened for writing or read/write it will be opened locally.",

mirrord/config/src/feature/fs.rs

@@ -87,6 +87,7 @@ impl MirrordConfig for FsUserConfig {
                 local: FromEnv::new("MIRRORD_FILE_LOCAL_PATTERN")
                     .source_value()
                     .transpose()?,
+                not_found: None,
             },
             FsUserConfig::Advanced(advanced) => advanced.generate_config()?,
         };
@@ -113,6 +114,7 @@ impl MirrordToggleableConfig for FsUserConfig {
             read_write,
             read_only,
             local,
+            not_found: None,
         })
     }
 }

mirrord/config/src/feature/fs/advanced.rs

@@ -25,6 +25,8 @@ use crate::{
 /// 1. `"read_write"` - List of patterns that should be read/write remotely.
 /// 2. `"read_only"` - List of patterns that should be read only remotely.
 /// 3. `"local"` - List of patterns that should be read locally.
+/// 4. `"not_found"` - List of patters that should never be read nor written. These files should be
+/// treated as non-existent.
 ///
 /// The logic for choosing the behavior is as follows:
 ///
@@ -50,7 +52,8 @@ use crate::{
 ///       "mode": "write",
 ///       "read_write": ".+\.json" ,
 ///       "read_only": [ ".+\.yaml", ".+important-file\.txt" ],
-///       "local": [ ".+\.js", ".+\.mjs" ]
+///       "local": [ ".+\.js", ".+\.mjs" ],
+///       "not_found": [ "\.config/gcloud" ]
 ///     }
 ///   }
 /// }
@@ -83,6 +86,11 @@ pub struct FsConfig {
     /// Specify file path patterns that if matched will be opened locally.
     #[config(env = "MIRRORD_FILE_LOCAL_PATTERN")]
     pub local: Option<VecOrSingle<String>>,
+
+    /// ### feature.fs.not_found {#feature-fs-not_found}
+    ///
+    /// Specify file path patterns that if matched will be treated as non-existent.
+    pub not_found: Option<VecOrSingle<String>>,
 }
 
 impl MirrordToggleableConfig for AdvancedFsUserConfig {
@@ -103,6 +111,7 @@ impl MirrordToggleableConfig for AdvancedFsUserConfig {
             read_write,
             read_only,
             local,
+            not_found: None,
         })
     }
 }
@@ -138,18 +147,31 @@ impl CollectAnalytics for &FsConfig {
         analytics.add("mode", self.mode);
         analytics.add(
             "local_paths",
-            self.local.as_ref().map(|v| v.len()).unwrap_or_default(),
+            self.local
+                .as_ref()
+                .map(VecOrSingle::len)
+                .unwrap_or_default(),
         );
         analytics.add(
             "read_write_paths",
             self.read_write
                 .as_ref()
-                .map(|v| v.len())
+                .map(VecOrSingle::len)
                 .unwrap_or_default(),
         );
         analytics.add(
             "read_only_paths",
-            self.read_only.as_ref().map(|v| v.len()).unwrap_or_default(),
+            self.read_only
+                .as_ref()
+                .map(VecOrSingle::len)
+                .unwrap_or_default(),
+        );
+        analytics.add(
+            "not_found_paths",
+            self.not_found
+                .as_ref()
+                .map(VecOrSingle::len)
+                .unwrap_or_default(),
         );
     }
 }

mirrord/config/src/feature/fs/mode.rs

@@ -46,6 +46,10 @@ pub enum FsModeConfig {
 }
 
 impl FsModeConfig {
+    pub fn is_local(self) -> bool {
+        matches!(self, FsModeConfig::Local)
+    }
+
     pub fn is_read(self) -> bool {
         matches!(self, FsModeConfig::Read | FsModeConfig::LocalWithOverrides)
     }

mirrord/layer/src/error.rs

@@ -96,6 +96,11 @@ pub(crate) enum HookError {
 
     #[error("mirrord-layer: Socket address `{0}` is already bound!")]
     AddressAlreadyBound(SocketAddr),
+
+    /// When the user's application tries to access a file filtered out by the `not-found` file
+    /// filter.
+    #[error("mirrord-layer: Ignored file")]
+    FileNotFound,
 }
 
 /// Errors internal to mirrord-layer.
@@ -307,6 +312,7 @@ impl From<HookError> for i64 {
             HookError::UnsupportedSocketType => libc::EAFNOSUPPORT,
             HookError::BadPointer => libc::EFAULT,
             HookError::AddressAlreadyBound(_) => libc::EADDRINUSE,
+            HookError::FileNotFound => libc::ENOENT,
         };
 
         set_errno(errno::Errno(libc_error));