Add U8 (newer .arc) support

Supports the U8 .arc format, just
like the older RARC format.

`u8 list`, `u8 extract` and support
for U8 archive paths in config.yml

Cargo.toml

@@ -72,3 +72,4 @@ tracing = "0.1.40"
 tracing-attributes = "0.1.27"
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 xxhash-rust = { version = "0.8.10", features = ["xxh3"] }
+zerocopy = { version = "0.7.34", features = ["derive"] }

README.md

@@ -43,6 +43,8 @@ project structure and build system that uses decomp-toolkit under the hood.
   - [nlzss decompress](#nlzss-decompress)
   - [rarc list](#rarc-list)
   - [rarc extract](#rarc-extract)
+  - [u8 list](#u8-list)
+  - [u8 extract](#u8-extract)
   - [yay0 decompress](#yay0-decompress)
   - [yay0 compress](#yay0-compress)
   - [yaz0 decompress](#yaz0-decompress)
@@ -390,20 +392,36 @@ $ dtk nlzss decompress rels/*.lz -o rels
 
 ### rarc list
 
-Lists the contents of an RARC archive.
+Lists the contents of an RARC (older .arc) archive.
 
 ```shell
 $ dtk rarc list input.arc
 ```
 
 ### rarc extract
 
-Extracts the contents of an RARC archive.
+Extracts the contents of an RARC (older .arc) archive.
 
 ```shell
 $ dtk rarc extract input.arc -o output_dir
 ```
 
+### u8 list
+
+Extracts the contents of a U8 (newer .arc) archive.
+
+```shell
+$ dtk u8 list input.arc
+```
+
+### u8 extract
+
+Extracts the contents of a U8 (newer .arc) archive.
+
+```shell
+$ dtk u8 extract input.arc -o output_dir
+```
+
 ### yay0 decompress
 
 Decompresses Yay0-compressed files.

src/cmd/mod.rs

@@ -13,5 +13,6 @@ pub mod rarc;
 pub mod rel;
 pub mod rso;
 pub mod shasum;
+pub mod u8_arc;
 pub mod yay0;
 pub mod yaz0;

src/cmd/rarc.rs

@@ -42,6 +42,9 @@ pub struct ExtractArgs {
     #[argp(option, short = 'o')]
     /// output directory
     output: Option<PathBuf>,
+    #[argp(switch, short = 'q')]
+    /// quiet output
+    quiet: bool,
 }
 
 pub fn run(args: Args) -> Result<()> {
@@ -95,8 +98,19 @@ fn extract(args: ExtractArgs) -> Result<()> {
                     &file.as_slice()[offset as usize..offset as usize + size as usize],
                 )?;
                 let file_path = current_path.join(&name.name);
-                let output_path =
-                    args.output.as_ref().map(|p| p.join(&file_path)).unwrap_or_else(|| file_path);
+                let output_path = args
+                    .output
+                    .as_ref()
+                    .map(|p| p.join(&file_path))
+                    .unwrap_or_else(|| file_path.clone());
+                if !args.quiet {
+                    println!(
+                        "Extracting {} to {} ({} bytes)",
+                        file_path.display(),
+                        output_path.display(),
+                        size
+                    );
+                }
                 if let Some(parent) = output_path.parent() {
                     DirBuilder::new().recursive(true).create(parent)?;
                 }

src/cmd/u8_arc.rs

@@ -0,0 +1,121 @@
+use std::{borrow::Cow, fs, fs::DirBuilder, path::PathBuf};
+
+use anyhow::{anyhow, Context, Result};
+use argp::FromArgs;
+use itertools::Itertools;
+
+use crate::util::{
+    file::{decompress_if_needed, map_file},
+    u8_arc::{U8Node, U8View},
+};
+
+#[derive(FromArgs, PartialEq, Debug)]
+/// Commands for processing U8 (arc) files.
+#[argp(subcommand, name = "u8")]
+pub struct Args {
+    #[argp(subcommand)]
+    command: SubCommand,
+}
+
+#[derive(FromArgs, PartialEq, Debug)]
+#[argp(subcommand)]
+enum SubCommand {
+    List(ListArgs),
+    Extract(ExtractArgs),
+}
+
+#[derive(FromArgs, PartialEq, Eq, Debug)]
+/// Views U8 (arc) file information.
+#[argp(subcommand, name = "list")]
+pub struct ListArgs {
+    #[argp(positional)]
+    /// U8 (arc) file
+    file: PathBuf,
+}
+
+#[derive(FromArgs, PartialEq, Eq, Debug)]
+/// Extracts U8 (arc) file contents.
+#[argp(subcommand, name = "extract")]
+pub struct ExtractArgs {
+    #[argp(positional)]
+    /// U8 (arc) file
+    file: PathBuf,
+    #[argp(option, short = 'o')]
+    /// output directory
+    output: Option<PathBuf>,
+    #[argp(switch, short = 'q')]
+    /// quiet output
+    quiet: bool,
+}
+
+pub fn run(args: Args) -> Result<()> {
+    match args.command {
+        SubCommand::List(c_args) => list(c_args),
+        SubCommand::Extract(c_args) => extract(c_args),
+    }
+}
+
+fn list(args: ListArgs) -> Result<()> {
+    let file = map_file(&args.file)?;
+    let view = U8View::new(file.as_slice())
+        .map_err(|e| anyhow!("Failed to open U8 file '{}': {}", args.file.display(), e))?;
+    visit_files(&view, |_, node, path| {
+        println!("{}: {} bytes, offset {:#X}", path, node.length(), node.offset());
+        Ok(())
+    })
+}
+
+fn extract(args: ExtractArgs) -> Result<()> {
+    let file = map_file(&args.file)?;
+    let view = U8View::new(file.as_slice())
+        .map_err(|e| anyhow!("Failed to open U8 file '{}': {}", args.file.display(), e))?;
+    visit_files(&view, |_, node, path| {
+        let offset = node.offset();
+        let size = node.length();
+        let file_data = decompress_if_needed(
+            &file.as_slice()[offset as usize..offset as usize + size as usize],
+        )?;
+        let output_path = args
+            .output
+            .as_ref()
+            .map(|p| p.join(&path))
+            .unwrap_or_else(|| PathBuf::from(path.clone()));
+        if !args.quiet {
+            println!("Extracting {} to {} ({} bytes)", path, output_path.display(), size);
+        }
+        if let Some(parent) = output_path.parent() {
+            DirBuilder::new().recursive(true).create(parent)?;
+        }
+        fs::write(&output_path, file_data)
+            .with_context(|| format!("Failed to write file '{}'", output_path.display()))?;
+        Ok(())
+    })
+}
+
+fn visit_files(
+    view: &U8View,
+    mut visitor: impl FnMut(usize, &U8Node, String) -> Result<()>,
+) -> Result<()> {
+    let mut path_segments = Vec::<(Cow<str>, usize)>::new();
+    for (idx, node, name) in view.iter() {
+        // Remove ended path segments
+        let mut new_size = 0;
+        for (_, end) in path_segments.iter() {
+            if *end == idx {
+                break;
+            }
+            new_size += 1;
+        }
+        path_segments.truncate(new_size);
+
+        // Add the new path segment
+        let end = if node.is_dir() { node.length() as usize } else { idx + 1 };
+        path_segments.push((name.map_err(|e| anyhow!("{}", e))?, end));
+
+        let path = path_segments.iter().map(|(name, _)| name.as_ref()).join("/");
+        if !node.is_dir() {
+            visitor(idx, node, path)?;
+        }
+    }
+    Ok(())
+}

src/main.rs

@@ -95,6 +95,7 @@ enum SubCommand {
     Rel(cmd::rel::Args),
     Rso(cmd::rso::Args),
     Shasum(cmd::shasum::Args),
+    U8(cmd::u8_arc::Args),
     Yay0(cmd::yay0::Args),
     Yaz0(cmd::yaz0::Args),
 }
@@ -169,6 +170,7 @@ fn main() {
         SubCommand::Rel(c_args) => cmd::rel::run(c_args),
         SubCommand::Rso(c_args) => cmd::rso::run(c_args),
         SubCommand::Shasum(c_args) => cmd::shasum::run(c_args),
+        SubCommand::U8(c_args) => cmd::u8_arc::run(c_args),
         SubCommand::Yay0(c_args) => cmd::yay0::run(c_args),
         SubCommand::Yaz0(c_args) => cmd::yaz0::run(c_args),
     });

src/util/file.rs

@@ -5,10 +5,11 @@ use std::{
     path::{Component, Path, PathBuf},
 };
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use filetime::{set_file_mtime, FileTime};
 use memmap2::{Mmap, MmapOptions};
 use path_slash::PathBufExt;
+use rarc::RarcReader;
 use sha1::{Digest, Sha1};
 use xxhash_rust::xxh3::xxh3_64;
 
@@ -19,6 +20,7 @@ use crate::{
         rarc,
         rarc::{Node, RARC_MAGIC},
         take_seek::{TakeSeek, TakeSeekExt},
+        u8_arc::{U8View, U8_MAGIC},
         Bytes,
     },
 };
@@ -105,11 +107,28 @@ where P: AsRef<Path> {
             ));
         }
 
-        let rarc = rarc::RarcReader::new(&mut Cursor::new(mmap.as_ref()))
-            .with_context(|| format!("Failed to open '{}' as RARC archive", base_path.display()))?;
-        rarc.find_file(&sub_path)?.map(|(o, s)| (o, s as u64)).ok_or_else(|| {
-            anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display())
-        })?
+        let buf = mmap.as_ref();
+        match *array_ref!(buf, 0, 4) {
+            RARC_MAGIC => {
+                let rarc = RarcReader::new(&mut Cursor::new(mmap.as_ref())).with_context(|| {
+                    format!("Failed to open '{}' as RARC archive", base_path.display())
+                })?;
+                let (offset, size) = rarc.find_file(&sub_path)?.ok_or_else(|| {
+                    anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display())
+                })?;
+                (offset, size as u64)
+            }
+            U8_MAGIC => {
+                let arc = U8View::new(buf).map_err(|e| {
+                    anyhow!("Failed to open '{}' as U8 archive: {}", base_path.display(), e)
+                })?;
+                let (_, node) = arc.find(sub_path.to_slash_lossy().as_ref()).ok_or_else(|| {
+                    anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display())
+                })?;
+                (node.offset() as u64, node.length() as u64)
+            }
+            _ => bail!("Couldn't detect archive type for '{}'", path.as_ref().display()),
+        }
     } else {
         (0, mmap.len() as u64)
     };
@@ -162,7 +181,7 @@ where P: AsRef<Path> {
     let mut file = File::open(&base_path)
         .with_context(|| format!("Failed to open file '{}'", base_path.display()))?;
     let (offset, size) = if let Some(sub_path) = sub_path {
-        let rarc = rarc::RarcReader::new(&mut BufReader::new(&file))
+        let rarc = RarcReader::new(&mut BufReader::new(&file))
             .with_context(|| format!("Failed to read RARC '{}'", base_path.display()))?;
         rarc.find_file(&sub_path)?.map(|(o, s)| (o, s as u64)).ok_or_else(|| {
             anyhow!("File '{}' not found in '{}'", sub_path.display(), base_path.display())
@@ -261,12 +280,12 @@ struct RarcIterator {
 
 impl RarcIterator {
     pub fn new(file: MappedFile, base_path: &Path) -> Result<Self> {
-        let reader = rarc::RarcReader::new(&mut file.as_reader())?;
+        let reader = RarcReader::new(&mut file.as_reader())?;
         let paths = Self::collect_paths(&reader, base_path);
         Ok(Self { file, base_path: base_path.to_owned(), paths, index: 0 })
     }
 
-    fn collect_paths(reader: &rarc::RarcReader, base_path: &Path) -> Vec<(PathBuf, u64, u32)> {
+    fn collect_paths(reader: &RarcReader, base_path: &Path) -> Vec<(PathBuf, u64, u32)> {
         let mut current_path = PathBuf::new();
         let mut paths = vec![];
         for node in reader.nodes() {

src/util/mod.rs

@@ -22,6 +22,7 @@ pub mod rso;
 pub mod signatures;
 pub mod split;
 pub mod take_seek;
+pub mod u8_arc;
 
 #[inline]
 pub const fn align_up(value: u32, align: u32) -> u32 { (value + (align - 1)) & !(align - 1) }
@@ -50,6 +51,14 @@ macro_rules! array_ref_mut {
     }};
 }
 
+/// Compile-time assertion.
+#[macro_export]
+macro_rules! static_assert {
+    ($condition:expr) => {
+        const _: () = core::assert!($condition);
+    };
+}
+
 pub trait IntoCow<'a, B>
 where B: ToOwned + ?Sized
 {