feat: release Dever 0.1.5 with direct workers and long transactions

crates/dever-process/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "dever-process"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+publish.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+goblin = { version = "=0.10.7", default-features = false, features = ["elf32", "elf64", "endian_fd", "std"] }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+rustix = { version = "=1.1.2", features = ["fs", "process"] }
+
+[[test]]
+name = "execution"
+path = "../../test/dever-process-tests/execution.rs"

crates/dever-process/src/assets.rs

@@ -0,0 +1,145 @@
+//! Validate the bootstrap dependency closure before executing any asset.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs;
+use std::io::Read;
+use std::path::Path;
+
+use goblin::elf::{Elf, header};
+
+const MAX_ASSET_BYTES: usize = 32 * 1024 * 1024;
+const MAX_ASSET_FILES: usize = 128;
+
+fn host_target() -> &'static str {
+    if cfg!(target_arch = "aarch64") {
+        "linux-aarch64"
+    } else {
+        "linux-x86_64"
+    }
+}
+
+fn target_abi(target: &str) -> Result<(u16, &'static str, &'static str), String> {
+    match target {
+        "linux-x86_64" => Ok((
+            header::EM_X86_64,
+            "ld-linux-x86-64.so.2",
+            "/lib64/ld-linux-x86-64.so.2",
+        )),
+        "linux-aarch64" => Ok((
+            header::EM_AARCH64,
+            "ld-linux-aarch64.so.1",
+            "/lib/ld-linux-aarch64.so.1",
+        )),
+        _ => Err(format!("unsupported process_runtime target '{target}'")),
+    }
+}
+
+/// Used both before signing a pack and when launching its verified extraction.
+/// Content authentication belongs to the release/bundle owner; this validates
+/// that the loader can resolve every dependency without a host-library search.
+pub fn validate_assets(files: &[(String, Vec<u8>)]) -> Result<(), String> {
+    validate_assets_for_target(files, host_target())
+}
+
+pub fn validate_assets_for_target(files: &[(String, Vec<u8>)], target: &str) -> Result<(), String> {
+    let (machine, loader_name, interpreter) = target_abi(target)?;
+    let total = files
+        .iter()
+        .try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len()))
+        .ok_or("process_runtime asset size overflow")?;
+    if files.len() > MAX_ASSET_FILES || total > MAX_ASSET_BYTES {
+        return Err("process_runtime assets exceed their file or byte budget".into());
+    }
+    let mut supplied = BTreeMap::new();
+    for (name, bytes) in files {
+        let valid = name.strip_prefix("lib/").is_some_and(|name| {
+            !name.is_empty() && name != "." && name != ".." && !name.contains(['/', '\\', '\0'])
+        });
+        if !valid || supplied.insert(name.as_str(), bytes.as_slice()).is_some() {
+            return Err("process_runtime asset has an invalid or repeated path".into());
+        }
+    }
+    let loader = format!("lib/{loader_name}");
+    if !supplied.contains_key(loader.as_str()) {
+        return Err(format!("process_runtime asset '{loader}' is missing"));
+    }
+    let mut reached = BTreeSet::new();
+    let mut remaining = supplied
+        .keys()
+        .map(|name| (*name).to_owned())
+        .collect::<Vec<_>>();
+    while let Some(name) = remaining.pop() {
+        if !reached.insert(name.clone()) {
+            continue;
+        }
+        let bytes = supplied
+            .get(name.as_str())
+            .ok_or_else(|| format!("process_runtime dependency '{name}' is missing"))?;
+        let elf = Elf::parse(bytes)
+            .map_err(|error| format!("invalid process_runtime ELF asset: {error}"))?;
+        if !elf.is_64
+            || !elf.little_endian
+            || elf.header.e_machine != machine
+            || !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
+        {
+            return Err(
+                "process_runtime ELF asset has an incompatible target or object type".into(),
+            );
+        }
+        // Reject paths that can outrank the explicit bootstrap library path.
+        if !elf.rpaths.is_empty() || !elf.runpaths.is_empty() {
+            return Err(
+                "process_runtime bootstrap assets must not declare RPATH or RUNPATH".into(),
+            );
+        }
+        if let Some(actual) = elf.interpreter
+            && actual != interpreter
+        {
+            return Err("process_runtime ELF interpreter differs from the packaged loader".into());
+        }
+        for library in elf.libraries {
+            if library.is_empty() || library.contains(['/', '\\']) || matches!(library, "." | "..")
+            {
+                return Err("process_runtime dependency is not a plain library name".into());
+            }
+            remaining.push(format!("lib/{library}"));
+        }
+    }
+    Ok(())
+}
+
+pub(super) fn verify_directory(root: &Path) -> Result<(), String> {
+    let mut files = Vec::new();
+    let mut remaining = MAX_ASSET_BYTES;
+    for directory in ["lib"] {
+        let directory_path = super::real_path(&root.join(directory))?;
+        for entry in fs::read_dir(directory_path).map_err(|error| error.to_string())? {
+            let entry = entry.map_err(|error| error.to_string())?;
+            let path = super::real_path(&entry.path())?;
+            if !entry
+                .file_type()
+                .map_err(|error| error.to_string())?
+                .is_file()
+                || files.len() >= MAX_ASSET_FILES
+            {
+                return Err(
+                    "process_runtime assets contain a non-regular file or too many files".into(),
+                );
+            }
+            let mut bytes = Vec::new();
+            fs::File::open(path)
+                .and_then(|file| file.take(remaining as u64 + 1).read_to_end(&mut bytes))
+                .map_err(|error| format!("cannot read process_runtime asset: {error}"))?;
+            remaining = remaining
+                .checked_sub(bytes.len())
+                .ok_or("process_runtime assets exceed their byte budget")?;
+            let name = entry
+                .file_name()
+                .into_string()
+                .map_err(|_| "process_runtime asset name is not UTF-8")?;
+            files.push((format!("{directory}/{name}"), bytes));
+        }
+    }
+    validate_assets(&files)?;
+    Ok(())
+}

crates/dever-process/src/lib.rs

@@ -0,0 +1,160 @@
+use std::ffi::OsString;
+use std::path::{Component, Path, PathBuf};
+use std::process::Command;
+
+mod assets;
+pub use assets::{validate_assets, validate_assets_for_target};
+
+pub fn asset_executable(path: &str) -> bool {
+    matches!(
+        path,
+        "lib/ld-linux-x86-64.so.2" | "lib/ld-linux-aarch64.so.1"
+    )
+}
+
+pub struct Launch<'a> {
+    pub assets: &'a Path,
+    pub worker: &'a Path,
+    pub executable: &'a Path,
+    pub arguments: &'a [OsString],
+    pub working_directory: &'a Path,
+}
+
+pub fn require_no_loader_preload() -> Result<(), String> {
+    match std::fs::symlink_metadata("/etc/ld.so.preload") {
+        Ok(_) => {
+            Err("private executable does not support host loader preload configuration".into())
+        }
+        Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
+        Err(error) => Err(format!(
+            "cannot inspect host loader preload configuration: {error}"
+        )),
+    }
+}
+
+impl Launch<'_> {
+    pub fn command(&self) -> Result<Command, String> {
+        if !cfg!(all(
+            target_os = "linux",
+            any(target_arch = "x86_64", target_arch = "aarch64")
+        )) {
+            return Err("managed external programs currently require Linux amd64 or arm64".into());
+        }
+        let assets = real_path(self.assets)?;
+        assets::verify_directory(&assets)?;
+        require_no_loader_preload()?;
+        let worker = real_path(self.worker)?;
+        let executable = real_path(self.executable)?;
+        let directory = real_path(self.working_directory)?;
+        if !executable.starts_with(&worker) || !directory.starts_with(&worker) {
+            return Err("managed execution path escapes its verified resource tree".into());
+        }
+        let bytes = std::fs::read(&executable).map_err(|error| error.to_string())?;
+        if !bytes.starts_with(b"#!/bin/sh\n") {
+            goblin::elf::Elf::parse(&bytes)
+                .map_err(|error| format!("invalid executable ELF: {error}"))?;
+        }
+        let mut paths = vec![worker.join("lib"), worker.join("runtime/lib")];
+        if let Some(inherited) = std::env::var_os("LD_LIBRARY_PATH") {
+            paths.extend(
+                std::env::split_paths(&inherited).filter(|path| !path.as_os_str().is_empty()),
+            );
+        }
+        let libraries = std::env::join_paths(paths).map_err(|error| error.to_string())?;
+        let mut command = Command::new(&executable);
+        command.args(self.arguments).current_dir(directory);
+        configure(&mut command);
+        command.env("LD_LIBRARY_PATH", libraries);
+        Ok(command)
+    }
+}
+
+pub fn configure(command: &mut Command) {
+    command.env_clear();
+    for name in [
+        "CUDA_VISIBLE_DEVICES",
+        "NVIDIA_VISIBLE_DEVICES",
+        "NVIDIA_DRIVER_CAPABILITIES",
+        "CUDA_DEVICE_ORDER",
+        "LD_LIBRARY_PATH",
+    ] {
+        if let Some(value) = std::env::var_os(name) {
+            command.env(name, value);
+        }
+    }
+    #[cfg(unix)]
+    {
+        use std::os::unix::process::CommandExt;
+        command.process_group(0);
+    }
+}
+
+pub fn loader_name() -> &'static str {
+    if cfg!(target_arch = "aarch64") {
+        "ld-linux-aarch64.so.1"
+    } else {
+        "ld-linux-x86-64.so.2"
+    }
+}
+
+fn real_path(path: &Path) -> Result<PathBuf, String> {
+    if !path.is_absolute()
+        || path
+            .components()
+            .any(|part| !matches!(part, Component::RootDir | Component::Normal(_)))
+    {
+        return Err("execution path must be absolute and normalized".into());
+    }
+    for ancestor in path.ancestors() {
+        let metadata = std::fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?;
+        if metadata.file_type().is_symlink() {
+            return Err("execution input must not contain symbolic links".into());
+        }
+    }
+    Ok(path.to_owned())
+}
+
+#[cfg(target_os = "linux")]
+pub struct ProcessGroup {
+    pid: rustix::process::Pid,
+    descriptor: std::os::fd::OwnedFd,
+}
+
+#[cfg(target_os = "linux")]
+impl ProcessGroup {
+    pub fn pin(pid: u32) -> Result<Self, String> {
+        use rustix::process::{Pid, PidfdFlags, pidfd_open};
+        let pid = i32::try_from(pid)
+            .ok()
+            .and_then(Pid::from_raw)
+            .ok_or("invalid child PID")?;
+        let descriptor =
+            pidfd_open(pid, PidfdFlags::NONBLOCK).map_err(|error| error.to_string())?;
+        Ok(Self { pid, descriptor })
+    }
+
+    pub fn exited(&self) -> Result<bool, String> {
+        use rustix::process::{WaitId, WaitIdOptions, waitid};
+        use std::os::fd::AsFd;
+        waitid(
+            WaitId::PidFd(self.descriptor.as_fd()),
+            WaitIdOptions::EXITED | WaitIdOptions::NOHANG | WaitIdOptions::NOWAIT,
+        )
+        .map(|status| status.is_some())
+        .map_err(|error| error.to_string())
+    }
+
+    pub fn terminate(&self) -> Result<(), String> {
+        match rustix::process::kill_process_group(self.pid, rustix::process::Signal::KILL) {
+            Ok(()) | Err(rustix::io::Errno::SRCH) => Ok(()),
+            Err(error) => Err(format!("cannot stop owned process group: {error}")),
+        }
+    }
+}
+
+#[cfg(target_os = "linux")]
+impl std::os::fd::AsRawFd for ProcessGroup {
+    fn as_raw_fd(&self) -> std::os::fd::RawFd {
+        std::os::fd::AsRawFd::as_raw_fd(&self.descriptor)
+    }
+}