feat: publish Dever core with project scaffolding and versioned AI skills

This commit is contained in:
Antigravity committed 2026-10-04 13:37:30 +00:00
1 parent 70fa73cdc2
commit ac458ffa7f
960 files changed
+221319 -1163

No files matched your search

+48
View File
@@ -0,0 +1,48 @@
use std::fs::{self, OpenOptions};
use std::io::{self, Write};
use std::path::Path;
use std::process::ExitCode;
use dever_core::hir::Program;
pub(super) fn check_baseline(program: &Program, root: &Path) -> Result<(), String> {
let path = root.join("dever.api");
let baseline = match fs::read_to_string(&path) {
Ok(baseline) => baseline,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(error) => {
return Err(format!(
"cannot read API baseline '{}': {error}",
path.display()
));
}
};
program
.check_api(&baseline)
.map_err(|error| format!("{}: {error}", path.display()))
}
pub(super) fn write(program: &Program, output: Option<&Path>) -> Result<ExitCode, String> {
let snapshot = program.api_snapshot();
match output {
None => io::stdout()
.lock()
.write_all(snapshot.as_bytes())
.map_err(|error| error.to_string())?,
Some(path) => {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|error| {
format!("cannot create API snapshot '{}': {error}", path.display())
})?;
file.write_all(snapshot.as_bytes())
.and_then(|()| file.sync_all())
.map_err(|error| {
format!("cannot write API snapshot '{}': {error}", path.display())
})?;
}
}
Ok(ExitCode::SUCCESS)
}
@@ -0,0 +1,44 @@
use std::ffi::OsString;
use std::io::{self, Write};
fn main() {
std::process::exit(run());
}
fn run() -> i32 {
let arguments: Vec<OsString> = std::env::args_os().skip(1).collect();
if arguments.as_slice() == ["--dever-bootstrap-health"] {
return 0;
}
#[cfg(target_os = "linux")]
if let [flag, root] = arguments.as_slice()
&& flag == "--dever-install"
{
return match dever_cli::toolchain::bootstrap::install(std::path::Path::new(root)) {
Ok(version) => {
println!("installed Dever {version}");
0
}
Err(error) => fail(error),
};
}
let executable = match std::env::current_exe() {
Ok(path) => path,
Err(error) => return fail(format!("cannot locate dever launcher: {error}")),
};
match dever_cli::toolchain::execute(&executable, &arguments) {
Ok(dever_cli::toolchain::CommandResult::Exited(code)) => code,
Ok(dever_cli::toolchain::CommandResult::Message(message)) => {
match writeln!(io::stdout().lock(), "{message}") {
Ok(()) => 0,
Err(error) => fail(format!("cannot write command output: {error}")),
}
}
Err(error) => fail(error),
}
}
fn fail(error: String) -> i32 {
let _ = writeln!(io::stderr().lock(), "{error}");
1
}
+34
View File
@@ -0,0 +1,34 @@
use std::env;
use std::path::PathBuf;
fn main() {
if env::args_os().skip(1).collect::<Vec<_>>().as_slice() == ["--dever-bootstrap-health"] {
return;
}
let mut arguments = env::args_os().skip(1);
let Some(flag) = arguments.next() else {
eprintln!("Usage: deverd --root <machine-toolchain-root>");
std::process::exit(2);
};
if flag != "--root" {
eprintln!("Usage: deverd --root <machine-toolchain-root>");
std::process::exit(2);
}
let Some(root) = arguments.next() else {
eprintln!("deverd: --root requires a machine toolchain root");
std::process::exit(2);
};
if arguments.next().is_some() {
eprintln!("deverd: unexpected arguments");
std::process::exit(2);
}
let layout = dever_cli::toolchain::Layout::new(PathBuf::from(root));
let result = layout
.initialize()
.and_then(|()| layout.validate_machine_permissions())
.and_then(|()| dever_cli::toolchain::serve(&layout));
if let Err(error) = result {
eprintln!("deverd: {error}");
std::process::exit(1);
}
}
+213
View File
@@ -0,0 +1,213 @@
//! The default compiler pipeline: checked HIR -> LLVM object -> verified LLD
//! inputs. Artifact lifetime, cache integrity and output publication stay in core.
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use dever_backend_bridge::{BinaryResource, LinkKind, Target, emit_object_with_resources, link};
use dever_core::hir::Program;
use dever_core::llvm::ResourceModule;
use dever_core::native::{EmbeddedResource, NativeProgram};
use dever_core::source::SourceMap;
use dever_runtime::config::{CompilationBindings, RuntimeProfile};
use sha2::{Digest, Sha256};
use dever_cli::toolchain::compilation::{CompileKind, CompileRequest};
use dever_cli::toolchain::runtime_pack::RuntimePack;
use dever_cli::toolchain::{BuildTarget, Layout, Version};
pub fn application(
program: &Program,
sources: &SourceMap,
bindings: &CompilationBindings,
resources: &[EmbeddedResource],
target: BuildTarget,
) -> Result<NativeProgram, String> {
if let Some((layout, version)) = managed_compiler()? {
let request = CompileRequest::new(
version,
CompileKind::Application,
sources,
Some(bindings.clone()),
resources,
target,
)?;
return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile(
&layout, &request,
)?);
}
let profile = bindings.profile();
compile(profile, target, || {
dever_core::llvm::emit_executable_with_resources(program, sources, resources)
})
}
pub fn test_suite(program: &Program, sources: &SourceMap) -> Result<NativeProgram, String> {
let target = BuildTarget::host()?;
if let Some((layout, version)) = managed_compiler()? {
let request =
CompileRequest::new(version, CompileKind::TestSuite, sources, None, &[], target)?;
return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile(
&layout, &request,
)?);
}
let profile = test_profile(program);
compile(profile, target, || {
dever_core::llvm::emit_test_executable(program, sources).map(|ir| ResourceModule {
ir,
resources: Vec::new(),
})
})
}
pub(super) fn compiler_directory() -> Result<PathBuf, String> {
let executable =
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
executable
.parent()
.map(Path::to_owned)
.ok_or_else(|| "compiler has no parent directory".into())
}
pub(super) fn managed_compiler() -> Result<Option<(Layout, String)>, String> {
let executable =
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
let directory = executable
.parent()
.ok_or("compiler has no parent directory")?;
let managed = executable.file_stem().and_then(|name| name.to_str()) == Some("dever-core")
|| directory
.parent()
.and_then(Path::file_name)
.is_some_and(|name| name == "versions");
if !managed {
return Ok(None);
}
let versions = directory
.parent()
.filter(|path| path.file_name().is_some_and(|name| name == "versions"))
.ok_or(
"managed compiler must be installed under <toolchain>/versions/<version>/dever-core",
)?;
if executable.file_stem().and_then(|name| name.to_str()) != Some("dever-core") {
return Err("managed compiler must use the installed dever-core executable".into());
}
let version = directory
.file_name()
.and_then(|name| name.to_str())
.ok_or("invalid managed compiler version directory")?;
Version::parse(version)?;
if version != env!("CARGO_PKG_VERSION") {
return Err("managed compiler version does not match its installation directory".into());
}
let root = versions
.parent()
.ok_or("managed compiler has no toolchain root")?;
Ok(Some((Layout::new(root), version.to_owned())))
}
fn test_profile(program: &Program) -> RuntimeProfile {
RuntimeProfile {
sqlite: program.tests().iter().any(|test| test.uses_database()),
postgres: false,
}
}
pub(super) fn worker_compile(
program: &Program,
sources: &SourceMap,
request: &CompileRequest,
build: &Path,
output: &Path,
) -> Result<(), String> {
let resources = request.embedded_resources();
let (profile, ir) = match request.kind {
CompileKind::Application => {
let profile = program.validate_database_bindings(
request
.bindings
.as_ref()
.ok_or("application bindings are missing")?,
)?;
let ir =
dever_core::llvm::emit_executable_with_resources(program, sources, &resources)?;
(profile, ir)
}
CompileKind::TestSuite => (
test_profile(program),
ResourceModule {
ir: dever_core::llvm::emit_test_executable(program, sources)?,
resources: Vec::new(),
},
),
};
let pack = RuntimePack::load(&compiler_directory()?, profile, request.target)?;
link_ir(&ir, &pack, build, output)
}
fn compile<'a>(
profile: RuntimeProfile,
target: BuildTarget,
emit: impl FnOnce() -> Result<ResourceModule<'a>, String>,
) -> Result<NativeProgram, String> {
let executable =
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
let directory = compiler_directory()?;
let pack = RuntimePack::load(&directory, profile, target)?;
let ir = emit()?;
let compiler = dever_cli::toolchain::sha256_file(&executable)?;
let mut identity = Sha256::new();
for part in [
"dever-llvm-executable-v1",
&compiler,
&pack.identity,
target.platform(),
&ir.ir,
] {
identity.update((part.len() as u64).to_le_bytes());
identity.update(part.as_bytes());
}
dever_core::native::compile_artifact(
&directory.join("cache/native"),
&format!("{:x}", identity.finalize()),
|build, output| link_ir(&ir, &pack, build, output),
)
}
fn link_ir(
module: &ResourceModule<'_>,
pack: &RuntimePack,
build: &Path,
output: &Path,
) -> Result<(), String> {
let target = Target::ALL
.into_iter()
.find(|target| target.triple() == pack.target.triple())
.ok_or("native runtime target is not supported by the compiler bridge")?;
let resources = module
.resources
.iter()
.map(|resource| BinaryResource {
symbol: &resource.symbol,
bytes: resource.bytes,
})
.collect::<Vec<_>>();
let object = emit_object_with_resources(&module.ir, target, &resources)?;
let path = build.join("application.o");
fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.and_then(|mut file| file.write_all(&object))
.map_err(|error| format!("cannot write application object: {error}"))?;
let snapshot = pack.snapshot(build)?;
let inputs = snapshot
.start
.into_iter()
.chain(std::iter::once(path))
.chain(snapshot.libraries)
.chain(snapshot.end)
.collect::<Vec<_>>();
link(target, LinkKind::StaticExecutable, &inputs, output, None)
}
+153
View File
@@ -0,0 +1,153 @@
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::ExitCode;
use std::sync::atomic::{AtomicUsize, Ordering};
use dever_core::source::SourceMap;
pub(super) fn execute(root: &Path, sources: &SourceMap, check: bool) -> Result<ExitCode, String> {
let mut changes = Vec::new();
let mut diagnostics = Vec::new();
for source in sources.files() {
match dever_core::format::format(source) {
Ok(formatted) if formatted != source.text() => {
changes.push((source.path(), destination(root, source), formatted));
}
Ok(_) => {}
Err(errors) => diagnostics.extend(errors),
}
}
if !diagnostics.is_empty() {
return Err(diagnostics
.iter()
.map(|error| error.render(sources))
.collect());
}
if check {
for (path, _, _) in &changes {
writeln!(std::io::stdout().lock(), "{}", path.display())
.map_err(|error| error.to_string())?;
}
return Ok(if changes.is_empty() {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
});
}
// Prepare all replacements before changing any source. A syntax, permission or staging
// failure leaves the original files intact; each final replacement is an atomic rename.
validate_snapshot(root, sources)?;
let mut staged = Vec::new();
for (_, destination, formatted) in changes {
staged.push(Replacement::prepare(destination, &formatted)?);
}
validate_snapshot(root, sources)?;
for replacement in &mut staged {
replacement.commit()?;
}
Ok(ExitCode::SUCCESS)
}
fn validate_snapshot(root: &Path, sources: &SourceMap) -> Result<(), String> {
// Reuse loading rules so added/deleted paths and new symlinks also invalidate the snapshot.
let current =
SourceMap::load_project(&root.join("module"), &root.join("test")).map_err(|errors| {
errors
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n")
})?;
if !current
.files()
.iter()
.map(|source| source.path())
.eq(sources.files().iter().map(|source| source.path()))
{
return Err(format!(
"{}: source file set changed during formatting",
root.display()
));
}
for (current, source) in current.files().iter().zip(sources.files()) {
if current.text() != source.text() {
return Err(format!(
"{}: source changed during formatting",
destination(root, source).display()
));
}
}
Ok(())
}
fn destination(root: &Path, source: &dever_core::source::SourceFile) -> PathBuf {
if source.is_test() {
root.join(source.path())
} else {
root.join("module").join(source.path())
}
}
struct Replacement {
destination: PathBuf,
temporary: Option<PathBuf>,
}
impl Replacement {
fn prepare(destination: PathBuf, formatted: &str) -> Result<Self, String> {
static NEXT: AtomicUsize = AtomicUsize::new(0);
let metadata =
fs::metadata(&destination).map_err(|error| file_error(&destination, error))?;
if metadata.permissions().readonly() {
return Err(format!("{}: source is read-only", destination.display()));
}
let parent = destination.parent().expect("source files have a parent");
let (temporary, mut file) = loop {
let temporary = parent.join(format!(
".dever-fmt-{}-{}.tmp",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
match OpenOptions::new()
.write(true)
.create_new(true)
.open(&temporary)
{
Ok(file) => break (temporary, file),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(file_error(&destination, error)),
}
};
let replacement = Self {
destination,
temporary: Some(temporary),
};
file.write_all(formatted.as_bytes())
.and_then(|()| file.set_permissions(metadata.permissions()))
.and_then(|()| file.sync_all())
.map_err(|error| file_error(&replacement.destination, error))?;
Ok(replacement)
}
fn commit(&mut self) -> Result<(), String> {
let temporary = self.temporary.as_ref().expect("replacement is pending");
fs::rename(temporary, &self.destination)
.map_err(|error| file_error(&self.destination, error))?;
self.temporary = None;
Ok(())
}
}
impl Drop for Replacement {
fn drop(&mut self) {
if let Some(temporary) = &self.temporary {
let _ = fs::remove_file(temporary);
}
}
}
fn file_error(path: &Path, error: std::io::Error) -> String {
format!("{}: cannot format source: {error}", path.display())
}
+5
View File
@@ -0,0 +1,5 @@
pub mod libs;
pub mod packages;
pub mod project;
pub mod toolchain;
pub mod workers;
File diff suppressed because it is too large. Load diff
+526
View File
@@ -0,0 +1,526 @@
//! Explicit Lib preparation: shared owned staging and bounded tool execution.
mod npm;
pub mod pack;
pub(crate) mod process;
mod python;
use super::{Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryRuntime, RuntimePack};
use serde::{Deserialize, Serialize};
use std::cell::RefCell;
use std::collections::{BTreeMap, BTreeSet};
fn require_execution_target(target: &str, operation: &str) -> Result<(), String> {
if target != crate::toolchain::BuildTarget::host()?.platform() {
return Err(format!(
"{operation} for {target} requires prepared target output and a complete build receipt; prepare the signed target inputs in a matching isolated execution environment before the offline build"
));
}
Ok(())
}
/// Production supplies only authenticated release resources. Explicit author
/// tests inject the same immutable inputs without creating a host-tool fallback.
pub trait Inputs {
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String>;
fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String>;
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String>;
/// Author tools may display bounded hook output without retaining temporary
/// staging trees or putting diagnostics in reproducible build identities.
fn diagnostic(&self, _output: &[u8]) {}
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PythonSystem {
pub requires: Vec<String>,
pub backend: String,
pub backend_path: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NativeRejection {
Format,
Target,
Closure,
Unloadable,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Receipt {
pub ecosystem: Ecosystem,
pub roots: Vec<LibSpec>,
pub sources: Vec<LockedArtifact>,
pub graph: Option<super::npm::Environment>,
pub runtime: RuntimePack,
pub auxiliary_runtime: Option<RuntimePack>,
pub native_entries: BTreeSet<String>,
pub native_rejections: BTreeMap<String, NativeRejection>,
pub python_markers: Option<pep508_rs::MarkerEnvironment>,
pub tools: pack::Descriptor,
pub frontend: String,
pub python: Option<PythonSystem>,
pub dynamic_requires: Vec<String>,
pub dependencies: Box<LockFile>,
pub config_settings: BTreeMap<String, serde_json::Value>,
pub output: LockedArtifact,
}
impl Receipt {
pub fn identity(&self) -> Result<String, String> {
serde_json::to_vec(self)
.map(|bytes| super::sha256(&bytes))
.map_err(|error| error.to_string())
}
}
#[derive(Clone)]
pub(crate) struct Product {
pub bytes: Vec<u8>,
pub filename: String,
pub receipt: Receipt,
}
pub struct Session<'a> {
inputs: &'a dyn Inputs,
active: RefCell<BTreeSet<String>>,
products: RefCell<BTreeMap<String, Product>>,
}
impl<'a> Session<'a> {
pub fn new(inputs: &'a dyn Inputs) -> Self {
Self {
inputs,
active: RefCell::new(BTreeSet::new()),
products: RefCell::new(BTreeMap::new()),
}
}
pub(crate) fn receipts(&self, identities: &BTreeSet<String>) -> Result<Vec<Receipt>, String> {
let receipts = self
.products
.borrow()
.values()
.map(|product| Ok((product.receipt.identity()?, product.receipt.clone())))
.collect::<Result<BTreeMap<_, _>, String>>()?;
identities
.iter()
.map(|identity| {
receipts
.get(identity)
.cloned()
.ok_or("prepared build receipt is missing".into())
})
.collect()
}
fn remember(&self, key: String, product: Product) -> Result<(), String> {
let total = self
.products
.borrow()
.values()
.try_fold(product.bytes.len(), |total, cached| {
total.checked_add(cached.bytes.len())
})
.ok_or("build candidate cache byte overflow")?;
if total > crate::workers::MAX_TREE_BYTES {
return Err("build candidate cache exceeds byte budget".into());
}
self.products.borrow_mut().insert(key, product);
Ok(())
}
}
pub fn normalize(receipts: &mut Vec<Receipt>) -> Result<(), String> {
let mut indexed = BTreeMap::new();
for receipt in receipts.drain(..) {
indexed.insert(receipt.identity()?, receipt);
}
*receipts = indexed.into_values().collect();
Ok(())
}
pub fn npm_outputs(lock: &LockFile) -> impl Iterator<Item = &Receipt> {
lock.builds
.iter()
.filter(|receipt| receipt.ecosystem == Ecosystem::Npm)
}
pub(crate) fn npm_receipt<'a>(
lock: &'a LockFile,
environment: &super::npm::Environment,
) -> Result<Option<&'a Receipt>, String> {
let Some(identity) = &environment.build else {
return Ok(None);
};
lock.builds
.iter()
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
.map(Some)
.ok_or("npm installation receipt is missing".into())
}
/// Offline consumption never invokes a frontend. Removing roots projects the
/// already built graph and output; it does not rerun lifecycle side effects.
pub fn npm_installation(
lock: &LockFile,
environment: &super::npm::Environment,
bytes: &[u8],
) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
let receipt = npm_receipt(lock, environment)?.ok_or("npm installation has no build receipt")?;
if bytes.len() as u64 != receipt.output.bytes || super::sha256(bytes) != receipt.output.sha256 {
return Err("npm built installation differs from its locked artifact".into());
}
let graph = receipt
.graph
.as_ref()
.ok_or("npm receipt lacks its exact graph")?;
let files = super::archive_files("tgz", bytes)?;
super::npm::validate_built_files(graph, &files)?;
let candidates = files
.iter()
.filter(|(path, _)| path.ends_with(".node"))
.map(|(path, _)| path.clone())
.collect::<BTreeSet<_>>();
let accounted = receipt
.native_entries
.iter()
.chain(receipt.native_rejections.keys())
.cloned()
.collect::<BTreeSet<_>>();
if candidates != accounted
|| receipt
.native_rejections
.keys()
.any(|path| receipt.native_entries.contains(path))
{
return Err("npm native admission does not cover its exact addon files".into());
}
for path in &receipt.native_entries {
if !files.iter().any(|(name, bytes)| {
name == path && path.ends_with(".node") && bytes.starts_with(b"\x7fELF")
}) {
return Err("npm locked native entry point is absent from output".into());
}
}
let executable = super::registry::archive_executables("tgz", bytes)?;
let retained = environment
.instances
.iter()
.map(|node| node.path.as_str())
.collect::<BTreeSet<_>>();
let mut output = Vec::new();
for (path, bytes) in files {
if path.ends_with(".node") && !receipt.native_entries.contains(&path) {
continue;
}
if retained.contains(super::npm::installed_owner(&path)?.as_str()) {
let mode = executable.contains(&path);
output.push((path, bytes, mode));
}
}
Ok(output)
}
pub fn doctor(lock: &LockFile) -> Result<(), String> {
doctor_at(lock, 0)
}
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
if depth > 16 {
return Err("build dependency depth exceeds limit".into());
}
let records = lock
.builds
.iter()
.map(|receipt| Ok((receipt.identity()?, receipt)))
.collect::<Result<BTreeMap<_, _>, String>>()?;
if records.len() != lock.builds.len() {
return Err("duplicate build receipt".into());
}
let mut used = BTreeSet::new();
for lib in &lock.libs {
if let Some(identity) = &lib.build {
let receipt = records
.get(identity)
.ok_or("Python Lib has no exact build receipt")?;
if lib.spec.ecosystem != Ecosystem::Pip
|| receipt.ecosystem != Ecosystem::Pip
|| receipt.runtime != lib.runtime
|| !lib.artifacts.contains(&receipt.output)
|| receipt.roots.len() != 1
|| receipt.roots[0].distribution_name() != lib.spec.distribution_name()
|| receipt.roots[0].version != lib.spec.version
{
return Err(
"Python Lib build receipt differs from locked artifact identity".into(),
);
}
used.insert(identity.clone());
}
}
for environment in &lock.npm {
if let Some(identity) = &environment.build {
let receipt = records
.get(identity)
.ok_or("npm environment has no exact build receipt")?;
let graph = receipt
.graph
.as_ref()
.ok_or("npm build receipt lacks its graph")?;
if receipt.ecosystem != Ecosystem::Npm
|| receipt.roots != graph.roots
|| graph.build.is_some()
|| !environment
.roots
.iter()
.all(|root| graph.roots.contains(root))
|| !environment
.instances
.iter()
.all(|node| graph.instances.contains(node))
|| !environment.instances.iter().all(|node| {
lock.libs.iter().any(|lib| {
lib.spec == node.source.archive
&& lib.runtime == receipt.runtime
&& lib
.artifacts
.iter()
.all(|artifact| receipt.sources.contains(artifact))
})
})
{
return Err("npm installation build receipt differs from locked graph".into());
}
used.insert(identity.clone());
}
}
if used.len() != records.len() {
return Err("lock contains an unreferenced build receipt".into());
}
for receipt in records.values() {
super::validate_runtime(&receipt.runtime)?;
if receipt.tools.runtime_sha256 != receipt.runtime.sha256
|| receipt.tools.ecosystem != receipt.ecosystem
|| receipt.sources.is_empty()
|| receipt.output.target != receipt.tools.target
|| receipt.roots.is_empty()
|| receipt.frontend != frontend(&receipt.ecosystem)?
{
return Err("build receipt has inconsistent tools, frontend, or target".into());
}
for artifact in receipt.sources.iter().chain([&receipt.output]) {
super::validate_artifact(artifact)?;
if artifact.target != receipt.tools.target {
return Err("build artifact target differs from its tools".into());
}
}
if receipt.ecosystem == Ecosystem::Pip
&& (receipt.python.is_none()
|| receipt.graph.is_some()
|| receipt.auxiliary_runtime.is_some())
{
return Err("Python build receipt lacks its backend configuration".into());
}
if receipt.ecosystem == Ecosystem::Pip
&& (!receipt.native_entries.is_empty() || !receipt.native_rejections.is_empty())
{
return Err("Python receipt cannot declare Node native entry points".into());
}
for path in receipt
.native_entries
.iter()
.chain(receipt.native_rejections.keys())
{
crate::workers::valid_path(path)?;
}
let policy = RegistryRuntime {
pack: receipt.runtime.clone(),
target: receipt.tools.target.clone(),
python_markers: receipt.python_markers.clone(),
python_wheel_tags: vec![],
npm_libc: None,
};
let auxiliary = receipt
.auxiliary_runtime
.as_ref()
.map(|runtime| RegistryRuntime {
pack: runtime.clone(),
target: receipt.tools.target.clone(),
python_markers: None,
python_wheel_tags: vec![],
npm_libc: None,
});
if let Some(runtime) = &receipt.auxiliary_runtime {
super::validate_runtime(runtime)?;
}
receipt.tools.validate(&policy, auxiliary.as_ref())?;
if receipt.ecosystem == Ecosystem::Npm
&& (receipt.python.is_some()
|| receipt.python_markers.is_some()
|| !receipt.dynamic_requires.is_empty()
|| !receipt.dependencies.libs.is_empty()
|| receipt.graph.is_none())
{
return Err("npm build receipt contains Python backend state".into());
}
if !receipt.config_settings.is_empty() {
return Err("unsupported source-build configuration settings".into());
}
if let Some(system) = &receipt.python {
validate_requirements(receipt, system)?;
}
if !receipt.dependencies.workers.is_empty()
|| !receipt.dependencies.packages.is_empty()
|| receipt
.dependencies
.libs
.iter()
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
{
return Err("build dependency closure is not a separate Python environment".into());
}
doctor_at(&receipt.dependencies, depth + 1)?;
// Full graph checks belong to the ordinary lock owner; recursion is
// bounded above before invoking that owner on a dependency lock.
let mut dependencies = (*receipt.dependencies).clone();
dependencies.builds.clear();
for lib in &mut dependencies.libs {
lib.build = None;
}
super::doctor(&dependencies)?;
}
Ok(())
}
fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(), String> {
let identifier = |path: &str| {
!path.is_empty()
&& path.split('.').all(|part| {
let mut bytes = part.bytes();
bytes
.next()
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
})
};
let (module, function) = system
.backend
.split_once(':')
.map(|(module, function)| (module, Some(function)))
.unwrap_or((&system.backend, None));
if !identifier(module)
|| function.is_some_and(|function| !identifier(function))
|| system.requires.len() > 1024
|| receipt.dynamic_requires.len() > 1024
|| system.backend_path.len() > 64
{
return Err("build receipt has an invalid backend or requirement set".into());
}
for path in &system.backend_path {
if path.is_empty() || path.starts_with('/') || path.contains(['\\', '\0']) {
return Err("invalid backend-path in receipt".into());
}
let mut depth = 0usize;
for part in path.split('/') {
match part {
"" | "." => {}
".." => {
depth = depth.checked_sub(1).ok_or("backend-path escapes source")?;
}
_ => depth += 1,
}
}
}
let markers = receipt
.python_markers
.as_ref()
.ok_or("Python build receipt lacks marker policy")?;
let requirements = system
.requires
.iter()
.chain(&receipt.dynamic_requires)
.cloned()
.collect::<Vec<_>>();
let document = serde_json::json!({"info":{"requires_dist":requirements}});
let expected = super::python_dependencies(&document, markers, &[])?;
let libraries = receipt
.dependencies
.libs
.iter()
.map(|lib| (lib.spec.name.as_str(), lib))
.collect::<BTreeMap<_, _>>();
let mut remaining = Vec::new();
for (name, rule) in expected {
let lib = libraries
.get(name.as_str())
.ok_or("Python build receipt omits a required dependency")?;
let version = lib
.spec
.version
.parse::<pep440_rs::Version>()
.map_err(|error| error.to_string())?;
if !rule.contains(&version) {
return Err("Python build dependency does not satisfy recorded requirements".into());
}
remaining.push(lib.spec.name.as_str());
}
let mut reached = BTreeSet::new();
while let Some(name) = remaining.pop() {
if !reached.insert(name) {
continue;
}
let lib = libraries
.get(name)
.ok_or("build dependency closure is incomplete")?;
if lib.runtime != receipt.runtime
|| lib
.artifacts
.iter()
.any(|artifact| artifact.target != receipt.tools.target)
{
return Err("build dependency runtime or target differs from its receipt".into());
}
remaining.extend(lib.dependencies.iter().map(|edge| edge.spec.name.as_str()));
}
if reached.len() != libraries.len() {
return Err("build dependency lock contains unrelated libraries".into());
}
Ok(())
}
/// Offline consumers bind recorded marker evaluation to the immutable runtime
/// manifest they have just verified, never to a lock's self-declared policy.
pub fn validate_python_policy(
lock: &LockFile,
lib: &super::LockedLib,
markers: &pep508_rs::MarkerEnvironment,
) -> Result<(), String> {
if let Some(identity) = &lib.build {
let receipt = lock
.builds
.iter()
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
.ok_or("Python build receipt missing")?;
if receipt.python_markers.as_ref() != Some(markers) {
return Err("Python build marker policy differs from signed runtime".into());
}
}
Ok(())
}
fn frontend(ecosystem: &Ecosystem) -> Result<String, String> {
match ecosystem {
Ecosystem::Pip => Ok(super::sha256(include_bytes!("build/python.py"))),
Ecosystem::Npm => Ok(super::sha256(
&[
include_bytes!("build/npm.js").as_slice(),
include_bytes!("build/npm/probe.cjs").as_slice(),
]
.concat(),
)),
_ => Err("source-build frontend is not available for this ecosystem".into()),
}
}
+103
View File
@@ -0,0 +1,103 @@
// Fixed registry-package lifecycle frontend. Runs only in the build sandbox.
const fs = require('node:fs');
const path = require('node:path');
const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script');
const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links');
async function main() {
const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8'));
const root = '/data/build/output/install';
fs.mkdirSync('/data/build/output');
fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true });
fs.mkdirSync('/data/build/tmp');
fs.mkdirSync('/data/build/home');
Object.assign(process.env, {
PATH: '/worker/runtime/bin:/usr/bin',
CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar',
PYTHON: request.python,
npm_config_python: request.python,
npm_config_nodedir: '/worker/npm',
npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js',
npm_config_build_from_source: 'true',
npm_config_cache: '/data/build/npm-cache',
TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0',
HOME: '/data/build/home',
});
const nodes = new Map(request.instances.map(node => [node.path, node]));
for (const node of request.instances) {
const directory = path.join(root, node.path);
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8'));
await binLinks({ path: directory, pkg, global: false, top: false, force: false });
}
const visited = new Set();
const required = new Set(request.required);
const failed = new Set();
const nativeEntries = new Set();
async function install(name) {
if (visited.has(name)) return;
visited.add(name);
const node = nodes.get(name);
try {
for (const edge of node.edges) {
if (edge.target && nodes.has(edge.target)) {
await install(edge.target);
if (failed.has(edge.target) && ['dependency', 'peer'].includes(edge.kind)) {
throw new Error('required lifecycle dependency failed: ' + edge.target);
}
}
}
const directory = path.join(root, name);
for (const event of ['preinstall', 'install', 'postinstall']) {
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' });
}
const native = path.join(directory, 'build', 'Release');
if (fs.existsSync(native)) {
for (const file of fs.readdirSync(native)) {
if (file.endsWith('.node')) {
const addon = path.join(native, file);
require(addon);
nativeEntries.add(path.relative(root, addon));
}
}
}
} catch (error) {
if (required.has(name)) throw error;
process.stderr.write('optional npm lifecycle failed: ' + name + ': ' + String(error.message || error) + '\n');
failed.add(name);
}
}
for (const node of request.instances) await install(node.path);
let changed = true;
while (changed) {
changed = false;
for (const node of request.instances) {
if (!failed.has(node.path) && node.edges.some(edge => ['dependency', 'peer'].includes(edge.kind) && failed.has(edge.target))) {
if (required.has(node.path)) throw new Error('required lifecycle dependency failed: ' + node.path);
failed.add(node.path);
changed = true;
}
}
}
const retained = new Set();
function retain(name) {
if (retained.has(name) || failed.has(name)) return;
retained.add(name);
for (const edge of nodes.get(name).edges) {
if (edge.target) retain(edge.target);
}
}
for (const node of request.roots) retain(node);
for (const node of request.instances) {
if (!retained.has(node.path)) fs.rmSync(path.join(root, node.path), { recursive: true, force: true });
}
// Bin links are a build-time PATH input. The immutable Worker tree stores
// package files only; it never persists symlinks created by installer hooks.
for (const node of request.instances) {
fs.rmSync(path.join(root, node.path, 'node_modules', '.bin'), { recursive: true, force: true });
fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true });
}
fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true });
fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
}
main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; });
+230
View File
@@ -0,0 +1,230 @@
//! Registry lifecycle scripts use the signed npm frontend and exact instance graph.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::io::Cursor;
use super::{Product, Receipt, Session};
use crate::libs::{
Ecosystem, LockFile, LockedArtifact, LockedLib, RegistryResolver, RegistryRuntime, npm, sha256,
};
mod environment;
use environment::{Environment, Output};
impl Session<'_> {
pub(crate) fn npm(
&self,
resolver: &RegistryResolver<'_>,
libraries: &mut Vec<LockedLib>,
graph: &mut npm::Environment,
runtime: &RegistryRuntime,
) -> Result<(), String> {
let mut lock = LockFile::new(libraries.clone())?;
lock.npm.push(graph.clone());
let archives = libraries
.iter()
.map(|lib| {
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
Ok((
lib.spec.clone(),
resolver.store.verify_exact(
&artifact.sha256,
artifact.bytes,
&artifact.target,
)?,
))
})
.collect::<Result<BTreeMap<_, _>, String>>()?;
let files = npm::build_files(graph, &lock, |lib| {
archives
.get(&lib.spec)
.map(Vec::as_slice)
.ok_or("npm source archive missing".into())
})?;
if !npm::requires_build(&files)? {
return Ok(());
}
let mut sources = libraries
.iter()
.flat_map(|lib| lib.artifacts.clone())
.collect::<Vec<_>>();
sources.sort();
sources.dedup();
let key = format!(
"npm:{}:{}:{}",
sha256(&serde_json::to_vec(graph).map_err(|error| error.to_string())?),
runtime.pack.sha256,
sha256(&serde_json::to_vec(&sources).map_err(|error| error.to_string())?)
);
if let Some(product) = self.products.borrow().get(&key) {
*graph = product
.receipt
.graph
.clone()
.ok_or("cached npm graph missing")?;
graph.build = Some(product.receipt.identity()?);
retain_archives(libraries, graph);
return Ok(());
}
let executables = npm::build_executables(graph, &archives)?;
let environment = Environment::new(self.inputs, runtime)?;
let output = environment.run(self.inputs, graph, &files, &executables)?;
prune_failed(graph, &output.failed)?;
retain_archives(libraries, graph);
npm::validate_built_files(graph, &output.files)?;
let bytes = archive(&output)?;
let digest = resolver.store.publish(&bytes, &runtime.target)?;
if digest != sha256(&bytes) {
return Err("built npm cache identity mismatch".into());
}
let output_artifact = LockedArtifact {
target: runtime.target.clone(),
path: format!("build/npm/{digest}/installation.tgz"),
bytes: bytes.len() as u64,
sha256: digest,
};
let receipt = Receipt {
ecosystem: Ecosystem::Npm,
roots: graph.roots.clone(),
sources,
graph: Some(graph.clone()),
runtime: runtime.pack.clone(),
auxiliary_runtime: Some(environment.python_runtime),
native_entries: output.native,
native_rejections: output.rejections,
python_markers: None,
tools: environment.descriptor,
frontend: super::frontend(&Ecosystem::Npm)?,
python: None,
dynamic_requires: vec![],
dependencies: Box::new(LockFile::new(vec![])?),
config_settings: BTreeMap::new(),
output: output_artifact,
};
graph.build = Some(receipt.identity()?);
self.remember(
key,
Product {
bytes,
filename: "installation.tgz".into(),
receipt,
},
)?;
Ok(())
}
}
fn archive(output: &Output) -> Result<Vec<u8>, String> {
let encoder = flate2::GzBuilder::new()
.mtime(0)
.operating_system(255)
.write(Vec::new(), flate2::Compression::default());
let mut archive = tar::Builder::new(encoder);
for (path, bytes) in &output.files {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(if output.executables.contains(path) {
0o755
} else {
0o644
});
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_cksum();
archive
.append_data(&mut header, path, Cursor::new(bytes))
.map_err(|error| error.to_string())?;
}
archive
.into_inner()
.and_then(|encoder| encoder.finish())
.map_err(|error| error.to_string())
}
fn retain_archives(libraries: &mut Vec<LockedLib>, graph: &npm::Environment) {
let used = graph
.instances
.iter()
.map(|node| &node.source.archive)
.collect::<BTreeSet<_>>();
libraries.retain(|lib| used.contains(&lib.spec));
}
fn required_nodes(graph: &npm::Environment) -> Result<BTreeSet<String>, String> {
reachable(graph, true)
}
fn reachable(graph: &npm::Environment, mandatory_only: bool) -> Result<BTreeSet<String>, String> {
let mut remaining = graph
.roots
.iter()
.map(|root| format!("node_modules/{}", root.name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = remaining.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = graph
.instances
.iter()
.find(|node| node.path == path)
.ok_or("npm reachable instance missing")?;
remaining.extend(
node.edges
.iter()
.filter(|edge| {
!mandatory_only || matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
})
.filter_map(|edge| edge.target.clone()),
);
}
Ok(reached)
}
fn prune_failed(graph: &mut npm::Environment, failed: &BTreeSet<String>) -> Result<(), String> {
if failed
.iter()
.any(|path| !graph.instances.iter().any(|node| &node.path == path))
{
return Err("npm build reported an unknown failed instance".into());
}
let mut failed = failed.clone();
loop {
let before = failed.len();
for node in &graph.instances {
if node.edges.iter().any(|edge| {
matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
&& edge
.target
.as_ref()
.is_some_and(|path| failed.contains(path))
}) {
failed.insert(node.path.clone());
}
}
if before == failed.len() {
break;
}
}
if !required_nodes(graph)?.is_disjoint(&failed) {
return Err("required npm lifecycle failed".into());
}
graph.instances.retain(|node| !failed.contains(&node.path));
for node in &mut graph.instances {
for edge in &mut node.edges {
if edge
.target
.as_ref()
.is_some_and(|path| failed.contains(path))
{
edge.target = None;
edge.omission = Some(npm::Omission::BuildFailure);
}
}
}
let reached = reachable(graph, false)?;
graph.instances.retain(|node| reached.contains(&node.path));
Ok(())
}
@@ -0,0 +1,326 @@
//! One immutable Node/Python/tool environment for registry lifecycle execution.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use crate::libs::build::NativeRejection;
use crate::libs::build::{Inputs, pack, process::Stage};
use crate::libs::{Ecosystem, RegistryRuntime, RuntimePack, archive_files, npm, sha256};
use crate::workers::runtime;
pub(super) struct Environment {
files: BTreeMap<String, (Vec<u8>, bool)>,
assets: Vec<(String, Vec<u8>)>,
node: String,
node_paths: BTreeSet<String>,
python: String,
pub python_runtime: RuntimePack,
pub descriptor: pack::Descriptor,
}
pub(super) struct Output {
pub files: Vec<(String, Vec<u8>)>,
pub executables: BTreeSet<String>,
pub failed: BTreeSet<String>,
pub native: BTreeSet<String>,
pub rejections: BTreeMap<String, NativeRejection>,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ResultMetadata {
failed: BTreeSet<String>,
native: BTreeSet<String>,
}
impl Environment {
pub fn new(inputs: &dyn Inputs, runtime: &RegistryRuntime) -> Result<Self, String> {
crate::libs::build::require_execution_target(&runtime.target, "npm lifecycle build")?;
let (node, bytes) = inputs.runtime(Ecosystem::Npm)?;
if node.pack != runtime.pack || sha256(&bytes) != runtime.pack.sha256 {
return Err("npm build runtime differs from resolver".into());
}
let node_files = archive_files("tgz", &bytes)?;
let node_manifest = runtime::validate(&node_files, &Ecosystem::Npm, &runtime.target)?;
let node_paths = node_files
.iter()
.map(|(path, _)| format!("runtime/{path}"))
.collect();
let node = format!(
"runtime/{}",
node_manifest
.executable
.as_deref()
.ok_or("Node executable missing")?
);
let (python, bytes) = inputs.runtime(Ecosystem::Pip)?;
if sha256(&bytes) != python.pack.sha256 {
return Err("auxiliary Python runtime hash mismatch".into());
}
let python_files = archive_files("tgz", &bytes)?;
let python_manifest = runtime::validate(&python_files, &Ecosystem::Pip, &runtime.target)?;
let python_executable = format!(
"runtime/{}",
python_manifest
.executable
.as_deref()
.ok_or("build Python executable missing")?
);
let (descriptor, bytes) = inputs.tools(Ecosystem::Npm)?;
descriptor.validate(runtime, Some(&python))?;
let (manifest, tools) = pack::unpack(&bytes, &descriptor)?;
let assets = inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
pack::validate_tools(&manifest, &tools, &assets)?;
let mut files = BTreeMap::new();
for (path, bytes) in node_files {
let executable = node_manifest.executable(&path);
files.insert(format!("runtime/{path}"), (bytes, executable));
}
for (path, bytes) in python_files {
if path == "dever-runtime.json" {
continue;
}
let executable = python_manifest.executable(&path);
let path = format!("runtime/{path}");
if let Some((existing, _)) = files.get(&path) {
if existing != &bytes {
return Err("Node and auxiliary Python runtime paths conflict".into());
}
} else {
files.insert(path, (bytes, executable));
}
}
for (path, bytes) in tools {
let executable = manifest.executables.contains(&path);
if files.insert(path, (bytes, executable)).is_some() {
return Err("npm tools overlap runtimes".into());
}
}
Ok(Self {
files,
assets,
node,
node_paths,
python: python_executable,
python_runtime: python.pack,
descriptor,
})
}
pub fn run(
&self,
inputs: &dyn Inputs,
graph: &npm::Environment,
files: &[(String, Vec<u8>)],
executables: &BTreeSet<String>,
) -> Result<Output, String> {
let stage = Stage::new()?;
let input = stage.0.join("input");
for (path, (bytes, executable)) in &self.files {
stage.write(&format!("input/{path}"), bytes, *executable)?;
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
bytes,
dever_sandbox::asset_executable(path),
)?;
}
for (path, bytes) in files {
stage.write(
&format!("input/install/{path}"),
bytes,
executables.contains(path),
)?;
}
let request = serde_json::json!({
"instances":graph.instances, "required":super::required_nodes(graph)?,
"roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(),
"python":format!("/worker/{}", self.python),
});
stage.write(
"input/request.json",
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
false,
)?;
stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [input.join("frontend.js").into_os_string()];
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated npm lifecycle",
&work,
)?;
inputs.diagnostic(&stage.diagnostics()?);
let mut outputs = super::super::python::output_files(&work.join("output"))?;
let response: ResultMetadata = serde_json::from_slice(
&outputs
.remove("result.json")
.ok_or("npm build result missing")?,
)
.map_err(|error| error.to_string())?;
let mut files = Vec::new();
let mut executables = BTreeSet::new();
for (path, bytes) in outputs {
let path = path
.strip_prefix("install/")
.ok_or("npm lifecycle output is outside its installation")?
.to_owned();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if fs::symlink_metadata(work.join("output/install").join(&path))
.map_err(|error| error.to_string())?
.permissions()
.mode()
& 0o111
!= 0
{
executables.insert(path.clone());
}
}
files.push((path, bytes));
}
let mut output = Output {
files,
executables,
failed: response.failed,
native: response.native,
rejections: BTreeMap::new(),
};
// Release the large compiler tree before staging the much smaller
// runtime-only native admission sandbox.
drop(stage);
self.admit_native(inputs, &mut output)?;
Ok(output)
}
fn admit_native(&self, inputs: &dyn Inputs, output: &mut Output) -> Result<(), String> {
let required = std::mem::take(&mut output.native);
let candidates = output
.files
.iter()
.filter(|(path, _)| path.ends_with(".node"))
.collect::<Vec<_>>();
if candidates.is_empty() {
return if required.is_empty() {
Ok(())
} else {
Err("npm generated native entry point is missing".into())
};
}
let machine = match self.descriptor.target.as_str() {
"linux-x86_64" => goblin::elf::header::EM_X86_64,
"linux-aarch64" => goblin::elf::header::EM_AARCH64,
_ => return Err("Node native admission target is unsupported".into()),
};
let mut files = self
.files
.iter()
.filter(|(path, _)| self.node_paths.contains(*path))
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
files.extend(
self.assets
.iter()
.map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())),
);
files.extend(
output
.files
.iter()
.map(|(path, bytes)| (path.clone(), bytes.as_slice())),
);
let stage = Stage::new()?;
for (path, bytes) in &files {
let relative = if path.starts_with("sandbox/") {
path.clone()
} else {
format!("input/{path}")
};
let executable = path == &self.node
|| path
.strip_prefix("sandbox/")
.is_some_and(dever_sandbox::asset_executable);
stage.write(&relative, bytes, executable)?;
}
stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?;
let input = stage.0.join("input");
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
for (path, bytes) in candidates {
let rejection = match goblin::elf::Elf::parse(bytes) {
Err(_) => Some(NativeRejection::Format),
Ok(elf) if !elf.is_64 || !elf.little_endian || elf.header.e_machine != machine => {
Some(NativeRejection::Target)
}
Ok(_) => {
if let Err(error) = crate::workers::python_wheel::validate_native(
&self.descriptor.target,
&[],
&self.node,
&files,
&BTreeSet::from([path.clone()]),
) {
inputs.diagnostic(
format!("npm addon '{path}' has no packaged loader closure: {error}\n")
.as_bytes(),
);
Some(NativeRejection::Closure)
} else {
let arguments = [
input.join("probe.cjs").into_os_string(),
input.join(path).into_os_string(),
];
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities::default(),
grants: &[],
};
let admitted = stage.probe(launch.command()?)?;
inputs.diagnostic(&stage.diagnostics()?);
if admitted {
None
} else {
Some(NativeRejection::Unloadable)
}
}
}
};
if let Some(reason) = rejection {
output.rejections.insert(path.clone(), reason);
} else {
output.native.insert(path.clone());
}
}
if !required.is_subset(&output.native) {
return Err("source-built npm addon did not pass runtime-only native admission".into());
}
Ok(())
}
}
@@ -0,0 +1,9 @@
// No package entry modules are executed. Probe exactly one immutable addon in
// an isolated runtime-only process, so addon globals cannot affect its peers.
try {
require(process.argv[2]);
process.exit(0);
} catch (error) {
process.stderr.write(String(error.stack || error) + '\n');
process.exit(10);
}
+166
View File
@@ -0,0 +1,166 @@
//! Signed, target-specific source-build tools. Never sourced from host PATH.
use crate::libs::{Ecosystem, RegistryRuntime};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
pub const MAX_PACK_BYTES: usize = 256 * 1024 * 1024;
pub const MAX_EXPANDED_BYTES: usize = 512 * 1024 * 1024;
pub const MAX_FILES: usize = 65_536;
pub type PackFiles = Vec<(String, Vec<u8>)>;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub format: String,
pub ecosystem: Ecosystem,
pub target: String,
pub runtime_sha256: String,
pub python_runtime_sha256: Option<String>,
pub executables: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Descriptor {
pub format: String,
pub ecosystem: Ecosystem,
pub target: String,
pub runtime_sha256: String,
pub python_runtime_sha256: Option<String>,
pub sha256: String,
}
fn hash(value: &str) -> Result<(), String> {
if value.len() != 64
|| !value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err("build pack identity must be canonical SHA-256".into());
}
Ok(())
}
impl Descriptor {
pub fn validate(
&self,
runtime: &RegistryRuntime,
python: Option<&RegistryRuntime>,
) -> Result<(), String> {
if self.format != "dever-registry-build-v1"
|| self.ecosystem == Ecosystem::Go
|| self.target != runtime.target
|| self.runtime_sha256 != runtime.pack.sha256
{
return Err("signed build pack does not match its runtime and target".into());
}
hash(&self.sha256)?;
hash(&self.runtime_sha256)?;
match (&self.ecosystem, &self.python_runtime_sha256, python) {
(Ecosystem::Pip, None, _) => Ok(()),
(Ecosystem::Npm, Some(identity), Some(python))
if identity == &python.pack.sha256 && python.target == self.target =>
{
hash(identity)
}
_ => Err("build pack has no exact auxiliary Python runtime binding".into()),
}
}
}
impl Manifest {
pub fn validate(
&self,
files: &[(String, Vec<u8>)],
descriptor: &Descriptor,
) -> Result<(), String> {
if self.format != "dever-lib-build-v1"
|| self.ecosystem != descriptor.ecosystem
|| self.target != descriptor.target
|| self.runtime_sha256 != descriptor.runtime_sha256
|| self.python_runtime_sha256 != descriptor.python_runtime_sha256
{
return Err("build pack manifest differs from its signed descriptor".into());
}
let total = files
.iter()
.try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len()))
.ok_or("build pack expanded size overflow")?;
if files.len() > MAX_FILES || total > MAX_EXPANDED_BYTES {
return Err("build pack exceeds file or byte budget".into());
}
let mut names = BTreeSet::new();
for (path, _) in files {
crate::workers::valid_path(path)?;
if path != "dever-build.json"
&& !path.starts_with("rootfs/usr/")
&& !(self.ecosystem == Ecosystem::Pip && path.starts_with("runtime/include/"))
&& !(self.ecosystem == Ecosystem::Npm
&& (path.starts_with("npm/include/node/")
|| path.starts_with("npm/frontend/node_modules/")))
{
return Err(format!(
"build pack path '{path}' is outside its fixed tool layout"
));
}
if !names.insert(path.as_str()) {
return Err("build pack repeats a path".into());
}
}
let mut executable_names = BTreeSet::new();
for path in &self.executables {
if !names.contains(path.as_str())
|| !executable_names.insert(path.as_str())
|| !(path.starts_with("rootfs/usr/")
|| self.ecosystem == Ecosystem::Npm
&& path.starts_with("npm/frontend/node_modules/"))
{
return Err(
"build executable is absent, repeated, or outside its tool root".into(),
);
}
}
for name in ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] {
if !executable_names.contains(format!("rootfs/usr/bin/{name}").as_str()) {
return Err(format!("build pack is missing required tool '{name}'"));
}
}
Ok(())
}
}
pub fn unpack(bytes: &[u8], descriptor: &Descriptor) -> Result<(Manifest, PackFiles), String> {
if bytes.len() > MAX_PACK_BYTES || crate::libs::sha256(bytes) != descriptor.sha256 {
return Err("build pack exceeds its budget or SHA-256 differs".into());
}
let files = super::super::registry::tar_files(bytes, "tgz", MAX_EXPANDED_BYTES as u64)?;
let metadata = files
.iter()
.find(|(path, _)| path == "dever-build.json")
.ok_or("build pack is missing dever-build.json")?;
let manifest: Manifest = serde_json::from_slice(&metadata.1)
.map_err(|error| format!("invalid build pack manifest: {error}"))?;
manifest.validate(&files, descriptor)?;
Ok((manifest, files))
}
pub fn validate_tools(
manifest: &Manifest,
files: &[(String, Vec<u8>)],
assets: &[(String, Vec<u8>)],
) -> Result<(), String> {
let mut supplied = files
.iter()
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in assets {
supplied.insert(format!("sandbox/{path}"), bytes);
}
crate::workers::python_wheel::native::validate_build_tools(
&manifest.target,
&manifest.executables,
&supplied,
)
}
+186
View File
@@ -0,0 +1,186 @@
//! One lifecycle for every managed compiler and package build hook.
use std::fs::{self, File};
use std::io::Read;
use std::path::Path;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
const DEADLINE: Duration = Duration::from_secs(300);
const MAX_OUTPUT: u64 = 16 * 1024 * 1024;
static NEXT_STAGE: AtomicU64 = AtomicU64::new(0);
mod budget;
pub(crate) struct Stage(pub(crate) PathBuf);
impl Stage {
pub(crate) fn new() -> Result<Self, String> {
for _ in 0..32 {
let path = std::env::temp_dir().join(format!(
"dever-lib-build-{}-{}",
std::process::id(),
NEXT_STAGE.fetch_add(1, Ordering::Relaxed)
));
#[cfg(unix)]
let created = {
use std::os::unix::fs::DirBuilderExt;
let mut builder = fs::DirBuilder::new();
builder.mode(0o700);
builder.create(&path)
};
#[cfg(not(unix))]
let created = fs::create_dir(&path);
match created {
Ok(()) => return Ok(Self(path)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(format!("cannot create private build staging: {error}")),
}
}
Err("cannot allocate private build staging".into())
}
pub(crate) fn write(
&self,
relative: &str,
bytes: &[u8],
executable: bool,
) -> Result<PathBuf, String> {
crate::workers::valid_path(relative)?;
let path = self.0.join(relative);
fs::create_dir_all(path.parent().expect("validated relative path"))
.map_err(|error| format!("cannot create build input directory: {error}"))?;
// This tree is private and never a writable child mount. A second write
// must not silently replace a previously verified input.
use std::io::Write;
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.map_err(|error| format!("cannot stage build input: {error}"))?;
file.write_all(bytes)
.map_err(|error| format!("cannot write build input: {error}"))?;
#[cfg(unix)]
if executable {
use std::os::unix::fs::PermissionsExt;
file.set_permissions(fs::Permissions::from_mode(0o700))
.map_err(|error| format!("cannot mark build tool executable: {error}"))?;
}
Ok(path)
}
pub(crate) fn run(
&self,
command: Command,
operation: &str,
writable: &Path,
) -> Result<Vec<u8>, String> {
let status = self.execute(command, operation, Some(writable))?;
if !status.success() {
return Err(format!(
"{operation} failed: {}",
String::from_utf8_lossy(&self.diagnostics()?)
));
}
fs::read(self.0.join("tool-stdout"))
.map_err(|error| format!("cannot read {operation} output: {error}"))
}
pub(crate) fn probe(&self, command: Command) -> Result<bool, String> {
match self
.execute(command, "isolated Node addon probe", None)?
.code()
{
Some(0) => Ok(true),
Some(10) => Ok(false),
_ => Err(format!(
"Node addon probe failed outside its loader check: {}",
String::from_utf8_lossy(&self.diagnostics()?)
)),
}
}
fn execute(
&self,
mut command: Command,
operation: &str,
writable: Option<&Path>,
) -> Result<std::process::ExitStatus, String> {
// Captures are outside every child-writable grant. Namespace teardown
// reaps descendants when the owned bootstrap is killed on any exit path.
let stdout = self.0.join("tool-stdout");
let stderr = self.0.join("tool-stderr");
let child = command
.stdin(Stdio::null())
.stdout(Stdio::from(
File::create(&stdout).map_err(|error| error.to_string())?,
))
.stderr(Stdio::from(
File::create(&stderr).map_err(|error| error.to_string())?,
))
.spawn()
.map_err(|error| format!("cannot execute {operation}: {error}"))?;
let mut child = OwnedChild(child);
let deadline = Instant::now() + DEADLINE;
let mut next_scratch_check = Instant::now();
let within_budget = || -> Result<(), String> {
for path in [&stdout, &stderr] {
if fs::metadata(path).map_err(|error| error.to_string())?.len() > MAX_OUTPUT {
return Err(format!("{operation} output exceeds byte limit"));
}
}
Ok(())
};
let status = loop {
within_budget()?;
if Instant::now() >= next_scratch_check {
if let Some(path) = writable {
budget::check(path)?;
}
next_scratch_check = Instant::now() + Duration::from_millis(250);
}
if let Some(status) = child
.0
.try_wait()
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
{
break status;
}
if Instant::now() >= deadline {
return Err(format!("{operation} exceeded its five-minute deadline"));
}
std::thread::sleep(Duration::from_millis(25));
};
within_budget()?;
if let Some(path) = writable {
budget::check(path)?;
}
Ok(status)
}
pub(crate) fn diagnostics(&self) -> Result<Vec<u8>, String> {
let mut bytes = Vec::new();
for name in ["tool-stdout", "tool-stderr"] {
File::open(self.0.join(name))
.and_then(|file| file.take(MAX_OUTPUT).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read build diagnostic: {error}"))?;
}
Ok(bytes)
}
}
impl Drop for Stage {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
struct OwnedChild(Child);
impl Drop for OwnedChild {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
@@ -0,0 +1,68 @@
//! Periodic scratch accounting is separate from immutable runtime/tool inputs.
use std::fs;
use std::path::Path;
const MAX_BYTES: u64 = 512 * 1024 * 1024;
const MAX_ENTRIES: usize = 65_536;
const MAX_DEPTH: usize = 128;
#[derive(Default)]
struct Budget {
bytes: u64,
entries: usize,
}
pub(super) fn check(root: &Path) -> Result<(), String> {
visit(root, 0, &mut Budget::default()).map_err(|error| format!("build scratch budget: {error}"))
}
#[cfg(unix)]
fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> {
use std::os::fd::AsRawFd;
use std::os::unix::fs::OpenOptionsExt;
if depth > MAX_DEPTH {
return Err("directory depth exceeds limit".into());
}
// An open directory pins the inode throughout enumeration. Child-created
// symlinks and rename races cannot redirect accounting outside this tree.
let directory = match fs::OpenOptions::new()
.read(true)
.custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::DIRECTORY).bits() as i32)
.open(path)
{
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(format!("cannot pin scratch directory: {error}")),
};
let pinned = format!("/proc/self/fd/{}", directory.as_raw_fd());
for entry in fs::read_dir(pinned).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let metadata = match entry.metadata() {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error.to_string()),
};
budget.entries += 1;
if budget.entries > MAX_ENTRIES {
return Err("entry count exceeds limit".into());
}
if metadata.is_dir() {
visit(&entry.path(), depth + 1, budget)?;
} else {
budget.bytes = budget
.bytes
.checked_add(metadata.len())
.ok_or("byte count overflow")?;
if budget.bytes > MAX_BYTES {
return Err("bytes exceed 512 MiB limit".into());
}
}
}
Ok(())
}
#[cfg(not(unix))]
fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> {
Err("managed build scratch accounting requires its Linux sandbox".into())
}
+119
View File
@@ -0,0 +1,119 @@
"""Fixed PEP 517 frontend, executed only inside the managed build sandbox."""
import importlib
import json
import os
from pathlib import Path
import re
import shutil
import sys
import tomllib
def build_system(source):
project = source / "pyproject.toml"
config = tomllib.loads(project.read_text("utf-8")) if project.exists() else {}
system = config.get("build-system")
if system is None:
system = {"requires": ["setuptools>=40.8.0"], "build-backend": "setuptools.build_meta:__legacy__"}
if not isinstance(system, dict) or not isinstance(system.get("requires"), list):
raise ValueError("build-system.requires must be an array")
requires = system["requires"]
if len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
raise ValueError("invalid static build requirements")
backend = system.get("build-backend", "setuptools.build_meta:__legacy__")
identifier = r"[A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*"
if not isinstance(backend, str) or not re.fullmatch(identifier + "(?::" + identifier + ")?", backend):
raise ValueError("invalid build-backend entry point")
paths = system.get("backend-path", [])
if not isinstance(paths, list) or len(paths) > 64:
raise ValueError("invalid backend-path")
for name in paths:
if not isinstance(name, str) or not name or Path(name).is_absolute() or "\\" in name:
raise ValueError("backend-path must be source-relative")
resolved = (source / name).resolve()
if not resolved.is_relative_to(source.resolve()) or not resolved.is_dir():
raise ValueError("backend-path escapes the source tree")
return {"requires": requires, "backend": backend, "backend_path": paths}
def backend_object(source, system):
paths = [(source / path).resolve() for path in system["backend_path"]]
sys.path[:0] = [str(path) for path in paths]
module_name, _, object_name = system["backend"].partition(":")
module = importlib.import_module(module_name)
if paths:
origin = Path(module.__file__).resolve()
if not any(origin.is_relative_to(path) for path in paths):
raise ValueError("in-tree backend was loaded outside backend-path")
backend = module
for part in object_name.split(".") if object_name else []:
backend = getattr(backend, part)
return backend
def leaf(value, suffix):
if not isinstance(value, str) or Path(value).name != value or not value.endswith(suffix):
raise ValueError("build hook returned an invalid output name")
return value
def execute(request, source, output):
system = build_system(source)
if request["phase"] == "inspect":
return system
if system != request["system"]:
raise ValueError("source build configuration changed between hooks")
backend = backend_object(source, system)
settings = request["config_settings"]
if request["phase"] == "requires":
hook = getattr(backend, "get_requires_for_build_wheel", None)
requires = hook(settings) if hook else []
if not isinstance(requires, list) or len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
raise ValueError("invalid dynamic build requirements")
return {"requires": requires}
if request["phase"] == "metadata":
metadata = output / "metadata"
metadata.mkdir()
hook = getattr(backend, "prepare_metadata_for_build_wheel", None)
prepared = metadata / leaf(hook(str(metadata), settings), ".dist-info") if hook else None
if prepared is not None and (not prepared.is_dir() or prepared.is_symlink()):
raise ValueError("metadata hook did not create its declared directory")
return {"metadata": str(prepared.relative_to(output)) if prepared else None}
prepared = None
if request.get("metadata") is not None:
shutil.copytree("/worker/prepared", output / "prepared")
prepared = output / "prepared" / request["metadata"]
if not prepared.resolve().is_relative_to((output / "prepared").resolve()):
raise ValueError("prepared metadata path escapes its snapshot")
wheels = output / "wheels"
wheels.mkdir()
name = leaf(backend.build_wheel(str(wheels), settings, str(prepared) if prepared else None), ".whl")
return {"filename": name}
def main():
request = json.loads(Path("/worker/request.json").read_text("utf-8"))
work = Path("/data/build")
source = work / "source"
shutil.copytree("/worker/source", source)
output = work / "output"
output.mkdir()
(work / "tmp").mkdir()
# These values are part of the fixed frontend, never project/host settings.
os.environ.update({
"PATH": "/worker/runtime/bin:/usr/bin",
"CC": "/usr/bin/cc", "CXX": "/usr/bin/c++",
"AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib",
"LDSHARED": "/usr/bin/cc -shared",
"CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.",
"CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"],
"TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0",
})
os.chdir(source)
result = execute(request, source, output)
(output / "result.json").write_text(json.dumps(result, sort_keys=True), "utf-8")
if __name__ == "__main__":
main()
+564
View File
@@ -0,0 +1,564 @@
//! PEP 517 source candidates enter the normal dependency solver as real wheels.
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsString;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
use crate::libs::{
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
sha256,
};
use crate::workers::{python_wheel::Wheel, runtime};
use serde_json::{Value, json};
struct Environment {
files: BTreeMap<String, (Vec<u8>, bool)>,
assets: Vec<(String, Vec<u8>)>,
executable: String,
version: String,
extension_suffixes: Vec<String>,
descriptor: pack::Descriptor,
}
struct Active<'a> {
session: &'a Session<'a>,
key: String,
}
impl Drop for Active<'_> {
fn drop(&mut self) {
self.session.active.borrow_mut().remove(&self.key);
}
}
impl Session<'_> {
pub(crate) fn python(
&self,
resolver: &RegistryResolver<'_>,
spec: &LibSpec,
runtime: &RegistryRuntime,
filename: &str,
source: &[u8],
) -> Result<Product, String> {
let key = format!(
"{}@{}:{}:{}",
spec.distribution_name(),
spec.version,
sha256(source),
runtime.pack.sha256
);
if let Some(product) = self.products.borrow().get(&key) {
return Ok(product.clone());
}
if self.active.borrow().len() >= 16 || !self.active.borrow_mut().insert(key.clone()) {
return Err(format!(
"cyclic or overly deep Python build requirement for {}",
spec.key()
));
}
let _active = Active {
session: self,
key: key.clone(),
};
let sources = source_files(filename, source)?;
let environment = Environment::new(self, runtime)?;
let empty = LockFile::new(vec![])?;
let inspected =
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
let system: PythonSystem = serde_json::from_value(inspected.response)
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
let mut requirements = system.requires.clone();
let mut dependencies = resolver.build_requirements(&requirements, runtime)?;
let mut stable = None;
let mut discovered = BTreeSet::new();
for _ in 0..8 {
let result = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"requires","system":system}),
None,
)?;
let dynamic: Vec<String> = serde_json::from_value(
result
.response
.get("requires")
.cloned()
.ok_or("build backend omitted requirements")?,
)
.map_err(|error| format!("invalid dynamic Python build requirements: {error}"))?;
discovered.extend(dynamic);
requirements = system.requires.iter().chain(&discovered).cloned().collect();
let next = resolver.build_requirements(&requirements, runtime)?;
if next == dependencies {
stable = Some(discovered.iter().cloned().collect());
break;
}
dependencies = next;
}
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
let metadata = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"metadata","system":system}),
None,
)?;
let result = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
Some(&metadata.outputs),
)?;
let filename = result
.response
.get("filename")
.and_then(Value::as_str)
.ok_or("build backend omitted wheel filename")?;
crate::workers::valid_path(filename)?;
if filename.contains('/') || !filename.ends_with(".whl") {
return Err("build backend returned invalid wheel filename".into());
}
let bytes = result
.outputs
.get(&format!("wheels/{filename}"))
.ok_or("build backend did not produce its declared wheel")?
.clone();
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
compare_metadata(prefix, &metadata.outputs, &bytes)?;
}
let output = artifact(
runtime,
format!(
"lib/pip/{}/{}/archive.whl",
sha256(spec.distribution_name().as_bytes()),
spec.version
),
&bytes,
);
let source_artifact = artifact(
runtime,
format!("build/source/{}/source", sha256(source)),
source,
);
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
return Err("build artifact cache identity mismatch".into());
}
}
let base: LibSpec = format!("pip:{}@{}", spec.distribution_name(), spec.version).parse()?;
let receipt = Receipt {
ecosystem: Ecosystem::Pip,
roots: vec![base],
sources: vec![source_artifact],
graph: None,
runtime: runtime.pack.clone(),
auxiliary_runtime: None,
native_entries: BTreeSet::new(),
native_rejections: BTreeMap::new(),
python_markers: runtime.python_markers.clone(),
tools: environment.descriptor,
frontend: super::frontend(&Ecosystem::Pip)?,
python: Some(system),
dynamic_requires: dynamic,
dependencies: Box::new(dependencies),
config_settings: BTreeMap::new(),
output,
};
let product = Product {
bytes,
filename: filename.into(),
receipt,
};
self.remember(key, product.clone())?;
Ok(product)
}
}
fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArtifact {
LockedArtifact {
target: runtime.target.clone(),
path,
bytes: bytes.len() as u64,
sha256: sha256(bytes),
}
}
struct Source {
files: Vec<(String, Vec<u8>)>,
executables: BTreeSet<String>,
}
fn source_files(filename: &str, bytes: &[u8]) -> Result<Source, String> {
let extension = if filename.ends_with(".tar.gz") {
"tgz"
} else if filename.ends_with(".zip") {
"zip"
} else {
return Err("unsupported Python source archive format".into());
};
let files = archive_files(extension, bytes)?;
let executables = crate::libs::registry::archive_executables(extension, bytes)?;
let root = files
.first()
.and_then(|(path, _)| path.split_once('/'))
.map(|(root, _)| format!("{root}/"))
.ok_or("Python source archive needs one root directory")?;
let files = files
.into_iter()
.map(|(path, bytes)| {
let path = path
.strip_prefix(&root)
.ok_or("Python source archive contains multiple roots")?
.to_owned();
crate::workers::valid_path(&path)?;
Ok((path, bytes))
})
.collect::<Result<Vec<_>, String>>()?;
let executables = executables
.into_iter()
.map(|path| {
path.strip_prefix(&root)
.map(str::to_owned)
.ok_or("source executable escapes archive root".into())
})
.collect::<Result<_, String>>()?;
Ok(Source { files, executables })
}
struct ResultFiles {
response: Value,
outputs: BTreeMap<String, Vec<u8>>,
}
impl Environment {
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
super::require_execution_target(&requested.target, "Python source build")?;
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
if runtime.pack != requested.pack || sha256(&bytes) != requested.pack.sha256 {
return Err("build Python runtime differs from resolver identity".into());
}
let runtime_files = archive_files("tgz", &bytes)?;
let manifest = runtime::validate(&runtime_files, &Ecosystem::Pip, &requested.target)?;
if manifest.python_wheel_tags != requested.python_wheel_tags
|| manifest.python_markers != requested.python_markers
{
return Err("build runtime policy differs from signed descriptor".into());
}
let executable = manifest
.executable
.clone()
.ok_or("build Python executable missing")?;
let executable = format!("runtime/{executable}");
let mut files = runtime_files
.into_iter()
.map(|(path, bytes)| {
let executable = manifest.executable(&path);
(format!("runtime/{path}"), (bytes, executable))
})
.collect::<BTreeMap<_, _>>();
let (descriptor, bytes) = session.inputs.tools(Ecosystem::Pip)?;
descriptor.validate(requested, None)?;
let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?;
let assets = session.inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
pack::validate_tools(&tools, &tool_files, &assets)?;
for (path, bytes) in tool_files {
let executable = tools.executables.contains(&path);
if files.insert(path, (bytes, executable)).is_some() {
return Err("build tools overlap runtime inputs".into());
}
}
let version = requested
.python_markers
.as_ref()
.ok_or("build Python markers absent")?
.python_version()
.to_string();
Ok(Self {
files,
assets,
executable,
version,
extension_suffixes: manifest.python_extension_suffixes,
descriptor,
})
}
fn run(
&self,
resolver: &RegistryResolver<'_>,
dependencies: &LockFile,
source: &Source,
mut request: Value,
prepared: Option<&BTreeMap<String, Vec<u8>>>,
) -> Result<ResultFiles, String> {
let stage = Stage::new()?;
let input = stage.0.join("input");
for (path, (bytes, executable)) in &self.files {
stage.write(&format!("input/{path}"), bytes, *executable)?;
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
bytes,
dever_sandbox::asset_executable(path),
)?;
}
for (path, bytes) in &source.files {
stage.write(
&format!("input/source/{path}"),
bytes,
source.executables.contains(path),
)?;
}
if let Some(prepared) = prepared {
for (path, bytes) in prepared {
stage.write(&format!("input/prepared/{path}"), bytes, false)?;
}
}
let mut installed = BTreeMap::<String, Vec<u8>>::new();
let runtime = resolver
.runtimes
.get(&Ecosystem::Pip)
.ok_or("build dependency runtime is missing")?;
let mut distributions = BTreeSet::new();
for lib in &dependencies.libs {
super::validate_python_policy(
dependencies,
lib,
runtime
.python_markers
.as_ref()
.ok_or("Python build markers absent")?,
)?;
if lib.runtime != runtime.pack || lib.artifacts.len() != 1 {
return Err(
"build dependency does not use the exact Python runtime and wheel".into(),
);
}
let artifact = &lib.artifacts[0];
let bytes =
resolver
.store
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
wheel.validate_dependencies(
lib,
runtime
.python_markers
.as_ref()
.ok_or("Python build markers absent")?,
)?;
if !distributions.insert((
lib.spec.distribution_name().to_owned(),
lib.spec.version.clone(),
)) {
continue;
}
for file in wheel.install(&self.executable, &self.version)? {
if self.files.contains_key(&file.path)
|| installed
.insert(file.path.clone(), file.bytes.clone())
.is_some()
{
return Err("build dependency installation has conflicting file owners".into());
}
stage.write(
&format!("input/{}", file.path),
&file.bytes,
file.executable,
)?;
}
}
let mut supplied = self
.files
.iter()
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in &self.assets {
supplied.insert(format!("sandbox/{path}"), bytes);
}
for (path, bytes) in &installed {
supplied.insert(path.clone(), bytes);
}
crate::workers::python_wheel::validate_native(
&runtime.target,
&self.extension_suffixes,
&self.executable,
&supplied,
&installed.keys().cloned().collect(),
)?;
request["python_version"] = json!(self.version);
request["config_settings"] = json!({});
stage.write(
"input/request.json",
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
false,
)?;
stage.write("input/frontend.py", include_bytes!("python.py"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [
OsString::from("-I"),
OsString::from("-B"),
input.join("frontend.py").into_os_string(),
];
let executable = input.join(&self.executable);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated Python build hook",
&work,
)?;
if let Some(session) = resolver.build {
session.inputs.diagnostic(&stage.diagnostics()?);
}
let outputs = output_files(&work.join("output"))?;
let response = serde_json::from_slice(
outputs
.get("result.json")
.ok_or("build hook produced no result")?,
)
.map_err(|error| format!("invalid build hook response: {error}"))?;
Ok(ResultFiles { response, outputs })
}
}
pub(super) fn output_files(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
let mut files = BTreeMap::new();
let mut pending = vec![root.to_owned()];
let mut remaining = crate::workers::MAX_TREE_BYTES;
let mut entries = 0usize;
#[cfg(unix)]
let mut links = BTreeMap::<(u64, u64), (u64, u64)>::new();
while let Some(directory) = pending.pop() {
if fs::symlink_metadata(&directory)
.map_err(|error| error.to_string())?
.file_type()
.is_symlink()
{
return Err("build output directory is a symbolic link".into());
}
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
entries += 1;
if entries > crate::workers::MAX_TREE_FILES {
return Err("build output exceeds file budget".into());
}
let entry = entry.map_err(|error| error.to_string())?;
let kind = entry.file_type().map_err(|error| error.to_string())?;
if kind.is_dir() {
pending.push(entry.path());
continue;
}
if !kind.is_file() {
return Err("build output contains a link or special file".into());
}
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
}
let file = options
.open(entry.path())
.map_err(|error| error.to_string())?;
let metadata = file.metadata().map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let aliases = links
.entry((metadata.dev(), metadata.ino()))
.or_insert((metadata.nlink(), 0));
if aliases.0 != metadata.nlink() {
return Err("build output link count changed during collection".into());
}
aliases.1 += 1;
}
if !metadata.is_file() || metadata.len() > remaining as u64 {
return Err("build output exceeds byte budget".into());
}
let mut bytes = Vec::new();
file.take(remaining as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
remaining = remaining
.checked_sub(bytes.len())
.ok_or("build output exceeds byte budget")?;
let path = entry.path();
let name = path
.strip_prefix(root)
.map_err(|error| error.to_string())?
.to_str()
.ok_or("build output path is not UTF-8")?
.to_owned();
crate::workers::valid_path(&name)?;
files.insert(name, bytes);
}
}
// node-gyp's COPY rule may hard-link two output names. Namespace teardown
// precedes this read: accept aliases only when every link is accounted for
// inside the output root, then publish independent immutable byte entries.
#[cfg(unix)]
if links
.values()
.any(|(expected, observed)| expected != observed)
{
return Err("build output has a hard-link alias outside its output root".into());
}
Ok(files)
}
fn compare_metadata(
prefix: &str,
outputs: &BTreeMap<String, Vec<u8>>,
wheel: &[u8],
) -> Result<(), String> {
crate::workers::valid_path(prefix)?;
let distribution = PathBuf::from(prefix)
.file_name()
.and_then(|name| name.to_str())
.ok_or("invalid prepared metadata directory")?
.to_owned();
let wheel = archive_files("whl", wheel)?
.into_iter()
.collect::<BTreeMap<_, _>>();
let prefix = format!("{prefix}/");
let mut count = 0;
for (path, bytes) in outputs {
if let Some(relative) = path.strip_prefix(&prefix) {
count += 1;
if wheel.get(&format!("{distribution}/{relative}")) != Some(bytes) {
return Err(
"built wheel metadata differs from prepare_metadata_for_build_wheel".into(),
);
}
}
}
if count == 0 {
return Err("metadata hook produced an empty directory".into());
}
Ok(())
}
+80
View File
@@ -0,0 +1,80 @@
//! Serialize dependency mutations from the first project read to publication.
//! The guard file is stable: replacing dever.lock must not replace its mutex.
use std::fs::{self, File, Metadata, OpenOptions};
use std::path::Path;
pub(crate) struct ProjectMutation {
_file: File,
}
impl ProjectMutation {
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
if !matches!(command, "add" | "update" | "remove") {
return Ok(None);
}
let directory = fs::symlink_metadata(root)
.map_err(|error| format!("cannot inspect project mutation directory: {error}"))?;
validate(&directory, true)?;
let path = root.join(".dever-project.lock");
if let Ok(metadata) = fs::symlink_metadata(&path) {
validate(&metadata, false)?;
}
let mut options = OpenOptions::new();
options.read(true).write(true).create(true).truncate(false);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options
.mode(0o600)
.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
}
let file = options
.open(&path)
.map_err(|error| format!("cannot open project mutation lock: {error}"))?;
let opened = file.metadata().map_err(|error| error.to_string())?;
validate(&opened, false)?;
let current = fs::symlink_metadata(&path).map_err(|error| error.to_string())?;
validate(&current, false)?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if opened.dev() != current.dev() || opened.ino() != current.ino() {
return Err("project mutation lock changed while opening".into());
}
}
fs2::FileExt::try_lock_exclusive(&file).map_err(|error| {
if error.kind() == std::io::ErrorKind::WouldBlock {
"project dependency mutation is busy; retry after the current command finishes"
.into()
} else {
format!("cannot acquire project mutation lock: {error}")
}
})?;
Ok(Some(Self { _file: file }))
}
}
fn validate(metadata: &Metadata, directory: bool) -> Result<(), String> {
if metadata.file_type().is_symlink()
|| (directory && !metadata.is_dir())
|| (!directory && !metadata.is_file())
{
return Err(
"project mutation path must be a real directory or regular lock file, without symlinks"
.into(),
);
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let forbidden = if directory { 0o022 } else { 0o077 };
if metadata.uid() != rustix::process::geteuid().as_raw()
|| metadata.mode() & forbidden != 0
|| (!directory && metadata.nlink() != 1)
{
return Err("project mutation directory must be owned and not group/other writable; lock must be owned, private and unlinked elsewhere".into());
}
}
Ok(())
}
File diff suppressed because it is too large. Load diff
+995
View File
@@ -0,0 +1,995 @@
//! npm package instances follow Node's ancestor lookup, not one global version
//! per name. Archives remain content-addressed and shared between instances.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use node_semver::{Range, Version};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::{Ecosystem, LibSpec, LockFile, LockedLib, RegistryRuntime, archive_files};
mod resolve;
pub(super) use resolve::resolve;
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Kind {
Dependency,
Optional,
Peer,
OptionalPeer,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Omission {
Target,
Unavailable,
Conflict,
OptionalPeerAbsent,
BuildFailure,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Edge {
pub name: String,
pub requirement: String,
pub kind: Kind,
pub target: Option<String>,
pub omission: Option<Omission>,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Source {
pub archive: LibSpec,
pub prefix: String,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Instance {
pub path: String,
pub spec: LibSpec,
pub source: Source,
/// Exact bundled layout of a root archive, including hoisted transitives.
pub bundled: BTreeMap<String, LibSpec>,
pub edges: Vec<Edge>,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Environment {
pub roots: Vec<LibSpec>,
pub instances: Vec<Instance>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub build: Option<String>,
}
impl Environment {
/// Explicit single-package graph for author fixtures with no dependencies.
/// Materialization still checks package.json against this graph.
pub fn single(spec: LibSpec) -> Self {
Self {
build: None,
roots: vec![spec.clone()],
instances: vec![Instance {
path: child("", &spec.name),
spec: spec.clone(),
source: Source {
archive: spec,
prefix: "package/".into(),
},
bundled: BTreeMap::new(),
edges: Vec::new(),
}],
}
}
}
pub fn roots(specs: &[LibSpec]) -> Vec<LibSpec> {
let mut roots = specs
.iter()
.filter(|spec| spec.ecosystem == Ecosystem::Npm)
.cloned()
.collect::<Vec<_>>();
roots.sort();
roots.dedup();
roots
}
pub fn environment<'a>(
lock: &'a LockFile,
requested: &[LibSpec],
) -> Result<Option<&'a Environment>, String> {
let requested = roots(requested);
if requested.is_empty() {
return Ok(None);
}
lock.npm
.iter()
.find(|env| env.roots == requested)
.map(Some)
.ok_or("npm Worker roots have no locked installation environment".into())
}
pub fn archive_roots(lock: &LockFile, requested: &[LibSpec]) -> Result<BTreeSet<LibSpec>, String> {
let requested = roots(requested);
if requested.is_empty() {
return Ok(BTreeSet::new());
}
if let Some(environment) = lock
.npm
.iter()
.find(|environment| environment.roots == requested)
{
return Ok(environment
.instances
.iter()
.map(|node| node.source.archive.clone())
.collect());
}
// Built-in metadata fixtures are not registry packages and have no tree.
let fixtures = super::super::FixtureRegistry::builtin();
if requested
.iter()
.all(|spec| fixtures.package(spec).is_some())
{
return Ok(requested.into_iter().collect());
}
Err("npm roots have no exact locked installation environment".into())
}
pub fn retain(lock: &LockFile, groups: &[Vec<LibSpec>]) -> Result<Vec<Environment>, String> {
let mut environments = Vec::new();
for group in groups {
let requested = roots(group);
if requested.is_empty() {
continue;
}
let original = lock
.npm
.iter()
.find(|env| env.roots == requested)
.or_else(|| {
lock.npm
.iter()
.find(|env| requested.iter().all(|root| env.roots.contains(root)))
});
let Some(original) = original else {
archive_roots(lock, group)?;
continue;
};
let mut queue = requested
.iter()
.map(|spec| child("", &spec.name))
.collect::<VecDeque<_>>();
let mut paths = BTreeSet::new();
while let Some(path) = queue.pop_front() {
if !paths.insert(path.clone()) {
continue;
}
let node = original
.instances
.iter()
.find(|node| node.path == path)
.ok_or("npm retained graph lacks a dependency instance")?;
queue.extend(node.edges.iter().filter_map(|edge| edge.target.clone()));
}
environments.push(Environment {
build: original.build.clone(),
roots: requested,
instances: original
.instances
.iter()
.filter(|node| paths.contains(&node.path))
.cloned()
.collect(),
});
}
normalize(&mut environments)?;
Ok(environments)
}
pub(super) fn child(parent: &str, name: &str) -> String {
if parent.is_empty() {
format!("node_modules/{name}")
} else {
format!("{parent}/node_modules/{name}")
}
}
pub(super) fn parent(path: &str) -> &str {
path.rsplit_once("/node_modules/")
.map_or("", |(parent, _)| parent)
}
fn validate_path(path: &str) -> Result<(), String> {
if path.len() > 4096 {
return Err("npm instance path exceeds its bound".into());
}
let path = path
.strip_prefix("node_modules/")
.ok_or("invalid npm instance path")?;
for name in path.split("/node_modules/") {
super::super::validate_name(&Ecosystem::Npm, name)?;
}
Ok(())
}
pub(super) fn visible<'a>(
instances: &'a BTreeMap<String, Instance>,
consumer: &str,
name: &str,
peer: bool,
) -> Option<&'a Instance> {
let mut scope = if peer { parent(consumer) } else { consumer };
loop {
if let Some(found) = instances.get(&child(scope, name)) {
return Some(found);
}
if scope.is_empty() {
return None;
}
scope = parent(scope);
}
}
#[derive(Clone, Debug)]
pub(super) struct Metadata {
pub spec: LibSpec,
pub edges: Vec<Edge>,
pub bundles: BTreeSet<String>,
pub document: Value,
}
fn requirements(document: &Value, field: &str) -> Result<BTreeMap<String, String>, String> {
let Some(value) = document.get(field) else {
return Ok(BTreeMap::new());
};
value
.as_object()
.ok_or_else(|| format!("npm {field} must be an object"))?
.iter()
.map(|(name, value)| {
super::super::validate_name(&Ecosystem::Npm, name)?;
let rule = value.as_str().ok_or("npm dependency range must be text")?;
rule.parse::<Range>()
.map_err(|error| format!("invalid npm range for {name}: {error}"))?;
Ok((name.clone(), rule.into()))
})
.collect()
}
impl Metadata {
pub(super) fn parse(document: Value) -> Result<Self, String> {
let spec: LibSpec = format!(
"npm:{}@{}",
super::field(&document, "name")?,
super::field(&document, "version")?
)
.parse()?;
let mut regular = requirements(&document, "dependencies")?;
let optional = requirements(&document, "optionalDependencies")?;
regular.retain(|name, _| !optional.contains_key(name));
let mut peers = requirements(&document, "peerDependencies")?;
let peer_meta = document
.get("peerDependenciesMeta")
.map(|value| {
value
.as_object()
.ok_or("npm peerDependenciesMeta must be an object")
})
.transpose()?;
if let Some(meta) = peer_meta {
for (name, value) in meta {
if !peers.contains_key(name)
|| !value.is_object()
|| value.get("optional").is_some_and(|flag| !flag.is_boolean())
{
return Err("invalid npm optional peer metadata".into());
}
}
}
// npm has one outgoing edge per name: optional > production > peer.
peers.retain(|name, _| !regular.contains_key(name) && !optional.contains_key(name));
let mut edges = Vec::new();
for (entries, kind) in [
(regular.clone(), Kind::Dependency),
(optional.clone(), Kind::Optional),
(peers, Kind::Peer),
] {
for (name, requirement) in entries {
let kind = if kind == Kind::Peer
&& peer_meta
.and_then(|meta| meta.get(&name))
.and_then(|value| value.get("optional"))
.and_then(Value::as_bool)
== Some(true)
{
Kind::OptionalPeer
} else {
kind.clone()
};
edges.push(Edge {
name,
requirement,
kind,
target: None,
omission: None,
});
}
}
edges.sort();
let bundles = |value: &Value| -> Result<BTreeSet<String>, String> {
match value {
Value::Bool(true) => Ok(regular.keys().chain(optional.keys()).cloned().collect()),
Value::Bool(false) => Ok(BTreeSet::new()),
Value::Array(values) => values
.iter()
.map(|value| {
let name = value
.as_str()
.ok_or("npm bundled dependency name must be text")?;
super::super::validate_name(&Ecosystem::Npm, name)?;
if !regular.contains_key(name) && !optional.contains_key(name) {
return Err(
"npm bundled dependency is not a declared dependency".into()
);
}
Ok(name.to_owned())
})
.collect(),
_ => Err("npm bundleDependencies must be an array or boolean".into()),
}
};
let first = document
.get("bundleDependencies")
.map(bundles)
.transpose()?;
let second = document
.get("bundledDependencies")
.map(bundles)
.transpose()?;
if first.is_some() && second.is_some() && first != second {
return Err("npm bundled dependency aliases disagree".into());
}
Ok(Self {
spec,
edges,
bundles: first.or(second).unwrap_or_default(),
document,
})
}
pub(super) fn eligible(&self, runtime: &RegistryRuntime) -> Result<bool, String> {
let (os, cpu) = runtime
.target
.split_once('-')
.ok_or("npm runtime target lacks platform identity")?;
let os = match os {
"windows" => "win32",
"macos" => "darwin",
other => other,
};
let cpu = match cpu {
"x86_64" => "x64",
"aarch64" => "arm64",
other => other,
};
for (field, selected) in [
("os", Some(os)),
("cpu", Some(cpu)),
("libc", runtime.npm_libc.as_deref()),
] {
if field == "libc" && os != "linux" {
continue;
}
let Some(value) = self.document.get(field) else {
continue;
};
let values = value
.as_array()
.ok_or_else(|| format!("npm {field} must be an array"))?;
let values = values
.iter()
.map(|value| value.as_str().ok_or("npm platform selector must be text"))
.collect::<Result<Vec<_>, _>>()?;
let Some(selected) = selected else {
return Ok(false);
};
if values
.iter()
.any(|value| value.strip_prefix('!') == Some(selected))
|| (values.iter().any(|value| !value.starts_with('!'))
&& !values.contains(&selected)
&& !values.contains(&"any"))
{
return Ok(false);
}
}
Ok(true)
}
}
pub(super) struct Archive {
pub files: Vec<(String, Vec<u8>)>,
pub packages: BTreeMap<String, Metadata>,
}
impl Archive {
pub(super) fn bundled(&self) -> BTreeMap<String, LibSpec> {
self.packages
.iter()
.filter(|(prefix, _)| prefix.as_str() != "package/")
.map(|(prefix, metadata)| {
(
prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/')
.into(),
metadata.spec.clone(),
)
})
.collect()
}
pub(super) fn parse(spec: &LibSpec, bytes: &[u8]) -> Result<Self, String> {
let files = archive_files("tgz", bytes)?;
let mut packages = BTreeMap::<String, Metadata>::new();
for (path, bytes) in &files {
let relative = path
.strip_prefix("package/")
.ok_or("npm archive must have a package/ root")?;
if relative == "package.json" || relative.ends_with("/package.json") {
let prefix = path.strip_suffix("package.json").unwrap();
if prefix != "package/" {
let nested = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
if validate_path(nested).is_err() {
continue;
}
}
let document = super::parse_json(bytes)?;
let metadata = Metadata::parse(document)?;
if prefix == "package/" {
if metadata.spec != *spec {
return Err(
"npm package.json identity differs from its registry archive".into(),
);
}
} else if !prefix.ends_with(&format!("node_modules/{}/", metadata.spec.name)) {
return Err("npm bundled package identity differs from its path".into());
}
packages.insert(prefix.into(), metadata);
}
}
let root = packages
.get("package/")
.ok_or("npm archive lacks package.json")?;
for name in &root.bundles {
if !packages.contains_key(&format!("package/node_modules/{name}/")) {
return Err("npm declared bundled dependency is absent from its archive".into());
}
}
for (path, _) in &files {
if !packages.contains_key(&source_owner(path)?) {
return Err("npm archive has unowned node_modules content".into());
}
}
let nodes = packages
.iter()
.filter(|(prefix, _)| prefix.as_str() != "package/")
.map(|(prefix, metadata)| {
let path = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/')
.to_owned();
(
path.clone(),
Instance {
path,
spec: metadata.spec.clone(),
source: Source {
archive: spec.clone(),
prefix: prefix.clone(),
},
bundled: BTreeMap::new(),
edges: Vec::new(),
},
)
})
.collect::<BTreeMap<_, _>>();
let mut pending = root
.bundles
.iter()
.map(|name| child("", name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = nodes
.get(&path)
.ok_or("npm bundle references a missing package")?;
for edge in &packages[&node.source.prefix].edges {
let peer = matches!(edge.kind, Kind::Peer | Kind::OptionalPeer);
if let Some(found) = visible(&nodes, &path, &edge.name, peer) {
if !Version::from_str(&found.spec.version)
.map_err(|error| error.to_string())?
.satisfies(
&edge
.requirement
.parse::<Range>()
.map_err(|error| error.to_string())?,
)
{
return Err("npm bundled dependency has an incompatible version".into());
}
pending.push_back(found.path.clone());
} else if edge.kind == Kind::Dependency
&& edge.name != spec.name
&& !root.edges.iter().any(|root| root.name == edge.name)
{
return Err("npm bundled dependency closure is incomplete".into());
}
}
}
if reached.len() != nodes.len() {
return Err("npm archive contains undeclared or unreachable bundled packages".into());
}
Ok(Self { files, packages })
}
}
fn source_owner(path: &str) -> Result<String, String> {
let Some((parent, tail)) = path.rsplit_once("/node_modules/") else {
return Ok("package/".into());
};
let mut parts = tail.split('/');
let first = parts.next().ok_or("npm archive has invalid package path")?;
let name = if first.starts_with('@') {
format!(
"{first}/{}",
parts
.next()
.ok_or("npm scoped package path is incomplete")?
)
} else {
first.into()
};
let prefix = format!("{parent}/node_modules/{name}");
validate_path(
prefix
.strip_prefix("package/")
.ok_or("npm bundled archive has invalid root")?,
)?;
Ok(format!("{prefix}/"))
}
fn validate_source_owner(environment: &Environment, node: &Instance) -> Result<(), String> {
for owner in &environment.instances {
for (relative, spec) in &owner.bundled {
if node.path == format!("{}/{relative}", owner.path)
&& (node.spec != *spec
|| node.source.archive != owner.source.archive
|| node.source.prefix != format!("package/{relative}/"))
{
return Err(
"npm bundled instance is not owned by its parent archive installation".into(),
);
}
}
}
if node.source.prefix == "package/" {
for (relative, spec) in &node.bundled {
validate_path(relative)?;
if spec.ecosystem != Ecosystem::Npm
|| !relative.ends_with(&format!("node_modules/{}", spec.name))
{
return Err("invalid npm bundled layout identity".into());
}
super::super::validate_name(&Ecosystem::Npm, &spec.name)?;
super::super::validate_version(&Ecosystem::Npm, &spec.version)?;
}
return if node.source.archive == node.spec {
Ok(())
} else {
Err("npm root archive identity differs from its instance".into())
};
}
let relative = node
.source
.prefix
.strip_prefix("package/")
.and_then(|prefix| prefix.strip_suffix('/'))
.ok_or("invalid npm bundled source prefix")?;
validate_path(relative)?;
if !node.bundled.is_empty()
|| !environment.instances.iter().any(|owner| {
owner.source.prefix == "package/"
&& owner.source.archive == node.source.archive
&& owner.spec == node.source.archive
&& node.path == format!("{}/{relative}", owner.path)
&& owner.bundled.get(relative) == Some(&node.spec)
})
{
return Err("npm bundled instance is not owned by its parent archive installation".into());
}
Ok(())
}
/// Validate immutable manifests and plan borrowed files before cloning any
/// installation output. Reused archives must not multiply past the tree budget.
pub fn files<'a>(
environment: &Environment,
lock: &LockFile,
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
) -> Result<Vec<(String, Vec<u8>)>, String> {
source_files(environment, lock, archive, false)
}
pub fn build_files<'a>(
environment: &Environment,
lock: &LockFile,
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
) -> Result<Vec<(String, Vec<u8>)>, String> {
source_files(environment, lock, archive, true)
}
pub(crate) fn build_executables(
environment: &Environment,
archives: &BTreeMap<LibSpec, Vec<u8>>,
) -> Result<BTreeSet<String>, String> {
let modes = archives
.iter()
.map(|(spec, bytes)| Ok((spec, super::archive_executables("tgz", bytes)?)))
.collect::<Result<BTreeMap<_, _>, String>>()?;
let mut installed = BTreeSet::new();
for node in &environment.instances {
for path in modes
.get(&node.source.archive)
.ok_or("npm source modes missing")?
{
if source_owner(path)? == node.source.prefix {
let relative = path
.strip_prefix(&node.source.prefix)
.ok_or("npm source mode outside owner")?;
installed.insert(format!("{}/{relative}", node.path));
}
}
}
Ok(installed)
}
pub(crate) fn installed_owner(path: &str) -> Result<String, String> {
source_owner(&format!("package/{path}"))?
.strip_prefix("package/")
.map(|owner| owner.trim_end_matches('/').into())
.ok_or("npm output is outside its installation".into())
}
fn source_files<'a>(
environment: &Environment,
lock: &LockFile,
mut archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
preparing: bool,
) -> Result<Vec<(String, Vec<u8>)>, String> {
let mut archives = BTreeMap::new();
let mut archive_bytes = 0usize;
for node in &environment.instances {
validate_source_owner(environment, node)?;
if archives.contains_key(&node.source.archive) {
continue;
}
let lib = lock
.libs
.iter()
.find(|lib| lib.spec == node.source.archive)
.ok_or("npm graph source archive is missing")?;
let parsed = Archive::parse(&lib.spec, archive(lib)?)?;
for (_, bytes) in &parsed.files {
archive_bytes = archive_bytes
.checked_add(bytes.len())
.ok_or("npm archive size overflow")?;
if archive_bytes > crate::workers::MAX_TREE_BYTES {
return Err("npm archive cache exceeds its tree byte budget".into());
}
}
archives.insert(lib.spec.clone(), parsed);
}
let mut files = BTreeMap::new();
let mut installed_bytes = 0usize;
for node in &environment.instances {
let source = &archives[&node.source.archive];
if node.source.prefix == "package/" && node.bundled != source.bundled() {
return Err("npm bundled layout differs from its parent archive".into());
}
let metadata = source
.packages
.get(&node.source.prefix)
.ok_or("npm instance source package is absent")?;
if metadata.spec != node.spec
|| metadata.edges.len() != node.edges.len()
|| metadata.edges.iter().any(|expected| {
!node.edges.iter().any(|edge| {
edge.name == expected.name
&& edge.requirement == expected.requirement
&& edge.kind == expected.kind
})
})
{
return Err("npm locked dependency graph differs from package.json".into());
}
if !preparing
&& (metadata.document.get("gypfile").and_then(Value::as_bool) == Some(true)
|| metadata
.document
.get("scripts")
.and_then(Value::as_object)
.is_some_and(|scripts| {
["preinstall", "install", "postinstall"]
.iter()
.any(|name| scripts.contains_key(*name))
}))
{
return Err(
"managed npm Worker rejects lifecycle scripts and native build hooks".into(),
);
}
for (path, bytes) in &source.files {
if source_owner(path)? != node.source.prefix {
continue;
}
let relative = path
.strip_prefix(&node.source.prefix)
.ok_or("npm file is outside its instance source")?;
if !preparing && (relative.ends_with(".node") || relative == "binding.gyp") {
return Err("managed npm Worker rejects native package content".into());
}
let installed = format!("{}/{relative}", node.path);
installed_bytes = installed_bytes
.checked_add(bytes.len())
.ok_or("npm installed size overflow")?;
if installed_bytes > crate::workers::MAX_TREE_BYTES
|| files.len() >= crate::workers::MAX_TREE_FILES
{
return Err("npm installation exceeds its tree file/byte budget".into());
}
if files.insert(installed, bytes).is_some() {
return Err("npm installed files have conflicting owners".into());
}
}
}
Ok(files
.into_iter()
.map(|(path, bytes)| (path, bytes.clone()))
.collect())
}
pub fn normalize(environments: &mut Vec<Environment>) -> Result<(), String> {
for env in environments.iter_mut() {
env.roots.sort();
env.roots.dedup();
for node in &mut env.instances {
node.edges.sort();
}
env.instances
.sort_by(|left, right| left.path.cmp(&right.path));
}
environments.sort_by(|left, right| left.roots.cmp(&right.roots));
for pair in environments.windows(2) {
if pair[0].roots == pair[1].roots && pair[0] != pair[1] {
return Err("npm roots have inconsistent installation environments".into());
}
}
environments.dedup();
Ok(())
}
pub fn requires_build(files: &[(String, Vec<u8>)]) -> Result<bool, String> {
for (path, bytes) in files {
if path.ends_with(".node") || path.ends_with("/binding.gyp") {
return Ok(true);
}
if path.ends_with("/package.json") {
let metadata: Value = serde_json::from_slice(bytes)
.map_err(|error| format!("invalid npm installation metadata: {error}"))?;
if metadata.get("gypfile").and_then(Value::as_bool) == Some(true)
|| metadata
.get("scripts")
.and_then(Value::as_object)
.is_some_and(|scripts| {
["preinstall", "install", "postinstall"]
.iter()
.any(|name| scripts.contains_key(*name))
})
{
return Ok(true);
}
}
}
Ok(false)
}
/// Lifecycle output may add generated files, but cannot rewrite dependency
/// identities, create undeclared package instances, or escape the locked tree.
pub fn validate_built_files(
environment: &Environment,
files: &[(String, Vec<u8>)],
) -> Result<(), String> {
let nodes = environment
.instances
.iter()
.map(|node| (node.path.as_str(), node))
.collect::<BTreeMap<_, _>>();
let mut manifests = BTreeSet::new();
let mut names = BTreeSet::new();
let mut total = 0usize;
for (path, bytes) in files {
crate::workers::valid_path(path)?;
let owner = source_owner(&format!("package/{path}"))?;
let owner = owner
.strip_prefix("package/")
.ok_or("npm output is outside its installation")?
.trim_end_matches('/');
let node = nodes
.get(owner)
.ok_or("npm build output introduces an unlocked instance")?;
if !names.insert(path) {
return Err("npm build output repeats a file".into());
}
total = total
.checked_add(bytes.len())
.ok_or("npm output byte overflow")?;
if total > crate::workers::MAX_TREE_BYTES || names.len() > crate::workers::MAX_TREE_FILES {
return Err("npm output exceeds installation budget".into());
}
if path == &format!("{owner}/package.json") {
let metadata =
Metadata::parse(serde_json::from_slice(bytes).map_err(|error| error.to_string())?)?;
if metadata.spec != node.spec
|| metadata.edges.len() != node.edges.len()
|| metadata.edges.iter().any(|expected| {
!node.edges.iter().any(|edge| {
edge.name == expected.name
&& edge.requirement == expected.requirement
&& edge.kind == expected.kind
})
})
{
return Err("npm lifecycle changed locked package identity or dependencies".into());
}
manifests.insert(owner.to_owned());
}
}
if manifests.len() != nodes.len() {
return Err("npm lifecycle removed a locked package manifest".into());
}
Ok(())
}
pub fn doctor(lock: &LockFile) -> Result<(), String> {
for env in &lock.npm {
if env.roots.is_empty()
|| env.roots != roots(&env.roots)
|| env.instances.len() > super::MAX_PACKAGES
{
return Err("invalid npm installation environment roots or size".into());
}
let nodes = env
.instances
.iter()
.map(|node| (node.path.clone(), node.clone()))
.collect::<BTreeMap<_, _>>();
if nodes.len() != env.instances.len() {
return Err("duplicate npm instance path".into());
}
let mut pending = VecDeque::new();
for root in &env.roots {
let path = child("", &root.name);
if nodes.get(&path).map(|node| &node.spec) != Some(root) {
return Err("npm root is not installed at its exact path".into());
}
pending.push_back(path);
}
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = nodes
.get(&path)
.ok_or("npm edge references a missing instance")?;
validate_source_owner(env, node)?;
validate_path(&node.path)?;
super::super::validate_name(&Ecosystem::Npm, &node.spec.name)?;
super::super::validate_version(&Ecosystem::Npm, &node.spec.version)?;
if node.spec.ecosystem != Ecosystem::Npm
|| !node
.path
.ends_with(&format!("node_modules/{}", node.spec.name))
{
return Err("npm instance identity differs from its path".into());
}
if !lock
.libs
.iter()
.any(|lib| lib.spec == node.source.archive && lib.spec.ecosystem == Ecosystem::Npm)
{
return Err("npm instance source archive is not locked".into());
}
if node.source.prefix != "package/" {
validate_path(
node.source
.prefix
.strip_prefix("package/")
.and_then(|path| path.strip_suffix('/'))
.ok_or("invalid npm bundled source prefix")?,
)?;
} else if node.source.archive != node.spec {
return Err("npm root archive identity differs from its instance".into());
}
let mut unique = BTreeSet::new();
for edge in &node.edges {
if !unique.insert((&edge.name, &edge.kind)) {
return Err("duplicate npm dependency edge".into());
}
let range = edge
.requirement
.parse::<Range>()
.map_err(|error| error.to_string())?;
let actual = visible(
&nodes,
&path,
&edge.name,
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
);
match (&edge.target, &edge.omission) {
(Some(target), None)
if actual.is_some_and(|found| {
found.path == *target
&& Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&range))
}) =>
{
pending.push_back(target.clone());
}
(None, Some(reason))
if actual.is_none()
&& (edge.kind == Kind::Optional
&& *reason != Omission::OptionalPeerAbsent
|| edge.kind == Kind::OptionalPeer
&& matches!(
reason,
Omission::OptionalPeerAbsent | Omission::BuildFailure
))
&& (*reason != Omission::BuildFailure || env.build.is_some()) => {}
_ => {
return Err(
"npm edge differs from Node lookup or its required range".into()
);
}
}
}
}
if reached.len() != nodes.len() {
return Err("npm environment contains unreachable package instances".into());
}
}
for worker in &lock.workers {
if worker.ecosystem == "npm" {
environment(lock, &worker.libs)?;
}
}
Ok(())
}
use std::str::FromStr;
@@ -0,0 +1,436 @@
use super::*;
use crate::libs::registry::{MAX_PACKAGES, MAX_RESOLUTION_ATTEMPTS, RegistryResolver, SearchError};
#[derive(Clone)]
struct Request {
consumer: Option<String>,
edge: Edge,
}
#[derive(Clone, Default)]
struct State {
nodes: BTreeMap<String, Instance>,
pending: VecDeque<Request>,
}
struct Catalog<'a, 'r> {
resolver: &'a RegistryResolver<'r>,
runtime: &'a RegistryRuntime,
indexes: BTreeMap<String, Vec<String>>,
metadata: BTreeMap<(String, String), Value>,
archives: BTreeMap<LibSpec, (LockedLib, Archive)>,
attempts: usize,
archive_bytes: usize,
}
pub(crate) fn resolve(
resolver: &RegistryResolver<'_>,
requested: &[LibSpec],
runtime: &RegistryRuntime,
) -> Result<(Vec<LockedLib>, Environment), String> {
let roots = roots(requested);
let mut state = State::default();
for spec in &roots {
state.pending.push_back(Request {
consumer: None,
edge: Edge {
name: spec.name.clone(),
requirement: format!("={}", spec.version),
kind: Kind::Dependency,
target: None,
omission: None,
},
});
}
let mut catalog = Catalog {
resolver,
runtime,
indexes: BTreeMap::new(),
metadata: BTreeMap::new(),
archives: BTreeMap::new(),
attempts: 0,
archive_bytes: 0,
};
let mut state = catalog.search(state).map_err(SearchError::message)?;
let mut pending = roots
.iter()
.map(|root| child("", &root.name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
pending.extend(
state.nodes[&path]
.edges
.iter()
.filter_map(|edge| edge.target.clone()),
);
}
state.nodes.retain(|path, _| reached.contains(path));
let used = state
.nodes
.values()
.map(|node| node.source.archive.clone())
.collect::<BTreeSet<_>>();
let libraries = used
.into_iter()
.map(|spec| {
catalog
.archives
.remove(&spec)
.expect("resolved npm archive")
.0
})
.collect();
Ok((
libraries,
Environment {
build: None,
roots,
instances: state.nodes.into_values().collect(),
},
))
}
impl Catalog<'_, '_> {
fn archive(&mut self, spec: &LibSpec) -> Result<(), SearchError> {
if self.archives.contains_key(spec) {
return Ok(());
}
let document = self
.resolver
.registry_metadata(
Ecosystem::Npm,
&spec.name,
&spec.version,
&mut self.metadata,
)
.map_err(SearchError::Fatal)?;
let bytes = self
.resolver
.npm_archive(spec, &document)
.map_err(SearchError::Fatal)?;
let archive = Archive::parse(spec, &bytes).map_err(SearchError::Fatal)?;
self.archive_bytes = self
.archive_bytes
.checked_add(
archive
.files
.iter()
.map(|(_, bytes)| bytes.len())
.sum::<usize>(),
)
.ok_or_else(|| SearchError::Fatal("npm archive cache size overflow".into()))?;
if self.archive_bytes > 256 * 1024 * 1024 || self.archives.len() >= MAX_PACKAGES {
return Err(SearchError::Fatal(
"npm archive cache exceeds its 256 MiB/package budget".into(),
));
}
let lib = self
.resolver
.lock_archive(
spec,
Vec::new(),
self.runtime,
"tgz",
&bytes,
"npm-instances-v1",
)
.map_err(SearchError::Fatal)?;
self.archives.insert(spec.clone(), (lib, archive));
Ok(())
}
fn metadata(&self, node: &Instance) -> &Metadata {
&self.archives[&node.source.archive].1.packages[&node.source.prefix]
}
fn bind(
state: &mut State,
request: &Request,
target: Option<String>,
omission: Option<Omission>,
) {
if let Some(consumer) = &request.consumer {
let edge = state
.nodes
.get_mut(consumer)
.expect("pending npm consumer")
.edges
.iter_mut()
.find(|edge| edge.name == request.edge.name && edge.kind == request.edge.kind)
.expect("pending npm edge");
edge.target = target;
edge.omission = omission;
}
}
fn add(&self, state: &mut State, path: &str, spec: &LibSpec) -> Result<(), SearchError> {
for (prefix, metadata) in &self.archives[spec].1.packages {
let relative = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
let location = if relative.is_empty() {
path.into()
} else {
format!("{path}/{relative}")
};
let instance = Instance {
path: location.clone(),
spec: metadata.spec.clone(),
source: Source {
archive: spec.clone(),
prefix: prefix.clone(),
},
bundled: if relative.is_empty() {
self.archives[spec].1.bundled()
} else {
BTreeMap::new()
},
edges: metadata.edges.clone(),
};
if state.nodes.insert(location.clone(), instance).is_some() {
return Err(SearchError::Conflict(
"npm package installation paths collide".into(),
));
}
for edge in &metadata.edges {
if edge.kind != Kind::OptionalPeer {
state.pending.push_back(Request {
consumer: Some(location.clone()),
edge: edge.clone(),
});
}
}
}
Ok(())
}
fn prune_bundled_orphans(&self, state: &mut State, root: &str) {
let mut pending = VecDeque::from([root.to_owned()]);
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let Some(node) = state.nodes.get(&path) else {
continue;
};
for edge in &self.metadata(node).edges {
if let Some(found) = visible(
&state.nodes,
&path,
&edge.name,
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
) {
pending.push_back(found.path.clone());
}
}
}
let prefix = format!("{root}/");
state.nodes.retain(|path, node| {
node.source.prefix == "package/" || !path.starts_with(&prefix) || reached.contains(path)
});
}
fn search(&mut self, mut state: State) -> Result<State, SearchError> {
self.attempts += 1;
if self.attempts > MAX_RESOLUTION_ATTEMPTS
|| state.nodes.len() > MAX_PACKAGES
|| state.nodes.keys().any(|path| path.len() > 4096)
{
return Err(SearchError::Fatal(
"npm instance graph exceeds its resolution budget".into(),
));
}
while let Some(request) = state.pending.pop_front() {
if request
.consumer
.as_ref()
.is_some_and(|consumer| !state.nodes.contains_key(consumer))
{
continue;
}
let consumer = request.consumer.as_deref().unwrap_or("");
let peer = request.edge.kind == Kind::Peer;
let scope = if peer { parent(consumer) } else { consumer };
let location = child(scope, &request.edge.name);
let rule = request
.edge
.requirement
.parse::<Range>()
.map_err(|error| SearchError::Fatal(error.to_string()))?;
let existing = visible(&state.nodes, consumer, &request.edge.name, peer).cloned();
if let Some(found) = &existing {
let eligible = self
.metadata(found)
.eligible(self.runtime)
.map_err(SearchError::Fatal)?;
if !eligible
&& request.edge.kind == Kind::Optional
&& found.source.prefix != "package/"
{
let prefix = format!("{}/", found.path);
state
.nodes
.retain(|path, _| path != &found.path && !path.starts_with(&prefix));
let relative = found
.source
.prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
let owner = found
.path
.strip_suffix(relative)
.unwrap()
.trim_end_matches('/');
self.prune_bundled_orphans(&mut state, owner);
Self::bind(&mut state, &request, None, Some(Omission::Target));
continue;
}
// Ordinary dependencies get a private nested slot. Reuse an
// ancestor only to close a real package cycle, never a sibling's
// independently resolved peer context.
let bundled = state.nodes.get(consumer).is_some_and(|node| {
node.source.prefix != "package/" && node.source.archive == found.source.archive
});
let reusable = peer
|| bundled
|| found.path == location
|| consumer == found.path
|| consumer.starts_with(&format!("{}/", found.path));
if reusable
&& eligible
&& Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&rule))
{
Self::bind(&mut state, &request, Some(found.path.clone()), None);
continue;
}
if found.path == location {
return Err(SearchError::Conflict(format!(
"npm dependency/peer conflict for {} at {location}",
request.edge.name
)));
}
}
if let Some(node) = request
.consumer
.as_ref()
.and_then(|consumer| state.nodes.get(consumer))
&& node.source.prefix != "package/"
&& !peer
{
if request.edge.kind == Kind::Optional && existing.is_none() {
Self::bind(&mut state, &request, None, Some(Omission::Unavailable));
continue;
}
return Err(SearchError::Fatal(
"npm bundled dependency closure is incomplete".into(),
));
}
let versions = if let Some(versions) = self.indexes.get(&request.edge.name) {
versions.clone()
} else {
let versions = self
.resolver
.registry_versions(Ecosystem::Npm, &request.edge.name)
.map_err(SearchError::Fatal)?;
self.indexes
.insert(request.edge.name.clone(), versions.clone());
versions
};
let mut reason = Omission::Unavailable;
for version in versions.into_iter().filter(|version| {
Version::from_str(version).is_ok_and(|version| version.satisfies(&rule))
}) {
self.attempts += 1;
if self.attempts > MAX_RESOLUTION_ATTEMPTS {
return Err(SearchError::Fatal(
"npm resolution exceeds its candidate budget".into(),
));
}
let spec = LibSpec {
ecosystem: Ecosystem::Npm,
name: request.edge.name.clone(),
version,
};
self.archive(&spec)?;
if !self.archives[&spec].1.packages["package/"]
.eligible(self.runtime)
.map_err(SearchError::Fatal)?
{
reason = Omission::Target;
continue;
}
reason = Omission::Conflict;
let mut candidate = state.clone();
self.add(&mut candidate, &location, &spec)?;
Self::bind(&mut candidate, &request, Some(location.clone()), None);
match self.search(candidate) {
Ok(resolved) => return Ok(resolved),
Err(SearchError::Conflict(_)) => {}
Err(fatal) => return Err(fatal),
}
}
if request.edge.kind == Kind::Optional && existing.is_none() {
Self::bind(&mut state, &request, None, Some(reason));
continue;
}
return Err(SearchError::Conflict(format!(
"no npm version satisfies {} at {location}",
request.edge.name
)));
}
let edges = state
.nodes
.values()
.flat_map(|node| {
node.edges.iter().map(|edge| Request {
consumer: Some(node.path.clone()),
edge: edge.clone(),
})
})
.collect::<Vec<_>>();
for request in edges {
let found = visible(
&state.nodes,
request.consumer.as_deref().unwrap(),
&request.edge.name,
matches!(request.edge.kind, Kind::Peer | Kind::OptionalPeer),
);
if let Some(found) = found {
let range = request
.edge
.requirement
.parse::<Range>()
.map_err(|error| SearchError::Fatal(error.to_string()))?;
if !Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&range))
{
return Err(SearchError::Conflict(
"npm dependency or peer conflicts with final Node lookup".into(),
));
}
let target = found.path.clone();
Self::bind(&mut state, &request, Some(target), None);
} else if request.edge.kind == Kind::OptionalPeer {
Self::bind(
&mut state,
&request,
None,
Some(Omission::OptionalPeerAbsent),
);
} else if request.edge.kind != Kind::Optional || request.edge.omission.is_none() {
return Err(SearchError::Conflict(
"npm required dependency is absent from final Node lookup".into(),
));
}
}
Ok(state)
}
}
+524
View File
@@ -0,0 +1,524 @@
//! Authenticated Go checksum database receipts. Trust comes from the verifier
//! supplied by the caller, never from a key stored in a project's lock file.
//! Formats follow golang.org/x/mod/sumdb/{note,tlog,dirhash} and RFC 6962.
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Mutex, MutexGuard};
use base64::{Engine, engine::general_purpose::STANDARD};
use ring::signature::{ED25519, UnparsedPublicKey};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::{RegistryTransport, go_path, zip_entries};
mod merkle;
pub const OFFICIAL_VERIFIER: &str =
"sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8";
const MAX_NOTE_BYTES: usize = 16 * 1024;
const MAX_RECORDS: usize = 1024;
const MAX_EVIDENCE_BYTES: usize = 16 * 1024 * 1024;
type Hash = [u8; 32];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct Tree {
size: u64,
hash: Hash,
}
fn encode_hash(hash: Hash) -> String {
STANDARD.encode(hash)
}
fn decode_hash(value: &str) -> Result<Hash, String> {
STANDARD
.decode(value)
.map_err(|_| "sumdb hash is not base64")?
.try_into()
.map_err(|_| "sumdb hash must contain 32 bytes".into())
}
/// Explicit verifier injection supports registry protocol tests and author
/// tooling. Released project commands always select `official()`.
#[derive(Clone)]
pub struct Verifier {
name: String,
key_hash: [u8; 4],
key: [u8; 32],
}
impl Verifier {
pub fn official() -> Self {
Self::new(OFFICIAL_VERIFIER).expect("fixed official sumdb verifier")
}
pub fn new(encoded: &str) -> Result<Self, String> {
// '+' also occurs inside base64; only the first two separators delimit.
let (name, rest) = encoded.split_once('+').ok_or("invalid sumdb verifier")?;
let (key_hash, encoded_key) = rest.split_once('+').ok_or("invalid sumdb verifier")?;
if name.is_empty() || name.chars().any(char::is_whitespace) || key_hash.len() != 8 {
return Err("invalid sumdb verifier identity".into());
}
let expected = u32::from_str_radix(key_hash, 16)
.map_err(|_| "invalid sumdb key hash")?
.to_be_bytes();
let key = STANDARD
.decode(encoded_key)
.map_err(|_| "invalid sumdb verifier encoding")?;
if key.len() != 33 || key[0] != 1 {
return Err("sumdb requires an Ed25519 verifier".into());
}
let mut hash = Sha256::new();
hash.update(name.as_bytes());
hash.update(b"\n");
hash.update(&key);
if hash.finalize()[..4] != expected {
return Err("sumdb verifier key hash mismatch".into());
}
Ok(Self {
name: name.into(),
key_hash: expected,
key: key[1..].try_into().expect("validated Ed25519 key length"),
})
}
fn checkpoint(&self, note: &str) -> Result<Tree, String> {
if note.len() > MAX_NOTE_BYTES || note.bytes().any(|byte| byte < 32 && byte != b'\n') {
return Err("invalid sumdb checkpoint note".into());
}
let (text, signatures) = note
.rsplit_once("\n\n")
.ok_or("sumdb checkpoint lacks signatures")?;
let message = format!("{text}\n");
if !signatures.ends_with('\n') {
return Err("unterminated sumdb signature".into());
}
let mut verified = false;
for (index, line) in signatures.lines().enumerate() {
if index >= 100 {
return Err("sumdb checkpoint has too many signatures".into());
}
let (name, encoded) = line
.strip_prefix("— ")
.and_then(|line| line.split_once(' '))
.ok_or("malformed sumdb signature")?;
let signature = STANDARD
.decode(encoded)
.map_err(|_| "invalid sumdb signature encoding")?;
if name.is_empty() || name.chars().any(char::is_whitespace) || signature.len() < 5 {
return Err("malformed sumdb signature identity".into());
}
if name == self.name && signature[..4] == self.key_hash {
UnparsedPublicKey::new(&ED25519, self.key)
.verify(message.as_bytes(), &signature[4..])
.map_err(|_| "sumdb checkpoint signature verification failed")?;
verified = true;
}
}
if !verified {
return Err("sumdb checkpoint has no trusted signature".into());
}
let mut lines = text.lines();
if lines.next() != Some("go.sum database tree") {
return Err("unsupported sumdb checkpoint format".into());
}
let size_text = lines.next().ok_or("sumdb checkpoint lacks tree size")?;
let size = decimal(size_text)?;
let hash = decode_hash(lines.next().ok_or("sumdb checkpoint lacks root hash")?)?;
let empty: Hash = Sha256::digest([]).into();
if size == 0 && hash != empty {
return Err("invalid sumdb empty root".into());
}
Ok(Tree { size, hash })
}
}
fn decimal(value: &str) -> Result<u64, String> {
let number = value
.parse::<u64>()
.map_err(|_| "invalid sumdb record/tree number")?;
if number > i64::MAX as u64 || value != number.to_string() {
return Err("noncanonical sumdb record/tree number".into());
}
Ok(number)
}
struct Lookup<'a> {
id: u64,
text: &'a str,
checkpoint: &'a str,
}
fn lookup(value: &str) -> Result<Lookup<'_>, String> {
if value.len() > MAX_NOTE_BYTES || value.bytes().any(|byte| byte < 32 && byte != b'\n') {
return Err("invalid sumdb lookup text".into());
}
let (id, rest) = value
.split_once('\n')
.ok_or("sumdb lookup lacks record number")?;
let end = rest
.find("\n\n")
.ok_or("sumdb lookup lacks record terminator")?;
Ok(Lookup {
id: decimal(id)?,
text: &rest[..end + 1],
checkpoint: &rest[end + 2..],
})
}
fn hashes(record: &str, module: &str, version: &str) -> Result<(String, String), String> {
let version = format!("v{version}");
let mod_version = format!("{version}/go.mod");
let mut values = BTreeMap::new();
for line in record.lines() {
let parts = line.split(' ').collect::<Vec<_>>();
if parts.len() != 3
|| parts[0] != module
|| (parts[1] != version && parts[1] != mod_version)
{
return Err("sumdb lookup record has a different module/version identity".into());
}
decode_hash(
parts[2]
.strip_prefix("h1:")
.ok_or("sumdb record requires h1 hashes")?,
)?;
if values.insert(parts[1], parts[2]).is_some() {
return Err("sumdb lookup repeats a module checksum".into());
}
}
Ok((
values
.get(mod_version.as_str())
.ok_or("sumdb lookup lacks go.mod hash")?
.to_string(),
values
.get(version.as_str())
.ok_or("sumdb lookup lacks module ZIP hash")?
.to_string(),
))
}
fn dirhash(files: &[(String, Vec<u8>)]) -> Result<String, String> {
let mut files = files.iter().collect::<Vec<_>>();
files.sort_by(|left, right| left.0.cmp(&right.0));
let mut seen = BTreeSet::new();
let mut hash = Sha256::new();
for (name, bytes) in files {
if name.contains('\n') || !seen.insert(name) {
return Err("invalid or duplicate Go dirhash filename".into());
}
hash.update(format!("{} {name}\n", crate::libs::sha256(bytes)));
}
Ok(format!("h1:{}", STANDARD.encode(hash.finalize())))
}
pub fn mod_hash(bytes: &[u8]) -> String {
// The Go protocol hashes a synthetic directory containing only go.mod.
dirhash(&[("go.mod".into(), bytes.to_vec())]).expect("fixed dirhash filename")
}
pub fn zip_hash(module: &str, version: &str, bytes: &[u8]) -> Result<String, String> {
let files = zip_entries(bytes)?;
let prefix = format!("{module}@v{version}/");
if files.iter().any(|(path, _)| !path.starts_with(&prefix)) {
return Err("Go module archive has a mismatched module root".into());
}
dirhash(&files)
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Record {
pub module: String,
pub version: String,
pub go_mod: String,
pub lookup: String,
pub inclusion: Vec<String>,
pub consistency: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Evidence {
pub previous: Option<String>,
pub checkpoint: String,
pub consistency: Vec<String>,
pub records: Vec<Record>,
}
#[derive(Debug)]
pub struct Verified {
pub checkpoint: String,
checksums: BTreeMap<(String, String), String>,
}
impl Verified {
pub fn verify_zip(&self, module: &str, version: &str, bytes: &[u8]) -> Result<(), String> {
let expected = self
.checksums
.get(&(module.into(), version.into()))
.ok_or("Go module lacks authenticated sumdb evidence")?;
if zip_hash(module, version, bytes)? != *expected {
return Err("Go module ZIP does not match authenticated sumdb h1".into());
}
Ok(())
}
pub fn contains(&self, module: &str, version: &str) -> bool {
self.checksums
.contains_key(&(module.into(), version.into()))
}
}
impl Evidence {
pub fn verify(&self, verifier: &Verifier) -> Result<Verified, String> {
if self.records.len() > MAX_RECORDS
|| serde_json::to_vec(self).map_err(|e| e.to_string())?.len() > MAX_EVIDENCE_BYTES
{
return Err("sumdb evidence exceeds its size budget".into());
}
let mut current = verifier.checkpoint(&self.checkpoint)?;
if let Some(previous) = &self.previous {
let older = verifier.checkpoint(previous)?;
if older.size > current.size {
return Err("sumdb checkpoint rollback rejected".into());
}
merkle::check_consistency(&self.consistency, older, current)?;
} else if !self.consistency.is_empty() {
return Err("sumdb first checkpoint has an unexpected consistency proof".into());
}
let mut latest = self.checkpoint.clone();
let mut checksums = BTreeMap::new();
for record in &self.records {
let entry = lookup(&record.lookup)?;
let tree = verifier.checkpoint(entry.checkpoint)?;
merkle::check_record(&record.inclusion, tree, entry.id, entry.text.as_bytes())?;
let (older, newer) = if tree.size < current.size {
(tree, current)
} else {
(current, tree)
};
merkle::check_consistency(&record.consistency, older, newer)?;
let (mod_h1, zip_h1) = hashes(entry.text, &record.module, &record.version)?;
if mod_hash(record.go_mod.as_bytes()) != mod_h1 {
return Err("Go go.mod does not match authenticated sumdb h1".into());
}
if checksums
.insert((record.module.clone(), record.version.clone()), zip_h1)
.is_some()
{
return Err("sumdb evidence repeats a module/version".into());
}
if tree.size > current.size {
current = tree;
latest = entry.checkpoint.into();
}
}
Ok(Verified {
checkpoint: latest,
checksums,
})
}
}
/// One explicit resolution sequence owns the checkpoint. The project lock is
/// its persistent anchor; deleting that lock intentionally starts first trust.
pub struct ChecksumDatabase {
verifier: Verifier,
latest: Mutex<Option<String>>,
}
impl ChecksumDatabase {
pub fn verifier(&self) -> &Verifier {
&self.verifier
}
pub fn new(verifier: Verifier, previous: Option<String>) -> Result<Self, String> {
if let Some(note) = &previous {
verifier.checkpoint(note)?;
}
Ok(Self {
verifier,
latest: Mutex::new(previous),
})
}
pub fn official(previous: Option<String>) -> Result<Self, String> {
Self::new(Verifier::official(), previous)
}
pub(super) fn begin<'a>(
&'a self,
transport: &'a dyn RegistryTransport,
) -> Result<Session<'a>, String> {
let guard = self
.latest
.lock()
.map_err(|_| "sumdb checkpoint state poisoned")?;
let checkpoint = String::from_utf8(transport.get_sumdb("/latest", MAX_NOTE_BYTES)?)
.map_err(|_| "sumdb checkpoint is not UTF-8")?;
let current = self.verifier.checkpoint(&checkpoint)?;
let mut tiles = merkle::Tiles::new(transport);
let consistency = if let Some(previous) = &*guard {
let older = self.verifier.checkpoint(previous)?;
if older.size > current.size {
return Err("sumdb checkpoint rollback rejected".into());
}
tiles.consistency(older, current)?
} else {
Vec::new()
};
let evidence = Evidence {
previous: guard.clone(),
checkpoint: checkpoint.clone(),
consistency,
records: Vec::new(),
};
Ok(Session {
verifier: &self.verifier,
transport,
guard,
tiles,
current,
latest: checkpoint,
evidence,
})
}
}
pub(super) struct Session<'a> {
verifier: &'a Verifier,
transport: &'a dyn RegistryTransport,
guard: MutexGuard<'a, Option<String>>,
tiles: merkle::Tiles<'a>,
current: Tree,
latest: String,
evidence: Evidence,
}
impl Session<'_> {
pub(super) fn verify_mod(
&mut self,
module: &str,
version: &str,
source: &[u8],
) -> Result<(), String> {
if let Some(record) = self
.evidence
.records
.iter()
.find(|record| record.module == module && record.version == version)
{
if record.go_mod.as_bytes() != source {
return Err("Go go.mod changed during resolution".into());
}
return Ok(());
}
if self.evidence.records.len() >= MAX_RECORDS {
return Err("sumdb resolution exceeds record limit".into());
}
let path = format!("/lookup/{}@v{}", go_path(module), go_path(version));
let response = String::from_utf8(self.transport.get_sumdb(&path, MAX_NOTE_BYTES)?)
.map_err(|_| "sumdb lookup is not UTF-8")?;
let entry = lookup(&response)?;
let tree = self.verifier.checkpoint(entry.checkpoint)?;
let inclusion = self.tiles.inclusion(tree.size, entry.id)?;
merkle::check_record(&inclusion, tree, entry.id, entry.text.as_bytes())?;
let (older, newer) = if tree.size < self.current.size {
(tree, self.current)
} else {
(self.current, tree)
};
let consistency = self.tiles.consistency(older, newer)?;
let (expected, _) = hashes(entry.text, module, version)?;
if mod_hash(source) != expected {
return Err("Go go.mod does not match authenticated sumdb h1".into());
}
if tree.size > self.current.size {
self.current = tree;
self.latest = entry.checkpoint.into();
}
let record = Record {
module: module.into(),
version: version.into(),
go_mod: String::from_utf8(source.to_vec()).map_err(|_| "Go go.mod is not UTF-8")?,
lookup: response,
inclusion,
consistency,
};
self.evidence.records.push(record);
if serde_json::to_vec(&self.evidence)
.map_err(|e| e.to_string())?
.len()
> MAX_EVIDENCE_BYTES
{
return Err("sumdb evidence exceeds its size budget".into());
}
Ok(())
}
pub(super) fn verify_zip(
&self,
module: &str,
version: &str,
bytes: &[u8],
) -> Result<(), String> {
let record = self
.evidence
.records
.iter()
.find(|record| record.module == module && record.version == version)
.ok_or("Go module ZIP has no authenticated metadata")?;
let (_, expected) = hashes(lookup(&record.lookup)?.text, module, version)?;
if zip_hash(module, version, bytes)? != expected {
return Err("Go module ZIP does not match authenticated sumdb h1".into());
}
Ok(())
}
pub(super) fn finish(mut self) -> Evidence {
*self.guard = Some(self.latest);
self.evidence
}
}
pub fn verify_chain(evidence: &[Evidence], verifier: &Verifier) -> Result<Verified, String> {
let mut latest = None;
let mut checksums = BTreeMap::new();
if evidence.len() > MAX_RECORDS {
return Err("sumdb resolution history exceeds its limit".into());
}
let mut bytes = 0usize;
for receipt in evidence {
bytes = bytes
.checked_add(
serde_json::to_vec(receipt)
.map_err(|error| error.to_string())?
.len(),
)
.ok_or("sumdb resolution history size overflow")?;
if bytes > MAX_EVIDENCE_BYTES {
return Err("sumdb resolution history exceeds its size budget".into());
}
if let Some(previous) = &latest
&& receipt.previous.as_ref() != Some(previous)
{
return Err("sumdb resolution checkpoints are not connected".into());
}
let verified = receipt.verify(verifier)?;
for (identity, hash) in verified.checksums {
if let Some(old) = checksums.insert(identity, hash.clone())
&& old != hash
{
return Err("sumdb resolutions disagree about module content".into());
}
}
latest = Some(verified.checkpoint);
}
Ok(Verified {
checkpoint: latest.unwrap_or_default(),
checksums,
})
}
@@ -0,0 +1,260 @@
//! RFC 6962 tree proofs and Go's height-eight tile addressing.
//! Protocol reference: golang.org/x/mod/sumdb/tlog (BSD-3-Clause).
use std::collections::BTreeMap;
use sha2::{Digest, Sha256};
use super::{Hash, Tree, decode_hash, encode_hash};
use crate::libs::RegistryTransport;
pub(super) fn leaf(bytes: &[u8]) -> Hash {
let mut hash = Sha256::new();
hash.update([0]);
hash.update(bytes);
hash.finalize().into()
}
fn node(left: Hash, right: Hash) -> Hash {
let mut hash = Sha256::new();
hash.update([1]);
hash.update(left);
hash.update(right);
hash.finalize().into()
}
fn split(size: u64) -> u64 {
1 << (63 - (size - 1).leading_zeros())
}
pub(super) fn check_record(
proof: &[String],
tree: Tree,
id: u64,
bytes: &[u8],
) -> Result<(), String> {
if id >= tree.size || proof.len() > 63 {
return Err("sumdb record proof has invalid bounds".into());
}
fn root(proof: &[String], size: u64, id: u64, value: Hash) -> Result<Hash, String> {
if size == 1 {
return if proof.is_empty() {
Ok(value)
} else {
Err("sumdb inclusion proof has surplus hashes".into())
};
}
let (last, rest) = proof
.split_last()
.ok_or("sumdb inclusion proof is incomplete")?;
let sibling = decode_hash(last)?;
let left = split(size);
if id < left {
Ok(node(root(rest, left, id, value)?, sibling))
} else {
Ok(node(sibling, root(rest, size - left, id - left, value)?))
}
}
if root(proof, tree.size, id, leaf(bytes))? != tree.hash {
return Err("sumdb inclusion proof does not match signed checkpoint".into());
}
Ok(())
}
pub(super) fn check_consistency(proof: &[String], older: Tree, newer: Tree) -> Result<(), String> {
if older.size > newer.size || proof.len() > 63 {
return Err("sumdb consistency proof has invalid bounds".into());
}
if older.size == 0 {
let empty: Hash = Sha256::digest([]).into();
return if proof.is_empty() && older.hash == empty {
Ok(())
} else {
Err("sumdb empty checkpoint is invalid".into())
};
}
fn roots(
proof: &[String],
lo: u64,
size: u64,
old_size: u64,
old_hash: Hash,
) -> Result<(Hash, Hash), String> {
if old_size == size {
return if lo == 0 && proof.is_empty() {
Ok((old_hash, old_hash))
} else if lo != 0 && proof.len() == 1 {
let hash = decode_hash(&proof[0])?;
Ok((hash, hash))
} else {
Err("sumdb consistency proof has surplus or missing hashes".into())
};
}
let (last, rest) = proof
.split_last()
.ok_or("sumdb consistency proof is incomplete")?;
let sibling = decode_hash(last)?;
let left = split(size);
if old_size <= left {
let (old, new) = roots(rest, lo, left, old_size, old_hash)?;
Ok((old, node(new, sibling)))
} else {
let (old, new) = roots(rest, lo + left, size - left, old_size - left, old_hash)?;
Ok((node(sibling, old), node(sibling, new)))
}
}
let (old, new) = roots(proof, 0, newer.size, older.size, older.hash)?;
if old != older.hash || new != newer.hash {
return Err("sumdb consistency proof detects a conflicting history".into());
}
Ok(())
}
/// Fetched tiles are only inputs to proofs; no byte is trusted merely because
/// it is cached. Persisted Evidence contains only completely verified proofs.
pub(super) struct Tiles<'a> {
transport: &'a dyn RegistryTransport,
cache: BTreeMap<String, Vec<u8>>,
}
impl<'a> Tiles<'a> {
pub(super) fn new(transport: &'a dyn RegistryTransport) -> Self {
Self {
transport,
cache: BTreeMap::new(),
}
}
fn subtree(&mut self, lo: u64, size: u64, tree_size: u64) -> Result<Hash, String> {
if size.is_power_of_two() {
let level = size.trailing_zeros();
let tile_level = level / 8;
let tile_scale = tile_level * 8;
let count = 1u64 << (level % 8);
let first = lo >> tile_scale;
let index = first / 256;
let offset = first % 256;
let width = (tree_size >> tile_scale)
.checked_sub(index * 256)
.ok_or("sumdb tile lies outside checkpoint")?
.min(256);
if offset + count > width {
return Err("sumdb tile does not cover requested subtree".into());
}
let mut chunks = vec![format!("{:03}", index % 1000)];
let mut prefix = index / 1000;
while prefix > 0 {
chunks.push(format!("x{:03}", prefix % 1000));
prefix /= 1000;
}
chunks.reverse();
let full = format!("/tile/8/{tile_level}/{}", chunks.join("/"));
let path = if width == 256 {
full.clone()
} else {
format!("{full}.p/{width}")
};
if !self.cache.contains_key(&path) {
if self.cache.len() >= 4096 {
return Err("sumdb tile cache exceeds its 32 MiB budget".into());
}
let bytes = match self.transport.get_sumdb(&path, 8192) {
Ok(bytes) => bytes,
Err(_) if width < 256 => {
let mut bytes = self.transport.get_sumdb(&full, 8192)?;
if bytes.len() != 8192 {
return Err("sumdb full tile has wrong length".into());
}
bytes.truncate(width as usize * 32);
bytes
}
Err(error) => return Err(error),
};
if bytes.len() != width as usize * 32 {
return Err("sumdb tile has wrong length".into());
}
self.cache.insert(path.clone(), bytes);
}
fn fold(bytes: &[u8]) -> Hash {
if bytes.len() == 32 {
return bytes.try_into().expect("one tile hash");
}
let (left, right) = bytes.split_at(bytes.len() / 2);
node(fold(left), fold(right))
}
return Ok(fold(
&self.cache[&path][offset as usize * 32..(offset + count) as usize * 32],
));
}
let left = split(size);
Ok(node(
self.subtree(lo, left, tree_size)?,
self.subtree(lo + left, size - left, tree_size)?,
))
}
pub(super) fn inclusion(&mut self, size: u64, id: u64) -> Result<Vec<String>, String> {
fn walk(
tiles: &mut Tiles<'_>,
lo: u64,
size: u64,
id: u64,
total: u64,
proof: &mut Vec<String>,
) -> Result<(), String> {
if size == 1 {
return Ok(());
}
let left = split(size);
let sibling = if id < lo + left {
walk(tiles, lo, left, id, total, proof)?;
tiles.subtree(lo + left, size - left, total)?
} else {
walk(tiles, lo + left, size - left, id, total, proof)?;
tiles.subtree(lo, left, total)?
};
proof.push(encode_hash(sibling));
Ok(())
}
if id >= size {
return Err("sumdb record lies outside signed checkpoint".into());
}
let mut proof = Vec::new();
walk(self, 0, size, id, size, &mut proof)?;
Ok(proof)
}
pub(super) fn consistency(&mut self, older: Tree, newer: Tree) -> Result<Vec<String>, String> {
fn walk(
tiles: &mut Tiles<'_>,
lo: u64,
size: u64,
old_size: u64,
total: u64,
proof: &mut Vec<String>,
) -> Result<(), String> {
if old_size == size {
if lo != 0 {
proof.push(encode_hash(tiles.subtree(lo, size, total)?));
}
return Ok(());
}
let left = split(size);
let sibling = if old_size <= left {
walk(tiles, lo, left, old_size, total, proof)?;
tiles.subtree(lo + left, size - left, total)?
} else {
walk(tiles, lo + left, size - left, old_size - left, total, proof)?;
tiles.subtree(lo, left, total)?
};
proof.push(encode_hash(sibling));
Ok(())
}
let mut proof = Vec::new();
if older.size != 0 {
walk(self, 0, newer.size, older.size, newer.size, &mut proof)?;
}
check_consistency(&proof, older, newer)?;
Ok(proof)
}
}
+754
View File
@@ -0,0 +1,754 @@
use std::env;
use std::ffi::OsString;
use std::io::{self, Write};
use std::path::PathBuf;
use std::process::{Command, ExitCode};
use dever_cli::libs;
use dever_cli::toolchain::BuildTarget;
use dever_core::source::SourceMap;
mod api;
mod compile;
mod format;
mod test_runner;
mod worker;
#[cfg(test)]
#[path = "../../../test/dever-cli-tests/build_arguments.rs"]
mod build_arguments;
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
enum Action {
Check,
Api {
output: Option<PathBuf>,
},
Format {
check: bool,
},
Test,
Run {
tenant_id: Option<i64>,
arguments: Vec<OsString>,
},
Build {
output: PathBuf,
target: BuildTarget,
},
Clean,
TenantMigrate {
tenant_id: i64,
},
TenantOwner {
tenant_id: i64,
site: String,
user_id: i64,
},
TenantComponent {
tenant_id: i64,
component: String,
enabled: bool,
},
Lib {
command: String,
specs: Vec<String>,
target: BuildTarget,
},
Package {
command: String,
specs: Vec<String>,
},
}
impl Action {
fn loads_settings(&self) -> bool {
matches!(self, Self::Check | Self::Api { .. }) || self.compiles_application()
}
fn compiles_application(&self) -> bool {
matches!(
self,
Self::Run { .. }
| Self::Build { .. }
| Self::TenantMigrate { .. }
| Self::TenantOwner { .. }
| Self::TenantComponent { .. }
)
}
}
fn main() -> ExitCode {
match execute() {
Ok(code) => code,
Err(error) => {
// Preserve failure status even if stderr is unavailable.
let _ = writeln!(io::stderr().lock(), "{error}");
ExitCode::FAILURE
}
}
}
fn execute() -> Result<ExitCode, String> {
let arguments: Vec<_> = env::args_os().skip(1).collect();
if arguments.first().is_some_and(|argument| argument == "new") {
let (destination, markdown) = match &arguments[1..] {
[destination] if !destination.to_string_lossy().starts_with('-') => {
(destination, false)
}
[destination, option]
if !destination.to_string_lossy().starts_with('-') && option == "--markdown" =>
{
(destination, true)
}
_ => return Err("Usage: dever new <new-directory> [--markdown]".into()),
};
dever_cli::project::create(std::path::Path::new(destination), markdown)?;
writeln!(
io::stdout().lock(),
"created Dever project {}",
destination.to_string_lossy()
)
.map_err(|error| error.to_string())?;
return Ok(ExitCode::SUCCESS);
}
if arguments.as_slice() == [OsString::from("--dever-compile-worker")] {
worker::execute()?;
return Ok(ExitCode::SUCCESS);
}
if arguments.as_slice() == [OsString::from("--dever-health")] {
return Ok(ExitCode::SUCCESS);
}
if arguments.as_slice() == [OsString::from("--version")] {
writeln!(io::stdout().lock(), "{}", env!("CARGO_PKG_VERSION"))
.map_err(|error| error.to_string())?;
return Ok(ExitCode::SUCCESS);
}
if arguments.as_slice() == [OsString::from("--help")] {
writeln!(
io::stdout().lock(),
"Create a project: dever new <new-directory> [--markdown]"
)
.map_err(|error| error.to_string())?;
writeln!(io::stdout().lock(), "{USAGE}").map_err(|error| error.to_string())?;
return Ok(ExitCode::SUCCESS);
}
let (root, action) = arguments_for(&arguments)?;
if let Action::Lib {
command,
specs,
target,
} = &action
{
let message = dever_cli::libs::execute_for_target(command, &root, specs, *target)?;
if !message.is_empty() {
writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?;
}
return Ok(ExitCode::SUCCESS);
}
if let Action::Package { command, specs } = &action {
let message = dever_cli::packages::execute(command, &root, specs)?;
if !message.is_empty() {
writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?;
}
return Ok(ExitCode::SUCCESS);
}
if matches!(action, Action::Clean) {
let summary = dever_core::native::clean_project_artifacts(&root)
.map_err(|error| format!("cannot clean Dever artifacts: {error}"))?;
writeln!(
io::stdout().lock(),
"cleaned {} directories, {} files, {} bytes",
summary.directories,
summary.files,
summary.bytes,
)
.map_err(|error| error.to_string())?;
return Ok(ExitCode::SUCCESS);
}
let source_root = root.join("module");
let packages = if matches!(action, Action::Format { .. }) {
Vec::new()
} else {
dever_cli::packages::sources(&root)?
};
let sources = SourceMap::load_with_packages(
&source_root,
matches!(action, Action::Test | Action::Format { .. })
.then(|| root.join("test"))
.as_deref(),
&packages,
)
.map_err(|errors| {
errors
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n")
})?;
if !sources.files().iter().any(|source| !source.is_test()) {
return Err(format!(
"{}: no .dever or .dever.md source files",
source_root.display()
));
}
if action.compiles_application()
&& sources.files().iter().any(|source| {
source
.path()
.file_name()
.is_some_and(|name| name == "main.dever" || name == "main.dever.md")
})
{
return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into());
}
if let Action::Format { check } = action {
return format::execute(&root, &sources, check);
}
let settings = action
.loads_settings()
.then(|| dever_runtime::config::Settings::load_project(&root))
.transpose()?;
let checked = match settings.as_ref() {
Some(settings) => dever_core::check_with_settings(&sources, settings),
None => dever_core::check(&sources),
};
let program = checked.map_err(|errors| {
errors
.iter()
.map(|error| error.render(&sources))
.collect::<String>()
})?;
for warning in program.warnings() {
if SourceMap::is_standard(warning.primary.source) {
continue;
}
write!(io::stderr().lock(), "{}", warning.render(&sources))
.map_err(|error| error.to_string())?;
}
if let Action::Api { output } = action {
return api::write(&program, output.as_deref());
}
api::check_baseline(&program, &root)?;
if matches!(action, Action::Check) {
return Ok(ExitCode::SUCCESS);
}
if matches!(action, Action::Test) {
return test_runner::execute(&program, &sources);
}
let target = match &action {
Action::Build { target, .. } => *target,
_ => BuildTarget::host()?,
};
let external_report = if action.compiles_application() {
libs::prepare_program(&root, &program, target)?
} else {
None
};
if action.compiles_application() {
let errors = program.application_errors()?;
if !errors.is_empty() {
return Err(errors.iter().map(|error| error.render(&sources)).collect());
}
}
match &action {
Action::Check => unreachable!("check returned before target selection"),
Action::Lib { .. } => unreachable!("lib returned before source checking"),
Action::Package { .. } => unreachable!("package returned before source checking"),
Action::Api { .. } => unreachable!("api returned before baseline checking"),
Action::Format { .. } => unreachable!("fmt returned before checking"),
Action::Test => unreachable!("test returned before application compilation"),
Action::Run { .. }
| Action::Build { .. }
| Action::TenantMigrate { .. }
| Action::TenantOwner { .. }
| Action::TenantComponent { .. } => {}
Action::Clean => unreachable!("clean returned before source loading"),
}
let settings = settings
.as_ref()
.expect("application action loaded settings before checking");
program.validate_database_settings(settings)?;
let resources = libs::prepare_resources(&root, &program, target)?;
let native = compile::application(
&program,
&sources,
&settings.compilation_bindings(),
&resources,
target,
)?;
match action {
Action::Package { .. } => unreachable!("package returned before source checking"),
Action::Build { output, .. } => {
native.save(&output).map_err(|error| {
format!("cannot save executable '{}': {error}", output.display())
})?;
if let Some(report) = external_report {
eprintln!("{}", report.summary());
libs::write_report(&output, &report)?;
}
Ok(ExitCode::SUCCESS)
}
Action::Run {
tenant_id,
arguments,
} => {
let arguments = match tenant_id {
Some(tenant_id) => [
vec![
OsString::from("--dever-tenant-cmd"),
OsString::from(tenant_id.to_string()),
],
arguments,
]
.concat(),
None => arguments,
};
run_native(&native, &root, &arguments)
}
Action::TenantMigrate { tenant_id } => run_native(
&native,
&root,
&[
OsString::from("--dever-tenant-migrate"),
OsString::from(tenant_id.to_string()),
],
),
Action::TenantOwner {
tenant_id,
site,
user_id,
} => run_native(
&native,
&root,
&[
OsString::from("--dever-tenant-owner"),
OsString::from(tenant_id.to_string()),
OsString::from(site),
OsString::from(user_id.to_string()),
],
),
Action::TenantComponent {
tenant_id,
component,
enabled,
} => run_native(
&native,
&root,
&[
OsString::from("--dever-tenant-component"),
OsString::from(tenant_id.to_string()),
OsString::from(if enabled { "enable" } else { "disable" }),
OsString::from(component),
],
),
Action::Check
| Action::Api { .. }
| Action::Format { .. }
| Action::Test
| Action::Lib { .. }
| Action::Clean => {
unreachable!("check and fmt returned before native compilation")
}
}
}
fn run_native(
native: &dever_core::native::NativeProgram,
project_root: &PathBuf,
arguments: &[OsString],
) -> Result<ExitCode, String> {
stage_runtime_config(native.executable(), project_root)?;
let status = Command::new(native.executable())
.current_dir(project_root)
.args(arguments)
.status()
.map_err(|error| format!("cannot run native program: {error}"))?;
Ok(if status.success() {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
})
}
fn stage_runtime_config(
executable: &std::path::Path,
project_root: &std::path::Path,
) -> Result<(), String> {
let source = project_root.join("config");
let destination = executable
.parent()
.ok_or("native executable has no parent directory")?
.join("config");
copy_config_directory(&source, &destination)
}
fn copy_config_directory(
source: &std::path::Path,
destination: &std::path::Path,
) -> Result<(), String> {
let metadata = std::fs::symlink_metadata(source)
.map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(format!(
"project config '{}' must be a real directory",
source.display()
));
}
std::fs::create_dir(destination)
.map_err(|error| format!("cannot stage config '{}': {error}", destination.display()))?;
for entry in std::fs::read_dir(source)
.map_err(|error| format!("cannot read '{}': {error}", source.display()))?
{
let entry =
entry.map_err(|error| format!("cannot read '{}': {error}", source.display()))?;
let source_path = entry.path();
let destination_path = destination.join(entry.file_name());
let metadata = std::fs::symlink_metadata(&source_path)
.map_err(|error| format!("cannot inspect '{}': {error}", source_path.display()))?;
if metadata.file_type().is_symlink() {
return Err(format!(
"project config '{}' must not contain symbolic links",
source_path.display()
));
}
if metadata.is_dir() {
copy_config_directory(&source_path, &destination_path)?;
} else if metadata.is_file() {
std::fs::copy(&source_path, &destination_path).map_err(|error| {
format!("cannot stage config '{}': {error}", source_path.display())
})?;
} else {
return Err(format!(
"project config '{}' must contain only files and directories",
source_path.display()
));
}
}
Ok(())
}
fn arguments_for(arguments: &[OsString]) -> Result<(PathBuf, Action), String> {
match arguments {
[command, operation, root, rest @ ..]
if command == "lib"
&& matches!(
operation.to_str(),
Some("add" | "list" | "update" | "remove" | "doctor")
) =>
{
let command = operation.to_str().unwrap().to_owned();
let (specs, target) = lib_arguments(rest)?;
Ok((
root.into(),
Action::Lib {
command,
specs,
target,
},
))
}
[command, operation, root, rest @ ..]
if command == "package"
&& matches!(
operation.to_str(),
Some("add" | "list" | "update" | "remove" | "doctor")
) =>
{
let command = operation.to_str().unwrap().to_owned();
let specs = rest
.iter()
.map(|value| {
value
.to_str()
.map(str::to_owned)
.ok_or("Package spec must be UTF-8".to_owned())
})
.collect::<Result<Vec<_>, _>>()?;
Ok((root.into(), Action::Package { command, specs }))
}
[command, root] if command == "check" => Ok((root.into(), Action::Check)),
[command, root] if command == "api" => Ok((root.into(), Action::Api { output: None })),
[command, root, flag, output] if command == "api" && flag == "--output" => Ok((
root.into(),
Action::Api {
output: Some(output.into()),
},
)),
[command, root] if command == "fmt" => Ok((root.into(), Action::Format { check: false })),
[command, root, flag] if command == "fmt" && flag == "--check" => {
Ok((root.into(), Action::Format { check: true }))
}
[command, root] if command == "run" => Ok((
root.into(),
Action::Run {
tenant_id: None,
arguments: Vec::new(),
},
)),
[command, root, separator, name, input] if command == "run" && separator == "--" => Ok((
root.into(),
Action::Run {
tenant_id: None,
arguments: vec![name.clone(), input.clone()],
},
)),
[command, root, tenant, tenant_id, separator, name, input]
if command == "run" && tenant == "--tenant" && separator == "--" =>
{
let tenant_id = positive_tenant_id(tenant_id)?;
Ok((
root.into(),
Action::Run {
tenant_id: Some(tenant_id),
arguments: vec![name.clone(), input.clone()],
},
))
}
[command, root] if command == "test" => Ok((root.into(), Action::Test)),
[command, root] if command == "clean" => Ok((root.into(), Action::Clean)),
[command, root, rest @ ..] if command == "build" => {
let mut output = None;
let mut target = None;
let mut options = rest.iter();
while let Some(option) = options.next() {
match option.to_str() {
Some("--output") if output.is_none() => {
output = Some(PathBuf::from(option_value(&mut options, "--output")?));
}
Some("--target") if target.is_none() => {
target = Some(parse_target(option_value(&mut options, "--target")?)?);
}
_ => return Err("build accepts --output and --target exactly once; unknown or duplicate option".into()),
}
}
Ok((
root.into(),
Action::Build {
output: output.ok_or("build requires --output <new-file>")?,
target: target.map_or_else(BuildTarget::host, Ok)?,
},
))
}
[tenant, migrate, root, tenant_id] if tenant == "tenant" && migrate == "migrate" => {
let tenant_id = positive_tenant_id(tenant_id)?;
Ok((root.into(), Action::TenantMigrate { tenant_id }))
}
[tenant, owner, root, tenant_id, site, user_id]
if tenant == "tenant" && owner == "owner" =>
{
let tenant_id = positive_id(tenant_id, "tenant id")?;
let site = site
.to_str()
.ok_or_else(|| "site must be UTF-8".to_owned())?
.to_owned();
let user_id = positive_id(user_id, "user id")?;
Ok((
root.into(),
Action::TenantOwner {
tenant_id,
site,
user_id,
},
))
}
[
tenant,
component_command,
operation,
root,
tenant_id,
component,
] if tenant == "tenant"
&& component_command == "component"
&& (operation == "enable" || operation == "disable") =>
{
let tenant_id = positive_tenant_id(tenant_id)?;
let component = component
.to_str()
.ok_or_else(|| "component must be UTF-8".to_owned())?
.to_owned();
Ok((
root.into(),
Action::TenantComponent {
tenant_id,
component,
enabled: operation == "enable",
},
))
}
_ => Err(USAGE.into()),
}
}
fn option_value<'a>(
options: &mut impl Iterator<Item = &'a OsString>,
option: &str,
) -> Result<&'a OsString, String> {
options
.next()
.filter(|value| !value.is_empty() && !value.to_string_lossy().starts_with("--"))
.ok_or_else(|| format!("{option} requires a value"))
}
fn parse_target(value: &OsString) -> Result<BuildTarget, String> {
value.to_str().ok_or("build target must be UTF-8")?.parse()
}
fn lib_arguments(arguments: &[OsString]) -> Result<(Vec<String>, BuildTarget), String> {
let mut specs = Vec::new();
let mut target = None;
let mut arguments = arguments.iter();
while let Some(argument) = arguments.next() {
if argument == "--target" && target.is_none() {
target = Some(parse_target(option_value(&mut arguments, "--target")?)?);
} else {
let spec = argument.to_str().ok_or("lib spec must be UTF-8")?;
if spec.starts_with('-') {
return Err("unknown or duplicate lib option".into());
}
specs.push(spec.to_owned());
}
}
Ok((specs, target.map_or_else(BuildTarget::host, Ok)?))
}
fn positive_tenant_id(value: &OsString) -> Result<i64, String> {
positive_id(value, "tenant id")
}
fn positive_id(value: &OsString, label: &str) -> Result<i64, String> {
let id = value
.to_str()
.ok_or_else(|| format!("{label} must be a positive integer"))?
.parse::<i64>()
.map_err(|_| format!("{label} must be a positive integer"))?;
if id <= 0 {
return Err(format!("{label} must be a positive integer"));
}
Ok(id)
}
#[cfg(test)]
mod tests {
use super::{Action, arguments_for};
use std::ffi::OsString;
#[test]
fn run_accepts_only_one_named_json_command_after_separator() {
let args = [
"run",
"project",
"--",
"user.account.greeting",
"{\"name\":\"Ada\"}",
]
.map(OsString::from);
let (root, action) = arguments_for(&args).unwrap();
assert_eq!(root.to_string_lossy(), "project");
let Action::Run {
tenant_id,
arguments,
} = action
else {
panic!("expected run action")
};
assert_eq!(tenant_id, None);
assert_eq!(arguments, args[3..]);
assert!(arguments_for(&args[..4]).is_err());
assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err());
}
#[test]
fn tenant_command_uses_a_separate_positive_control_plane_id() {
let args = [
"run",
"project",
"--tenant",
"42",
"--",
"content.article.rebuild",
"{\"force\":true}",
]
.map(OsString::from);
let (root, action) = arguments_for(&args).unwrap();
assert_eq!(root.to_string_lossy(), "project");
let Action::Run {
tenant_id,
arguments,
} = action
else {
panic!("expected run action")
};
assert_eq!(tenant_id, Some(42));
assert_eq!(arguments, args[5..]);
for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] {
let args = [
"run",
"project",
"--tenant",
invalid,
"--",
"content.article.rebuild",
"{}",
]
.map(OsString::from);
let error = arguments_for(&args)
.err()
.expect("invalid tenant id must fail");
assert_eq!(error, "tenant id must be a positive integer");
}
}
#[test]
fn tenant_migrate_requires_a_positive_integer_id() {
let args = ["tenant", "migrate", "project", "42"].map(OsString::from);
let (root, action) = arguments_for(&args).unwrap();
assert_eq!(root.to_string_lossy(), "project");
assert!(matches!(action, Action::TenantMigrate { tenant_id: 42 }));
for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] {
let args = ["tenant", "migrate", "project", invalid].map(OsString::from);
let error = arguments_for(&args)
.err()
.expect("invalid tenant id must fail");
assert_eq!(error, "tenant id must be a positive integer");
}
}
#[test]
fn tenant_component_command_selects_one_manifest_component() {
for (operation, expected) in [("enable", true), ("disable", false)] {
let args =
["tenant", "component", operation, "project", "42", "news"].map(OsString::from);
let (root, action) = arguments_for(&args).unwrap();
assert_eq!(root.to_string_lossy(), "project");
let Action::TenantComponent {
tenant_id,
component,
enabled,
} = action
else {
panic!("expected tenant component action");
};
assert_eq!(tenant_id, 42);
assert_eq!(component, "news");
assert_eq!(enabled, expected);
}
let invalid =
["tenant", "component", "toggle", "project", "42", "news"].map(OsString::from);
assert!(arguments_for(&invalid).is_err());
}
#[test]
fn clean_accepts_exactly_one_project_root() {
let args = ["clean", "project"].map(OsString::from);
let (root, action) = arguments_for(&args).unwrap();
assert_eq!(root.to_string_lossy(), "project");
assert!(matches!(action, Action::Clean));
assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err());
}
}
File diff suppressed because it is too large. Load diff
+128
View File
@@ -0,0 +1,128 @@
//! Create complete starter projects without replacing caller-owned paths.
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
macro_rules! template {
($path:literal) => {
include_str!(concat!("../../../skills/assets/template/", $path))
};
}
const FILES: &[(&str, &str)] = &[
("config/setting.json", template!("config/setting.json")),
("AGENTS.md", template!("AGENTS.md")),
("README.md", template!("README.md")),
(".gitignore", template!(".gitignore")),
];
const SOURCES: &[(&str, &str, &str)] = &[
(
"module/hello/greeting/app",
template!("module/hello/greeting/app.dever"),
template!("module/hello/greeting/app.md"),
),
(
"module/hello/greeting/api",
template!("module/hello/greeting/api.dever"),
template!("module/hello/greeting/api.md"),
),
(
"test/hello/greeting/greet",
template!("test/hello/greeting/greet.dever"),
template!("test/hello/greeting/greet.md"),
),
];
/// Publish a new project beside its private staging directory. Existing targets,
/// including empty directories and dangling symlinks, are never replaced.
pub fn create(destination: &Path, markdown: bool) -> Result<(), String> {
let parent = destination
.parent()
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."));
let name = destination
.file_name()
.ok_or_else(|| "new project requires a new directory name".to_owned())?;
let parent = fs::canonicalize(parent)
.map_err(|error| format!("cannot resolve new project parent: {error}"))?;
let destination = parent.join(name);
match fs::symlink_metadata(&destination) {
Ok(_) => {
return Err(format!(
"project target already exists: {}",
destination.display()
));
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(format!("cannot inspect project target: {error}")),
}
let staging = Staging::create(&parent)?;
for (path, content) in FILES {
write(&staging.0, path, content)?;
}
for (path, source, document) in SOURCES {
let (suffix, content) = if markdown {
("dever.md", document.replace("{{source}}", source))
} else {
("dever", (*source).to_owned())
};
write(&staging.0, &format!("{path}.{suffix}"), &content)?;
}
publish(&staging.0, &destination)
}
fn write(root: &Path, path: &str, content: &str) -> Result<(), String> {
let path = root.join(path);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)
.map_err(|error| format!("cannot create template directory: {error}"))?;
}
fs::write(&path, content)
.map_err(|error| format!("cannot write template {}: {error}", path.display()))
}
struct Staging(PathBuf);
impl Staging {
fn create(parent: &Path) -> Result<Self, String> {
loop {
let sequence = NEXT_STAGING.fetch_add(1, Ordering::Relaxed);
let path = parent.join(format!(".dever-new-{}-{sequence}", std::process::id()));
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
match builder.create(&path) {
Ok(()) => return Ok(Self(path)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(format!("cannot stage new project: {error}")),
}
}
}
}
impl Drop for Staging {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
fn publish(staging: &Path, destination: &Path) -> Result<(), String> {
#[cfg(target_os = "linux")]
{
use rustix::fs::{CWD, RenameFlags, renameat_with};
renameat_with(CWD, staging, CWD, destination, RenameFlags::NOREPLACE)
.map_err(|error| format!("cannot publish new project: {error}"))
}
#[cfg(not(target_os = "linux"))]
{
let _ = (staging, destination);
Err("new project publication is not available for this platform".into())
}
}
+203
View File
@@ -0,0 +1,203 @@
use std::fs;
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode};
use std::sync::atomic::{AtomicU64, Ordering};
use dever_core::hir::{Program, TestCase};
use dever_core::source::SourceMap;
pub(super) fn execute(program: &Program, sources: &SourceMap) -> Result<ExitCode, String> {
let mut stdout = io::stdout().lock();
writeln!(stdout, "running {} tests", program.tests().len()).map_err(io_error)?;
let mut passed = 0;
let mut failures = Vec::new();
if !program.tests().is_empty() {
match crate::compile::test_suite(program, sources) {
Ok(native) => {
for (index, test) in program.tests().iter().enumerate() {
match run_case(program, test, &native, index) {
Ok(()) => {
passed += 1;
writeln!(stdout, "test {} ... ok", test.name()).map_err(io_error)?;
}
Err(failure) => {
writeln!(stdout, "test {} ... FAILED", test.name())
.map_err(io_error)?;
failures.push((test.name(), failure));
}
}
}
}
Err(message) => {
for test in program.tests() {
writeln!(stdout, "test {} ... FAILED", test.name()).map_err(io_error)?;
failures.push((test.name(), CaseFailure::message(message.clone())));
}
}
}
}
for (name, failure) in &failures {
writeln!(stdout, "\n---- {name} failure ----").map_err(io_error)?;
writeln!(stdout, "{}", failure.message).map_err(io_error)?;
write_output(&mut stdout, "stdout", &failure.stdout)?;
write_output(&mut stdout, "stderr", &failure.stderr)?;
}
let failed = failures.len();
let status = if failed == 0 { "ok" } else { "FAILED" };
writeln!(
stdout,
"\ntest result: {status}. {passed} passed; {failed} failed; 0 not run"
)
.map_err(io_error)?;
Ok(if failed == 0 {
ExitCode::SUCCESS
} else {
ExitCode::FAILURE
})
}
fn run_case(
program: &Program,
test: &TestCase,
native: &dever_core::native::NativeProgram,
index: usize,
) -> Result<(), CaseFailure> {
let project = TemporaryProject::new().map_err(CaseFailure::message)?;
if test.uses_database() {
write_settings(project.path(), test).map_err(CaseFailure::message)?;
let settings = dever_runtime::config::Settings::load_project(project.path())
.map_err(CaseFailure::message)?;
let profile = program
.validate_database_settings(&settings)
.map_err(CaseFailure::message)?;
if !profile.sqlite || profile.postgres {
return Err(CaseFailure::message(
"generated test settings must select SQLite only".into(),
));
}
}
let executable = project
.path()
.join(format!("test-program{}", std::env::consts::EXE_SUFFIX));
native.save(&executable).map_err(|error| {
CaseFailure::message(format!(
"cannot stage test executable '{}': {error}",
executable.display()
))
})?;
let output = Command::new(&executable)
.arg(index.to_string())
.current_dir(project.path())
.output()
.map_err(|error| CaseFailure::message(format!("cannot run test process: {error}")))?;
if output.status.success() {
Ok(())
} else {
Err(CaseFailure {
message: format!("test process exited with {}", output.status),
stdout: output.stdout,
stderr: output.stderr,
})
}
}
fn write_settings(root: &Path, test: &TestCase) -> Result<(), String> {
let config = root.join("config");
fs::create_dir(&config)
.map_err(|error| format!("cannot create '{}': {error}", config.display()))?;
let mut databases = serde_json::Map::new();
for name in test.database_connections() {
databases.insert(
name.clone(),
serde_json::json!({
"type": "sqlite",
"path": format!("data/db/{name}.db"),
"max_connections": 1,
"max_page_size": 100
}),
);
}
let document = serde_json::json!({ "database": databases });
let path = config.join("setting.json");
let bytes = serde_json::to_vec_pretty(&document)
.map_err(|error| format!("cannot encode test settings: {error}"))?;
fs::write(&path, bytes).map_err(|error| format!("cannot write '{}': {error}", path.display()))
}
fn write_output(output: &mut impl Write, label: &str, bytes: &[u8]) -> Result<(), String> {
if bytes.is_empty() {
return Ok(());
}
writeln!(output, "---- {label} ----").map_err(io_error)?;
output.write_all(bytes).map_err(io_error)?;
if !bytes.ends_with(b"\n") {
writeln!(output).map_err(io_error)?;
}
Ok(())
}
fn io_error(error: io::Error) -> String {
error.to_string()
}
struct CaseFailure {
message: String,
stdout: Vec<u8>,
stderr: Vec<u8>,
}
impl CaseFailure {
fn message(message: String) -> Self {
Self {
message,
stdout: Vec::new(),
stderr: Vec::new(),
}
}
}
struct TemporaryProject(PathBuf);
impl TemporaryProject {
fn new() -> Result<Self, String> {
static NEXT: AtomicU64 = AtomicU64::new(0);
loop {
let path = std::env::temp_dir().join(format!(
"dever-application-test-{}-{}",
std::process::id(),
NEXT.fetch_add(1, Ordering::Relaxed)
));
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
match builder.create(&path) {
Ok(()) => return Ok(Self(path)),
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
Err(error) => {
return Err(format!(
"cannot create temporary test project '{}': {error}",
path.display()
));
}
}
}
}
fn path(&self) -> &Path {
&self.0
}
}
impl Drop for TemporaryProject {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
+434
View File
@@ -0,0 +1,434 @@
//! Explicit Linux first installation. The caller must already trust this installer.
//! Release keys come from an independent administrator input, never the release.
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
use std::path::{Component, Path, PathBuf};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use super::release::{self, InstallLock};
use super::{Artifact, Layout, MachineManager, ReleaseManifest, Version};
mod sandbox;
mod service;
mod transaction;
pub use service::{RecoveryAction, recovery_actions};
pub(crate) const SERVICE: &str = "[Unit]\nDescription=Dever shared compiler service\nAfter=local-fs.target\n\n[Service]\nType=exec\nUser=root\nGroup=root\nExecStart=/opt/dever/bin/deverd --root /opt/dever\nRuntimeDirectory=dever\nRuntimeDirectoryMode=0755\nUMask=0077\nRestart=on-failure\nRestartSec=1\nKillMode=control-group\nTimeoutStopSec=30\nNoNewPrivileges=yes\nPrivateTmp=yes\nProtectHome=yes\nProtectSystem=strict\nReadWritePaths=/opt/dever/state /opt/dever/cache /run/dever\n\n[Install]\nWantedBy=multi-user.target\n";
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Settings {
release: PathBuf,
system_root: PathBuf,
trusted_key: PathBuf,
#[serde(default)]
activate_service: bool,
}
/// Private distribution entry; ordinary version install/update remains unchanged.
pub fn install(installer_root: &Path) -> Result<Version, String> {
require_root()?;
absolute_root(installer_root)?;
trusted_directory(installer_root)?;
let config = installer_root.join("config/setting.json");
trusted_file(&config)?;
let settings: Settings =
serde_json::from_slice(&read_bounded(&config, 1024 * 1024)?).map_err(|error| {
format!(
"invalid installer config at {}:{}",
error.line(),
error.column()
)
})?;
absolute_root(&settings.system_root)?;
trusted_directory(&settings.system_root)?;
if settings.system_root == Path::new("/") {
// Publishing an AppArmor file can trigger a later system-wide reload,
// even when this installer does not load profiles or activate services.
dever_sandbox::validate_host_policy()?;
}
if settings.activate_service && settings.system_root != Path::new("/") {
return Err(
"service activation requires the actual system root; image installs are offline".into(),
);
}
let package = absolute_input(installer_root, &settings.release)?;
let key_path = absolute_input(installer_root, &settings.trusted_key)?;
trusted_file(&key_path)?;
let package = fs::canonicalize(package).map_err(|error| error.to_string())?;
if key_path.starts_with(&package) {
return Err("trusted key must be supplied independently outside the release".into());
}
let key = release::decode_hex_file(&key_path, 32)?;
create_directory(&settings.system_root, "opt")?;
let layout = Layout::new(settings.system_root.join("opt/dever"));
if layout.root().exists() {
trusted_directory(layout.root())?;
}
for directory in [
layout.bin(),
layout.state(),
layout.versions(),
layout.downloads(),
layout.staging(),
layout.native_cache(),
] {
let relative = directory
.strip_prefix(&settings.system_root)
.map_err(|error| error.to_string())?
.to_str()
.ok_or("installer directory is not UTF-8")?;
create_directory(&settings.system_root, relative)?;
}
layout.validate_machine_permissions()?;
let _lock = InstallLock::acquire(&layout)?;
transaction::recover(&settings.system_root, &layout)?;
let pinned = layout.state().join("trusted-release-key");
if pinned.exists() {
trusted_file(&pinned)?;
if release::decode_hex_file(&pinned, 32)? != key {
return Err("installer key differs from the machine's pinned release key".into());
}
}
// The private copy is checked again by the version owner before any health
// command executes. A writable download cannot race signed bytes into it.
let manifest = MachineManager::verify_package(&package, None, &key)?;
validate_payload(&package, &manifest.artifacts, true)?;
pin_key(&layout, &key)?;
let manager = MachineManager::new(layout.clone());
if manager
.active_version()?
.is_some_and(|active| manifest.version < active)
{
return Err("bootstrap installation cannot downgrade the active compiler".into());
}
let installed = layout.versions().join(manifest.version.as_str());
if installed.exists()
&& super::sha256_file(&installed.join("manifest.json"))?
!= super::sha256_file(&package.join("manifest.json"))?
{
return Err("installed version already has a different signed release identity".into());
}
manager.install_package_locked(&package, &manifest.version)?;
transaction::install(
&settings.system_root,
&layout,
&installed,
&manifest,
&key,
settings.activate_service,
)?;
Ok(manifest.version)
}
/// Recover only this installer's fixed, recorded targets. Normal version
/// operations must finish a pending bootstrap transaction first.
pub fn recover(system_root: &Path) -> Result<(), String> {
require_root()?;
absolute_root(system_root)?;
trusted_directory(system_root)?;
let layout = Layout::new(system_root.join("opt/dever"));
layout.validate_machine_permissions()?;
let _lock = InstallLock::acquire(&layout)?;
transaction::recover(system_root, &layout)
}
pub(crate) fn validate_payload(
root: &Path,
artifacts: &[Artifact],
current_template: bool,
) -> Result<(), String> {
let scoped = artifacts
.iter()
.filter_map(|artifact| {
artifact
.path
.strip_prefix("bootstrap/")
.map(|path| Artifact {
path: path.into(),
bytes: artifact.bytes,
sha256: artifact.sha256.clone(),
})
})
.collect::<Vec<_>>();
validate_files(&root.join("bootstrap"), &scoped, current_template)
}
fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Result<(), String> {
let bytes = artifacts
.iter()
.try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes))
.ok_or("bootstrap size overflow")?;
if artifacts.len() > 64 || bytes > 64 * 1024 * 1024 {
return Err("bootstrap closure exceeds its 64-file/64-MiB budget".into());
}
for artifact in artifacts {
if !matches!(
artifact.path.as_str(),
"dever" | "deverd" | "deverd.service" | "dever-sandbox"
) && !artifact.path.starts_with("lib/")
{
return Err("bootstrap has an unsupported artifact".into());
}
let path = super::runtime_pack::checked_path(root, &artifact.path, true)?;
let bytes = read_bounded(&path, artifact.bytes)?;
if bytes.len() as u64 != artifact.bytes || digest(&bytes) != artifact.sha256 {
return Err(format!(
"bootstrap artifact '{}' differs from its signature",
artifact.path
));
}
}
if !artifacts
.iter()
.any(|artifact| artifact.path == "deverd.service")
|| new_payload && read_bounded(&root.join("deverd.service"), 8192)? != SERVICE.as_bytes()
{
return Err(
"bootstrap service definition is absent or differs from the supported contract".into(),
);
}
if new_payload
&& (!artifacts
.iter()
.any(|artifact| artifact.path == "dever-sandbox")
|| read_bounded(&root.join("dever-sandbox"), 8192)?
!= dever_sandbox::APPARMOR_PROFILE.as_bytes())
{
return Err(
"bootstrap AppArmor profile is absent or differs from the supported contract".into(),
);
}
super::core_libraries::validate_entries(root, artifacts, &["dever", "deverd"])
}
pub(super) fn health_check(path: &Path, flag: &str) -> Result<(), String> {
if !service::run(
std::process::Command::new(path).arg(flag),
"health-check installed executable",
)? {
return Err(format!(
"installed executable '{}' failed its health check",
path.display()
));
}
Ok(())
}
pub(super) fn require_no_pending(layout: &Layout) -> Result<(), String> {
if fs::symlink_metadata(layout.state().join("bootstrap-install.json")).is_ok() {
return Err(
"bootstrap installation is incomplete; rerun the explicit installer to recover it"
.into(),
);
}
Ok(())
}
fn validate_bin(root: &Path, key: &[u8]) -> Result<Vec<u8>, String> {
trusted_directory(root)?;
let manifest = MachineManager::signed_manifest(root, key)?;
let artifacts = manifest
.artifacts
.iter()
.filter_map(|artifact| {
artifact
.path
.strip_prefix("bootstrap/")
.map(|path| Artifact {
path: path.into(),
bytes: artifact.bytes,
sha256: artifact.sha256.clone(),
})
})
.collect::<Vec<_>>();
validate_files(root, &artifacts, false)?;
let expected = artifacts
.iter()
.map(|artifact| artifact.path.as_str())
.chain(["manifest.json", "manifest.sig"])
.collect::<std::collections::BTreeSet<_>>();
let found = transaction::files(root)?;
if found
.iter()
.map(String::as_str)
.collect::<std::collections::BTreeSet<_>>()
!= expected
{
return Err("installed bootstrap contains unsigned files".into());
}
read_bounded(&root.join("deverd.service"), 8192)
}
fn pin_key(layout: &Layout, key: &[u8]) -> Result<(), String> {
let path = layout.state().join("trusted-release-key");
let next = layout.state().join(".trusted-release-key.next");
if fs::symlink_metadata(&next).is_ok() {
trusted_file(&next)?;
if fs::metadata(&next)
.map_err(|error| error.to_string())?
.len()
> 65
{
return Err("temporary release key exceeds its fixed size".into());
}
fs::remove_file(&next).map_err(|error| error.to_string())?;
release::sync_directory(&layout.state()).map_err(|error| error.to_string())?;
}
if fs::symlink_metadata(&path).is_ok() {
trusted_file(&path)?;
if release::decode_hex_file(&path, 32)? != key {
return Err("installer key differs from the machine's pinned release key".into());
}
return Ok(());
}
write_new(&next, format!("{}\n", hex(key)).as_bytes(), 0o644)?;
rustix::fs::renameat_with(
rustix::fs::CWD,
&next,
rustix::fs::CWD,
&path,
rustix::fs::RenameFlags::NOREPLACE,
)
.map_err(|error| format!("cannot publish initial release key: {error}"))?;
release::sync_directory(&layout.state()).map_err(|error| error.to_string())
}
fn absolute_root(path: &Path) -> Result<(), String> {
if !path.is_absolute()
|| path
.components()
.any(|part| matches!(part, Component::ParentDir))
{
return Err("installer roots must be absolute paths without '..'".into());
}
Ok(())
}
fn require_root() -> Result<(), String> {
if rustix::process::geteuid().as_raw() != 0 {
return Err("machine bootstrap installation requires root".into());
}
Ok(())
}
fn absolute_input(root: &Path, path: &Path) -> Result<PathBuf, String> {
let path = if path.is_absolute() {
path.to_path_buf()
} else {
root.join(path)
};
if path
.components()
.any(|part| matches!(part, Component::ParentDir))
{
return Err("installer paths cannot contain '..'".into());
}
Ok(path)
}
fn trusted_directory(path: &Path) -> Result<(), String> {
trusted_path(path, true)
}
fn trusted_file(path: &Path) -> Result<(), String> {
trusted_path(path, false)
}
fn trusted_path(path: &Path, directory: bool) -> Result<(), String> {
let metadata = fs::symlink_metadata(path).map_err(|error| {
format!(
"cannot inspect installer path '{}': {error}",
path.display()
)
})?;
if metadata.file_type().is_symlink()
|| metadata.uid() != 0
|| metadata.mode() & 0o022 != 0
|| directory && !metadata.is_dir()
|| !directory && (!metadata.is_file() || metadata.nlink() != 1)
{
return Err(format!(
"installer path '{}' must be a real root-owned {} without shared write permission",
path.display(),
if directory { "directory" } else { "file" }
));
}
for ancestor in path.ancestors().skip(1) {
if ancestor.as_os_str().is_empty() {
continue;
}
let parent = fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?;
// A root-owned sticky temporary ancestor is safe for an owned fixture;
// the actual installation root itself may never be shared-writable.
if !parent.is_dir()
|| parent.file_type().is_symlink()
|| parent.uid() != 0
|| parent.mode() & 0o022 != 0 && parent.mode() & 0o1000 == 0
{
return Err("installer path crosses an untrusted ancestor".into());
}
}
Ok(())
}
fn create_directory(root: &Path, relative: &str) -> Result<PathBuf, String> {
let mut path = root.to_owned();
for part in Path::new(relative).components() {
let Component::Normal(part) = part else {
return Err("invalid installer directory".into());
};
path.push(part);
match fs::create_dir(&path) {
Ok(()) => {
fs::set_permissions(&path, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
release::sync_directory(&path).map_err(|error| error.to_string())?;
release::sync_directory(path.parent().ok_or("installer directory has no parent")?)
.map_err(|error| error.to_string())?;
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => return Err(error.to_string()),
}
trusted_directory(&path)?;
}
Ok(path)
}
fn read_bounded(path: &Path, maximum: u64) -> Result<Vec<u8>, String> {
let mut bytes = Vec::new();
File::open(path)
.and_then(|file| file.take(maximum.saturating_add(1)).read_to_end(&mut bytes))
.map_err(|error| error.to_string())?;
if bytes.len() as u64 > maximum {
return Err("installer input exceeds its size bound".into());
}
Ok(bytes)
}
fn write_new(path: &Path, bytes: &[u8], mode: u32) -> Result<(), String> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)
.map_err(|error| error.to_string())?;
file.write_all(bytes)
.and_then(|()| file.sync_all())
.map_err(|error| error.to_string())?;
fs::set_permissions(path, fs::Permissions::from_mode(mode))
.map_err(|error| error.to_string())?;
file.sync_all().map_err(|error| error.to_string())?;
release::sync_directory(path.parent().ok_or("installer file has no parent")?)
.map_err(|error| error.to_string())
}
fn digest(bytes: &[u8]) -> String {
format!("{:x}", Sha256::digest(bytes))
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
}
@@ -0,0 +1,138 @@
//! Preserve signed content-addressed helpers across compiler upgrades/removal.
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use super::{
MachineManager, ReleaseManifest, digest, read_bounded, release, trusted_directory, trusted_file,
};
const MAX_HELPERS: usize = 24;
const MAX_HELPER_BYTES: u64 = 32 * 1024 * 1024;
fn artifact_path() -> String {
format!(
"sandbox/{}/bin/bwrap",
crate::toolchain::platform_identity()
)
}
pub(super) fn stage(
installed: &Path,
next: &Path,
source: &Path,
manifest: &ReleaseManifest,
key: &[u8],
) -> Result<(), String> {
fs::create_dir(next).map_err(|error| error.to_string())?;
fs::set_permissions(next, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
let mut hashes = std::collections::BTreeSet::new();
if installed.exists() {
trusted_directory(installed)?;
for entry in fs::read_dir(installed).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let hash = entry
.file_name()
.into_string()
.map_err(|_| "sandbox helper identity is not UTF-8")?;
if hashes.len() >= MAX_HELPERS || !valid_hash(&hash) {
return Err("installed sandbox helper store has unsupported entries or exceeds its version bound".into());
}
validate_installed(&entry.path(), &hash, key)?;
let destination = next.join(&hash);
copy_helper(&entry.path().join("bwrap"), &entry.path(), &destination)?;
hashes.insert(hash);
}
}
if let Some(artifact) = manifest
.artifacts
.iter()
.find(|artifact| artifact.path == artifact_path())
&& !hashes.contains(&artifact.sha256)
{
if hashes.len() >= MAX_HELPERS {
return Err("sandbox helper version store is full; retained applications must be retired before removing old helpers".into());
}
let bytes = read_bounded(&source.join(&artifact.path), MAX_HELPER_BYTES)?;
if bytes.len() as u64 != artifact.bytes
|| dever_sandbox::validate_bwrap(&bytes)? != artifact.sha256
{
return Err("sandbox helper differs from its signed release".into());
}
let destination = next.join(&artifact.sha256);
copy_helper(&source.join(&artifact.path), source, &destination)?;
validate_installed(&destination, &artifact.sha256, key)?;
}
release::sync_tree(next)
}
fn copy_helper(helper: &Path, signed_release: &Path, destination: &Path) -> Result<(), String> {
fs::create_dir(destination).map_err(|error| error.to_string())?;
fs::set_permissions(destination, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
release::copy_file(helper, &destination.join("bwrap"), true)?;
for name in ["manifest.json", "manifest.sig"] {
release::copy_file(&signed_release.join(name), &destination.join(name), false)?;
}
Ok(())
}
fn valid_hash(hash: &str) -> bool {
hash.len() == 64
&& hash
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
fn validate_installed(root: &Path, hash: &str, key: &[u8]) -> Result<(), String> {
use std::os::unix::fs::MetadataExt;
trusted_directory(root)?;
let names = fs::read_dir(root)
.map_err(|error| error.to_string())?
.map(|entry| entry.map(|entry| entry.file_name()))
.collect::<Result<std::collections::BTreeSet<_>, _>>()
.map_err(|error| error.to_string())?;
let expected = ["bwrap", "manifest.json", "manifest.sig"]
.map(std::ffi::OsString::from)
.into_iter()
.collect();
if names != expected {
return Err("installed sandbox helper contains unsigned entries".into());
}
for name in ["bwrap", "manifest.json", "manifest.sig"] {
trusted_file(&root.join(name))?;
}
let helper = root.join("bwrap");
if fs::metadata(&helper)
.map_err(|error| error.to_string())?
.mode()
& 0o6111
!= 0o111
{
return Err("installed sandbox helper has unsupported executable permissions".into());
}
match rustix::fs::lgetxattr(&helper, "security.capability", &mut [0u8; 64]) {
Err(rustix::io::Errno::NODATA | rustix::io::Errno::OPNOTSUPP) => {}
Ok(_) => return Err("installed sandbox helper must not have file capabilities".into()),
Err(error) => {
return Err(format!(
"cannot inspect installed sandbox helper capabilities: {error}"
));
}
}
let manifest = MachineManager::signed_manifest(root, key)?;
let artifact = manifest
.artifacts
.iter()
.find(|artifact| artifact.path == artifact_path())
.ok_or("installed sandbox helper has no signed artifact")?;
let bytes = read_bounded(&helper, MAX_HELPER_BYTES)?;
if artifact.sha256 != hash || artifact.bytes != bytes.len() as u64 || digest(&bytes) != hash {
return Err(
"installed sandbox helper differs from its signature or directory identity".into(),
);
}
dever_sandbox::validate_bwrap(&bytes)?;
Ok(())
}
@@ -0,0 +1,174 @@
//! Bounded Linux platform actions. Image installation never calls systemctl.
use std::io::Read;
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};
struct OwnedChild(Child);
impl Drop for OwnedChild {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
pub(super) fn run(command: &mut Command, operation: &str) -> Result<bool, String> {
let mut child = OwnedChild(
command
.env_clear()
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
.map_err(|error| format!("cannot {operation}: {error}"))?,
);
let deadline = Instant::now() + Duration::from_secs(30);
loop {
if let Some(status) = child
.0
.try_wait()
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
{
return Ok(status.success());
}
if Instant::now() >= deadline {
return Err(format!("{operation} exceeded 30 seconds"));
}
std::thread::sleep(Duration::from_millis(20));
}
}
pub(super) fn active() -> Result<bool, String> {
if !loaded()? {
return Ok(false);
}
match property("ActiveState")?.as_str() {
"inactive" | "failed" => Ok(false),
"active" | "activating" | "deactivating" | "reloading" => Ok(true),
_ => Err("Dever service has an unsupported active state".into()),
}
}
pub(super) fn loaded() -> Result<bool, String> {
match property("LoadState")?.as_str() {
"loaded" => Ok(true),
"not-found" => Ok(false),
_ => Err("Dever service has an unsupported load state".into()),
}
}
fn property(name: &str) -> Result<String, String> {
let mut command = Command::new("/usr/bin/systemctl");
command
.env_clear()
.args(["show", "--value", "--property", name, "deverd.service"])
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null());
let mut child = OwnedChild(
command
.spawn()
.map_err(|error| format!("cannot inspect Dever service: {error}"))?,
);
let deadline = Instant::now() + Duration::from_secs(30);
let status = loop {
if let Some(status) = child.0.try_wait().map_err(|error| error.to_string())? {
break status;
}
if Instant::now() >= deadline {
return Err("service state inspection exceeded 30 seconds".into());
}
std::thread::sleep(Duration::from_millis(20));
};
let mut bytes = Vec::new();
child
.0
.stdout
.take()
.ok_or("service state output missing")?
.take(4097)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
if bytes.len() > 4096 {
return Err("service state output exceeded its bound".into());
}
let value = String::from_utf8(bytes)
.map_err(|_| "service state output is not UTF-8")?
.trim()
.to_owned();
if !status.success()
&& !(status.code() == Some(1) && name == "LoadState" && value == "not-found")
{
return Err(format!("cannot inspect Dever service {name}: {status}"));
}
Ok(value)
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum RecoveryAction {
Stop,
Reload,
Restart,
}
/// Files are restored between these two action lists. A service that never
/// existed is not stopped after its new unit has already been removed.
pub fn recovery_actions(
start_requested: bool,
was_active: bool,
currently_loaded: bool,
) -> (Vec<RecoveryAction>, Vec<RecoveryAction>) {
if !start_requested {
return (vec![], vec![]);
}
let before = if currently_loaded {
vec![RecoveryAction::Stop]
} else {
vec![]
};
let mut after = vec![RecoveryAction::Reload];
if was_active {
after.push(RecoveryAction::Restart);
}
(before, after)
}
pub(super) fn execute(actions: &[RecoveryAction]) -> Result<(), String> {
for action in actions {
let arguments = match action {
RecoveryAction::Reload => vec!["daemon-reload"],
RecoveryAction::Stop => vec!["stop", "deverd.service"],
RecoveryAction::Restart => vec!["restart", "deverd.service"],
};
if !run(
Command::new("/usr/bin/systemctl").args(&arguments),
"restore Dever service",
)? {
return Err(format!("systemctl {} failed", arguments.join(" ")));
}
}
Ok(())
}
pub(super) fn apply(restart: bool) -> Result<(), String> {
execute(&[
RecoveryAction::Reload,
if restart {
RecoveryAction::Restart
} else {
RecoveryAction::Stop
},
])
}
pub(super) fn ready(layout: &crate::toolchain::Layout) -> Result<(), String> {
let deadline = Instant::now() + Duration::from_secs(15);
loop {
match crate::toolchain::cache_status(layout) {
Ok(_) => return Ok(()),
Err(error) if Instant::now() >= deadline => {
return Err(format!("Dever service did not become ready: {error}"));
}
Err(_) => std::thread::sleep(Duration::from_millis(50)),
}
}
}
@@ -0,0 +1,522 @@
//! One bounded journal for the fixed bootstrap destinations. It contains
//! identities, never caller-selected recovery paths or arbitrary commands.
use std::collections::BTreeSet;
use std::fs;
use std::os::unix::fs::{MetadataExt, symlink};
use std::path::{Path, PathBuf};
use std::process::Command;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::{
Layout, ReleaseManifest, create_directory, digest, read_bounded, release, service,
trusted_directory, trusted_file, validate_bin, write_new,
};
const JOURNAL: &str = "bootstrap-install.json";
const JOURNAL_NEXT: &str = ".bootstrap-install.next";
const MAX_TREE_BYTES: u64 = 80 * 1024 * 1024;
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
enum Target {
Bin,
Sandbox,
Apparmor,
Service,
Entry,
Enable,
Active,
}
const TARGETS: [Target; 7] = [
Target::Sandbox,
Target::Apparmor,
Target::Bin,
Target::Service,
Target::Entry,
Target::Enable,
Target::Active,
];
impl Target {
fn relative(self) -> &'static str {
match self {
Self::Bin => "opt/dever/bin",
Self::Sandbox => "opt/dever/sandbox",
Self::Apparmor => "etc/apparmor.d/dever-sandbox",
Self::Service => "etc/systemd/system/deverd.service",
Self::Entry => "usr/local/bin/dever",
Self::Enable => "etc/systemd/system/multi-user.target.wants/deverd.service",
Self::Active => "opt/dever/state/active-version",
}
}
fn path(self, root: &Path) -> PathBuf {
root.join(self.relative())
}
fn next(self, root: &Path) -> PathBuf {
let path = self.path(root);
path.with_file_name(format!(
".dever-bootstrap-{}.next",
path.file_name().unwrap().to_string_lossy()
))
}
fn link(self) -> Option<&'static str> {
match self {
Self::Entry => Some("/opt/dever/bin/dever"),
Self::Enable => Some("../deverd.service"),
_ => None,
}
}
}
#[derive(Clone, Copy, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
enum Phase {
Preparing,
Prepared,
Committed,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Change {
target: Target,
before: Option<String>,
after: Option<String>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
struct Journal {
format: String,
phase: Phase,
service_was_active: Option<bool>,
service_start_requested: bool,
changes: Vec<Change>,
}
pub(super) fn install(
root: &Path,
layout: &Layout,
source: &Path,
manifest: &ReleaseManifest,
key: &[u8],
activate_service: bool,
) -> Result<(), String> {
let previous_service = if fs::read_dir(layout.bin())
.map_err(|error| error.to_string())?
.next()
.is_some()
{
Some(validate_bin(&layout.bin(), key)?)
} else {
None
};
for target in TARGETS {
let parent = Path::new(target.relative())
.parent()
.unwrap()
.to_str()
.unwrap();
create_directory(root, parent)?;
if fs::symlink_metadata(target.next(root)).is_ok() {
return Err("unrecorded bootstrap staging destination already exists".into());
}
if let Some(link) = target.link() {
if let Ok(metadata) = fs::symlink_metadata(target.path(root))
&& (previous_service.is_none()
|| metadata.uid() != 0
|| !metadata.file_type().is_symlink()
|| fs::read_link(target.path(root)).map_err(|error| error.to_string())?
!= Path::new(link))
{
return Err(
"bootstrap cannot replace an unrelated public entry or service link".into(),
);
}
} else if target == Target::Apparmor && target.path(root).exists() {
trusted_file(&target.path(root))?;
let installed_profile = layout.bin().join("dever-sandbox");
if previous_service.is_none()
|| !installed_profile.exists()
|| read_bounded(&target.path(root), 8192)?
!= read_bounded(&installed_profile, 8192)?
{
return Err("bootstrap cannot replace an unrelated AppArmor profile".into());
}
} else if target == Target::Service && target.path(root).exists() {
trusted_file(&target.path(root))?;
if previous_service.as_deref()
!= Some(read_bounded(&target.path(root), 8192)?.as_slice())
{
return Err(
"bootstrap cannot replace an unrelated system service definition".into(),
);
}
}
}
let changes = TARGETS
.into_iter()
.map(|target| {
Ok(Change {
target,
before: identity(&target.path(root), target.link())?,
after: None,
})
})
.collect::<Result<Vec<_>, String>>()?;
let mut journal = Journal {
format: "dever-bootstrap-transaction-v1".into(),
phase: Phase::Preparing,
service_was_active: if activate_service {
Some(service::active()?)
} else {
None
},
service_start_requested: false,
changes,
};
save(layout, &journal)?;
let result = (|| {
stage(root, source, manifest, key)?;
for change in &mut journal.changes {
change.after = identity(&change.target.next(root), change.target.link())?;
if change.after.is_none() {
return Err("bootstrap staging is incomplete".into());
}
}
journal.phase = Phase::Prepared;
save(layout, &journal)?;
for index in 0..journal.changes.len() {
if journal.changes[index].target == Target::Active && activate_service {
journal.service_start_requested = true;
save(layout, &journal)?;
service::apply(true)?;
service::ready(layout)?;
}
publish(root, &journal.changes[index])?;
}
journal.phase = Phase::Committed;
save(layout, &journal)
})();
if let Err(primary) = result {
// The journal is still Prepared unless its commit was persisted. Read
// the durable phase rather than trusting the in-memory assignment.
return match recover(root, layout) {
Ok(()) => Err(primary),
Err(rollback) => Err(format!("{primary}; bootstrap recovery failed: {rollback}")),
};
}
if let Err(error) = recover(root, layout) {
// Publication is already committed. Never roll it back because an old
// staging copy could not be removed; preserve the journal for retry.
return Err(format!(
"bootstrap publication committed, but cleanup requires recovery: {error}"
));
}
Ok(())
}
fn stage(root: &Path, source: &Path, manifest: &ReleaseManifest, key: &[u8]) -> Result<(), String> {
super::sandbox::stage(
&Target::Sandbox.path(root),
&Target::Sandbox.next(root),
source,
manifest,
key,
)?;
write_new(
&Target::Apparmor.next(root),
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
0o644,
)?;
let bin = Target::Bin.next(root);
fs::create_dir(&bin).map_err(|error| error.to_string())?;
for artifact in &manifest.artifacts {
let Some(relative) = artifact.path.strip_prefix("bootstrap/") else {
continue;
};
release::copy_file(
&source.join(&artifact.path),
&bin.join(relative),
matches!(relative, "dever" | "deverd"),
)?;
}
for file in ["manifest.json", "manifest.sig"] {
release::copy_file(&source.join(file), &bin.join(file), false)?;
}
validate_bin(&bin, key)?;
for name in ["dever", "deverd"] {
if !service::run(
Command::new(bin.join(name)).arg("--dever-bootstrap-health"),
"health-check bootstrap",
)? {
return Err(format!("bootstrap {name} failed its health check"));
}
}
write_new(
&Target::Service.next(root),
super::SERVICE.as_bytes(),
0o644,
)?;
for target in [Target::Entry, Target::Enable] {
symlink(target.link().unwrap(), target.next(root)).map_err(|error| error.to_string())?;
sync_parent(&target.next(root))?;
}
let active = Target::Active.path(root);
let next = Target::Active.next(root);
if active.exists() {
release::copy_file(&active, &next, false)?;
}
release::append_active_version(&next, &manifest.version).map_err(|error| error.to_string())?;
release::sync_tree(&bin)?;
sync_parent(&bin)
}
fn publish(root: &Path, change: &Change) -> Result<(), String> {
if change.target == Target::Apparmor && root == Path::new("/") {
dever_sandbox::validate_host_policy()?;
}
let path = change.target.path(root);
let next = change.target.next(root);
if identity(&path, change.target.link())? != change.before
|| identity(&next, change.target.link())? != change.after
{
return Err("bootstrap destination changed during installation".into());
}
if change.before == change.after {
return Ok(());
}
if change.before.is_some() {
exchange(&path, &next)?;
} else {
rustix::fs::renameat_with(
rustix::fs::CWD,
&next,
rustix::fs::CWD,
&path,
rustix::fs::RenameFlags::NOREPLACE,
)
.map_err(|error| error.to_string())?;
}
sync_parent(&path)
}
pub(super) fn recover(root: &Path, layout: &Layout) -> Result<(), String> {
let temporary = layout.state().join(JOURNAL_NEXT);
if fs::symlink_metadata(&temporary).is_ok() {
trusted_file(&temporary)?;
fs::remove_file(&temporary).map_err(|error| error.to_string())?;
}
let path = layout.state().join(JOURNAL);
let bytes = match fs::symlink_metadata(&path) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error.to_string()),
Ok(_) => {
trusted_file(&path)?;
read_bounded(&path, 64 * 1024)?
}
};
let journal: Journal = serde_json::from_slice(&bytes)
.map_err(|error| format!("invalid bootstrap recovery journal: {error}"))?;
if journal.format != "dever-bootstrap-transaction-v1"
|| journal
.changes
.iter()
.map(|change| change.target)
.collect::<Vec<_>>()
!= TARGETS
|| journal.service_was_active.is_some() && root != Path::new("/")
|| journal.service_start_requested
&& (journal.service_was_active.is_none() || journal.phase == Phase::Preparing)
|| journal.changes.iter().any(|change| {
change.before.iter().chain(&change.after).any(|hash| {
hash.len() != 64
|| !hash
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
})
})
|| journal.phase != Phase::Preparing
&& journal.changes.iter().any(|change| change.after.is_none())
{
return Err("bootstrap recovery journal has unsupported targets or identities".into());
}
// Validate all affected parents before touching any target. The journal can
// never turn an image-root path into a write through a substituted symlink.
for change in &journal.changes {
trusted_directory(change.target.path(root).parent().unwrap())?;
}
if journal.phase == Phase::Committed {
for change in &journal.changes {
if identity(&change.target.path(root), change.target.link())? != change.after {
return Err("committed bootstrap destination changed before cleanup".into());
}
}
} else {
let (before_files, after_files) = if let Some(was_active) = journal.service_was_active {
service::recovery_actions(
journal.service_start_requested,
was_active,
journal.service_start_requested && service::loaded()?,
)
} else {
(vec![], vec![])
};
service::execute(&before_files)?;
for change in journal.changes.iter().rev() {
rollback(root, change, journal.phase)?;
}
service::execute(&after_files)?;
}
for change in &journal.changes {
remove_owned(&change.target.next(root), change.target.link())?;
}
fs::remove_file(&path).map_err(|error| error.to_string())?;
sync_parent(&path)
}
fn rollback(root: &Path, change: &Change, phase: Phase) -> Result<(), String> {
let path = change.target.path(root);
let next = change.target.next(root);
let current = identity(&path, change.target.link())?;
if current == change.before {
return Ok(());
}
if phase != Phase::Prepared || current != change.after {
return Err("bootstrap recovery refuses a destination outside this transaction".into());
}
if let Some(before) = &change.before {
if identity(&next, change.target.link())?.as_ref() != Some(before) {
return Err("bootstrap rollback copy is missing or changed".into());
}
exchange(&path, &next)?;
} else {
remove_owned(&path, change.target.link())?;
}
sync_parent(&path)
}
fn save(layout: &Layout, journal: &Journal) -> Result<(), String> {
let next = layout.state().join(JOURNAL_NEXT);
write_new(
&next,
&serde_json::to_vec(journal).map_err(|error| error.to_string())?,
0o600,
)?;
fs::rename(&next, layout.state().join(JOURNAL)).map_err(|error| error.to_string())?;
release::sync_directory(&layout.state()).map_err(|error| error.to_string())
}
fn exchange(path: &Path, next: &Path) -> Result<(), String> {
rustix::fs::renameat_with(
rustix::fs::CWD,
path,
rustix::fs::CWD,
next,
rustix::fs::RenameFlags::EXCHANGE,
)
.map_err(|error| format!("atomic bootstrap exchange failed: {error}"))
}
fn sync_parent(path: &Path) -> Result<(), String> {
release::sync_directory(path.parent().ok_or("bootstrap destination has no parent")?)
.map_err(|error| error.to_string())
}
fn remove_owned(path: &Path, link: Option<&str>) -> Result<(), String> {
let Some(_) = identity(path, link)? else {
return Ok(());
};
if fs::symlink_metadata(path)
.map_err(|error| error.to_string())?
.is_dir()
{
fs::remove_dir_all(path).map_err(|error| error.to_string())?;
} else {
fs::remove_file(path).map_err(|error| error.to_string())?;
}
sync_parent(path)
}
fn identity(path: &Path, link: Option<&str>) -> Result<Option<String>, String> {
let metadata = match fs::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(error.to_string()),
};
if let Some(link) = link {
if !metadata.file_type().is_symlink()
|| metadata.uid() != 0
|| fs::read_link(path).map_err(|error| error.to_string())? != Path::new(link)
{
return Err("bootstrap link has an unsupported owner or target".into());
}
return Ok(Some(digest(format!("link\0{link}").as_bytes())));
}
let mut tree = Tree::default();
tree.visit(path, path, 0)?;
Ok(Some(format!("{:x}", tree.hash.finalize())))
}
pub(super) fn files(root: &Path) -> Result<Vec<String>, String> {
let mut tree = Tree::default();
tree.visit(root, root, 0)?;
Ok(tree.files.into_iter().collect())
}
#[derive(Default)]
struct Tree {
hash: Sha256,
bytes: u64,
entries: usize,
files: BTreeSet<String>,
}
impl Tree {
fn visit(&mut self, root: &Path, path: &Path, depth: usize) -> Result<(), String> {
self.entries += 1;
if self.entries > 128 || depth > 4 {
return Err("bootstrap tree exceeds its entry/depth bound".into());
}
let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?;
let name = path
.strip_prefix(root)
.unwrap()
.to_str()
.ok_or("bootstrap path is not UTF-8")?;
self.hash.update((name.len() as u64).to_le_bytes());
self.hash.update(name.as_bytes());
self.hash.update((metadata.mode() & 0o777).to_le_bytes());
if metadata.is_dir() && !metadata.file_type().is_symlink() {
trusted_directory(path)?;
self.hash.update(b"directory");
let mut entries = fs::read_dir(path)
.map_err(|error| error.to_string())?
.map(|entry| entry.map(|entry| entry.path()))
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?;
entries.sort();
for entry in entries {
self.visit(root, &entry, depth + 1)?;
}
} else {
trusted_file(path)?;
self.bytes = self
.bytes
.checked_add(metadata.len())
.ok_or("bootstrap tree size overflow")?;
if self.bytes > MAX_TREE_BYTES {
return Err("bootstrap tree exceeds its byte bound".into());
}
let bytes = read_bounded(path, metadata.len())?;
self.hash.update(b"file");
self.hash.update((bytes.len() as u64).to_le_bytes());
self.hash.update(bytes);
self.files.insert(name.into());
}
Ok(())
}
}
+778
View File
@@ -0,0 +1,778 @@
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::release::{Layout, Version, hex};
static NEXT_CACHE_OPERATION: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Debug)]
pub struct BuildIdentity<'a> {
pub version: &'a Version,
pub compiler: &'a [u8],
pub runtime: &'a [u8],
pub target: &'a str,
pub generated: &'a [u8],
pub options: &'a [u8],
pub environment: &'a [u8],
}
impl BuildIdentity<'_> {
pub fn key(&self) -> String {
let mut digest = Sha256::new();
digest.update(b"dever-machine-cache-v1\0");
for value in [
self.version.as_str().as_bytes(),
self.compiler,
self.runtime,
self.target.as_bytes(),
self.generated,
self.options,
self.environment,
] {
digest.update((value.len() as u64).to_le_bytes());
digest.update(value);
}
hex(&digest.finalize())
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CacheEntry {
pub format: String,
pub key: String,
pub version: Version,
pub target: String,
pub bytes: u64,
pub sha256: String,
}
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
pub struct CacheStatus {
#[serde(default)]
pub verified: bool,
pub entries: u64,
pub bytes: u64,
pub versions: BTreeMap<String, u64>,
pub corrupt_entries: u64,
pub staging_entries: u64,
#[serde(default)]
pub artifact_entries: u64,
#[serde(default)]
pub artifact_bytes: u64,
#[serde(default)]
pub corrupt_artifacts: u64,
}
/// The service accepts content identities, never caller-selected cache paths.
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ArtifactReceipt {
pub sha256: String,
pub bytes: u64,
}
pub const MAX_ARTIFACT_BYTES: u64 = 64 * 1024 * 1024;
pub(super) const MAX_COMPILED_BYTES: u64 = 320 * 1024 * 1024;
const MAX_ARTIFACT_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024;
const MAX_ARTIFACT_ENTRIES: usize = 4096;
impl ArtifactReceipt {
pub fn validate(&self) -> Result<(), String> {
validate_key(&self.sha256)?;
if self.bytes > MAX_ARTIFACT_BYTES {
return Err("shared artifact exceeds the 64 MiB transfer limit".into());
}
Ok(())
}
pub(super) fn validate_compiled(&self) -> Result<(), String> {
validate_key(&self.sha256)?;
if self.bytes == 0 || self.bytes > MAX_COMPILED_BYTES {
return Err("compiled output must be between 1 byte and 320 MiB".into());
}
Ok(())
}
}
pub struct StoredArtifact {
pub receipt: ArtifactReceipt,
pub file: File,
_operation: CacheOperation,
}
/// Filesystem engine owned by the privileged cache process. It deliberately has
/// no transport or caller-authentication semantics; project processes must use
/// the authenticated `deverd` IPC client rather than constructing this store directly.
pub struct CacheStore {
root: PathBuf,
}
impl CacheStore {
pub fn new(layout: &Layout) -> Self {
Self {
root: layout.native_cache(),
}
}
/// Hold across input staging, compilation, publication and response transfer.
pub(crate) fn operation(&self) -> Result<CacheOperation, String> {
CacheOperation::acquire(&self.root)
}
pub(crate) fn compilation_directory(&self) -> Result<CompilationDirectory, String> {
let root = self.root.join("staging");
create_directory(&root)?;
let path = root.join(format!(
"compile-{}-{}",
std::process::id(),
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed)
));
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder
.create(&path)
.map_err(|error| format!("cannot stage compilation: {error}"))?;
Ok(CompilationDirectory(path))
}
/// Hash while receiving into an owned staging file. Only verified complete
/// bytes are published, and concurrent identical uploads share one entry.
pub fn publish_artifact(
&self,
expected: &ArtifactReceipt,
input: &mut impl Read,
) -> Result<ArtifactReceipt, String> {
expected.validate()?;
let _operation = CacheOperation::acquire(&self.root)?;
let staging_root = self.root.join("staging");
create_directory(&staging_root)?;
let staging = staging_root.join(format!(
"artifact-{}-{}",
std::process::id(),
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed)
));
let result = (|| {
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options
.open(&staging)
.map_err(|error| format!("cannot stage shared artifact: {error}"))?;
let mut digest = Sha256::new();
let mut remaining = expected.bytes;
let mut buffer = [0_u8; 64 * 1024];
while remaining > 0 {
let limit = remaining.min(buffer.len() as u64) as usize;
input
.read_exact(&mut buffer[..limit])
.map_err(|error| format!("incomplete shared artifact upload: {error}"))?;
file.write_all(&buffer[..limit])
.map_err(|error| format!("cannot stage shared artifact: {error}"))?;
digest.update(&buffer[..limit]);
remaining -= limit as u64;
}
if hex(&digest.finalize()) != expected.sha256 {
return Err("shared artifact upload failed SHA-256 verification".into());
}
file.sync_all()
.map_err(|error| format!("cannot sync shared artifact: {error}"))?;
drop(file);
let artifacts = self.root.join("artifacts");
create_directory(&artifacts)?;
let _publication = ArtifactPublication::acquire(&self.root)?;
let destination = artifacts.join(&expected.sha256);
match fs::symlink_metadata(&destination) {
Ok(_) => {
self.read_artifact_file(expected)?;
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {
let (used, count) = directory_entries(&artifacts)?.try_fold(
(0_u64, 0_usize),
|(sum, count), entry| {
let metadata = fs::symlink_metadata(entry.path())
.map_err(|error| error.to_string())?;
if !metadata.is_file() || metadata.file_type().is_symlink() {
return Err(
"shared artifact cache contains a non-regular file".into()
);
}
Ok::<_, String>((sum.saturating_add(metadata.len()), count + 1))
},
)?;
if count >= MAX_ARTIFACT_ENTRIES
|| used.saturating_add(expected.bytes) > MAX_ARTIFACT_CACHE_BYTES
{
return Err("shared artifact cache is full; administrator must run 'dever cache clean'".into());
}
fs::rename(&staging, &destination)
.map_err(|error| format!("cannot publish shared artifact: {error}"))?;
sync_directory(&artifacts)
.map_err(|error| format!("cannot sync shared artifacts: {error}"))?;
}
Err(error) => return Err(format!("cannot inspect shared artifact: {error}")),
}
Ok(expected.clone())
})();
if staging.exists() {
let _ = fs::remove_file(&staging);
}
result
}
pub fn artifact(&self, expected: &ArtifactReceipt) -> Result<StoredArtifact, String> {
expected.validate()?;
let operation = CacheOperation::acquire(&self.root)?;
let file = self.read_artifact_file(expected)?;
Ok(StoredArtifact {
receipt: expected.clone(),
file,
_operation: operation,
})
}
fn read_artifact_file(&self, expected: &ArtifactReceipt) -> Result<File, String> {
let path = self.artifact_path(expected)?;
if sha256_file(&path)? != expected.sha256 {
return Err("shared artifact failed SHA-256 verification".into());
}
File::open(&path).map_err(|error| format!("cannot read shared artifact: {error}"))
}
fn artifact_path(&self, expected: &ArtifactReceipt) -> Result<PathBuf, String> {
expected.validate()?;
let directory = self.root.join("artifacts");
ensure_real_directory(&directory)
.map_err(|error| format!("shared artifact unavailable: {error}"))?;
let path = directory.join(&expected.sha256);
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("shared artifact unavailable: {error}"))?;
if !metadata.is_file()
|| metadata.file_type().is_symlink()
|| metadata.len() != expected.bytes
{
return Err("shared artifact is incomplete or modified".into());
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let owner = fs::symlink_metadata(&self.root)
.map_err(|error| error.to_string())?
.uid();
if metadata.uid() != owner || metadata.mode() & 0o077 != 0 {
return Err("shared artifact must be private to the service owner".into());
}
}
Ok(path)
}
pub fn publish(
&self,
identity: &BuildIdentity<'_>,
executable: &Path,
) -> Result<String, String> {
let _operation = CacheOperation::acquire(&self.root)?;
let _publication = ArtifactPublication::acquire(&self.root)?;
let key = identity.key();
let destination = self.root.join("entries").join(&key);
if destination.exists() {
self.read_entry(&key)?;
return Ok(key);
}
let staging_root = self.root.join("staging");
create_directory(&staging_root)?;
create_directory(&self.root.join("entries"))?;
let bytes = fs::symlink_metadata(executable)
.map_err(|error| format!("cannot inspect compiled output: {error}"))?
.len();
if bytes == 0 || bytes > MAX_COMPILED_BYTES {
return Err("compiled output must be between 1 byte and 320 MiB".into());
}
let (used, count) = directory_entries(&self.root.join("entries"))?.try_fold(
(0_u64, 0_usize),
|(used, count), entry| {
let key = entry.file_name().to_string_lossy().into_owned();
let manifest = self.inspect_entry(&key, false)?;
Ok::<_, String>((used.saturating_add(manifest.bytes), count + 1))
},
)?;
if count >= MAX_ARTIFACT_ENTRIES || used.saturating_add(bytes) > MAX_ARTIFACT_CACHE_BYTES {
return Err("shared compilation cache quota exceeded (2 GiB / 4096 entries)".into());
}
let staging = staging_root.join(format!(
"publish-{}-{}",
std::process::id(),
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
));
create_directory(&staging)?;
let result = (|| {
let program = staging.join("program");
copy_regular_file(executable, &program)?;
let metadata = fs::metadata(&program).map_err(|error| error.to_string())?;
let entry = CacheEntry {
format: "dever-machine-cache-entry-v1".into(),
key: key.clone(),
version: identity.version.clone(),
target: identity.target.to_owned(),
bytes: metadata.len(),
sha256: sha256_file(&program)?,
};
let manifest = serde_json::to_vec(&entry).map_err(|error| error.to_string())?;
write_new(&staging.join("entry.json"), &manifest)?;
sync_directory(&staging).map_err(|error| error.to_string())?;
match fs::rename(&staging, &destination) {
Ok(()) => {
sync_directory(&self.root.join("entries")).map_err(|error| error.to_string())
}
Err(_error) if destination.exists() => {
self.read_entry(&key)?;
Ok(())
}
Err(error) => Err(format!("cannot publish shared cache entry: {error}")),
}
})();
if staging.exists() {
let _ = fs::remove_dir_all(&staging);
}
result.map(|()| key)
}
pub fn restore(&self, identity: &BuildIdentity<'_>) -> Result<Option<CachedArtifact>, String> {
let key = identity.key();
let entry = match self.read_entry(&key) {
Ok(entry) => entry,
Err(error) if error.starts_with("shared cache miss:") => return Ok(None),
Err(error) => return Err(error),
};
if entry.version != *identity.version || entry.target != identity.target {
return Err("shared cache identity collision".into());
}
let lease = CacheLease::acquire(&self.root, &key)?;
let path = self.root.join("entries").join(&key).join("program");
let bytes = read_regular_file(&path)?;
if bytes.len() as u64 != entry.bytes || sha256(&bytes) != entry.sha256 {
return Err(format!(
"shared cache entry {key} is incomplete or modified"
));
}
Ok(Some(CachedArtifact {
bytes,
_lease: lease,
}))
}
pub fn status(&self) -> Result<CacheStatus, String> {
self.inspect_status(true)
}
/// Unprivileged status is metadata-only: it cannot force hashing all
/// libraries/programs. It never establishes an artifact integrity claim.
pub fn summary(&self) -> Result<CacheStatus, String> {
self.inspect_status(false)
}
fn inspect_status(&self, verify_bytes: bool) -> Result<CacheStatus, String> {
let mut status = CacheStatus {
verified: verify_bytes,
..CacheStatus::default()
};
for entry in directory_entries(&self.root.join("staging"))? {
let _ = entry;
status.staging_entries += 1;
}
for entry in directory_entries(&self.root.join("entries"))? {
let key = entry
.file_name()
.into_string()
.map_err(|_| "shared cache key is not UTF-8".to_owned())?;
match self.inspect_entry(&key, verify_bytes) {
Ok(manifest) => {
status.entries += 1;
status.bytes = status.bytes.saturating_add(manifest.bytes);
*status
.versions
.entry(manifest.version.to_string())
.or_default() += 1;
}
Err(_) => status.corrupt_entries += 1,
}
}
for entry in directory_entries(&self.root.join("artifacts"))? {
let metadata = fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?;
let expected = ArtifactReceipt {
sha256: entry.file_name().to_string_lossy().into_owned(),
bytes: metadata.len(),
};
let inspected = if verify_bytes {
self.read_artifact_file(&expected).map(|_| ())
} else {
self.artifact_path(&expected).map(|_| ())
};
match inspected {
Ok(_) => {
status.artifact_entries += 1;
status.artifact_bytes = status.artifact_bytes.saturating_add(expected.bytes);
}
Err(_) => status.corrupt_artifacts += 1,
}
}
Ok(status)
}
pub fn clean(&self, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> {
let _lock = CacheCleanLock::acquire(&self.root)?;
if directory_entries(&self.root.join("operations"))?
.next()
.is_some()
{
return Err("shared cache is busy; clean did not remove active build state".into());
}
remove_children(&self.root.join("staging"))?;
// Artifacts are a recoverable download cache. Explicit administrator
// cleanup removes them only while no upload or reader is active.
remove_children(&self.root.join("artifacts"))?;
for entry in directory_entries(&self.root.join("entries"))? {
let key = entry
.file_name()
.into_string()
.map_err(|_| "shared cache key is not UTF-8".to_owned())?;
if self.has_lease(&key)? {
continue;
}
let remove = self
.read_entry(&key)
.map(|entry| !installed.contains(&entry.version))
.unwrap_or(true);
if remove {
remove_entry(entry.path())?;
}
}
self.status()
}
fn read_entry(&self, key: &str) -> Result<CacheEntry, String> {
self.inspect_entry(key, true)
}
fn inspect_entry(&self, key: &str, verify_bytes: bool) -> Result<CacheEntry, String> {
validate_key(key)?;
let root = self.root.join("entries").join(key);
if !root.exists() {
return Err(format!("shared cache miss: {key}"));
}
ensure_real_directory(&root)?;
let manifest: CacheEntry =
serde_json::from_slice(&read_regular_file(&root.join("entry.json"))?)
.map_err(|error| format!("invalid shared cache manifest {key}: {error}"))?;
if manifest.format != "dever-machine-cache-entry-v1"
|| manifest.key != key
|| manifest.bytes == 0
|| manifest.bytes > MAX_COMPILED_BYTES
{
return Err(format!("invalid shared cache manifest {key}"));
}
let program = root.join("program");
let metadata = fs::symlink_metadata(&program)
.map_err(|error| format!("cannot inspect shared cache entry {key}: {error}"))?;
if !metadata.file_type().is_file()
|| metadata.len() != manifest.bytes
|| (verify_bytes && sha256_file(&program)? != manifest.sha256)
{
return Err(format!(
"shared cache entry {key} is incomplete or modified"
));
}
Ok(manifest)
}
fn has_lease(&self, key: &str) -> Result<bool, String> {
let directory = self.root.join("leases").join(key);
Ok(directory_entries(&directory)?.next().is_some())
}
}
pub struct CachedArtifact {
bytes: Vec<u8>,
_lease: CacheLease,
}
impl CachedArtifact {
pub fn as_bytes(&self) -> &[u8] {
&self.bytes
}
}
struct CacheLease {
path: PathBuf,
}
impl CacheLease {
fn acquire(root: &Path, key: &str) -> Result<Self, String> {
if root.join("clean.lock").exists() {
return Err("shared cache maintenance is active".into());
}
let directory = root.join("leases").join(key);
create_directory(&directory)?;
let path = directory.join(format!(
"{}-{}",
std::process::id(),
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
));
write_new(&path, b"dever-cache-lease-v1\n")?;
if root.join("clean.lock").exists() {
let _ = fs::remove_file(&path);
return Err("shared cache maintenance is active".into());
}
Ok(Self { path })
}
}
impl Drop for CacheLease {
fn drop(&mut self) {
let parent = self.path.parent().map(Path::to_path_buf);
let _ = fs::remove_file(&self.path);
if let Some(parent) = parent {
let _ = fs::remove_dir(parent);
}
}
}
struct CacheCleanLock(PathBuf);
impl CacheCleanLock {
fn acquire(root: &Path) -> Result<Self, String> {
create_directory(root)?;
let path = root.join("clean.lock");
write_new(&path, format!("{}\n", std::process::id()).as_bytes())?;
Ok(Self(path))
}
}
pub(crate) struct CacheOperation(PathBuf);
pub(crate) struct CompilationDirectory(pub(crate) PathBuf);
impl Drop for CompilationDirectory {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
impl CacheOperation {
fn acquire(root: &Path) -> Result<Self, String> {
create_directory(root)?;
if root.join("clean.lock").exists() {
return Err("shared cache maintenance is active".into());
}
let directory = root.join("operations");
create_directory(&directory)?;
let path = directory.join(format!(
"{}-{}",
std::process::id(),
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
));
write_new(&path, b"dever-cache-operation-v1\n")?;
if root.join("clean.lock").exists() {
let _ = fs::remove_file(&path);
return Err("shared cache maintenance is active".into());
}
Ok(Self(path))
}
}
impl Drop for CacheOperation {
fn drop(&mut self) {
let _ = fs::remove_file(&self.0);
}
}
impl Drop for CacheCleanLock {
fn drop(&mut self) {
let _ = fs::remove_file(&self.0);
}
}
/// Serializes quota accounting and publication, not network reception.
struct ArtifactPublication(File);
impl ArtifactPublication {
fn acquire(root: &Path) -> Result<Self, String> {
let path = root.join("artifact-publish.lock");
let mut options = OpenOptions::new();
options.read(true).write(true);
match fs::symlink_metadata(&path) {
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {}
Ok(_) => return Err("artifact publication lock must be a regular file".into()),
Err(error) if error.kind() == io::ErrorKind::NotFound => {
options.create_new(true);
}
Err(error) => return Err(format!("cannot inspect artifact publication lock: {error}")),
}
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let file = match options.open(&path) {
Ok(file) => file,
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.map_err(|error| format!("cannot open artifact publication lock: {error}"))?,
Err(error) => return Err(format!("cannot open artifact publication lock: {error}")),
};
fs2::FileExt::lock_exclusive(&file)
.map_err(|error| format!("cannot lock artifact publication: {error}"))?;
Ok(Self(file))
}
}
impl Drop for ArtifactPublication {
fn drop(&mut self) {
let _ = fs2::FileExt::unlock(&self.0);
}
}
fn validate_key(key: &str) -> Result<(), String> {
if key.len() != 64
|| !key
.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err("invalid shared cache key".into());
}
Ok(())
}
fn directory_entries(path: &Path) -> Result<impl Iterator<Item = fs::DirEntry>, String> {
let entries = match fs::read_dir(path) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new().into_iter()),
Err(error) => return Err(format!("cannot inspect '{}': {error}", path.display())),
};
let entries = entries
.collect::<Result<Vec<_>, _>>()
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
Ok(entries.into_iter())
}
fn remove_children(path: &Path) -> Result<(), String> {
for entry in directory_entries(path)? {
remove_entry(entry.path())?;
}
Ok(())
}
fn remove_entry(path: PathBuf) -> Result<(), String> {
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
if metadata.is_dir() && !metadata.file_type().is_symlink() {
fs::remove_dir_all(&path)
} else {
fs::remove_file(&path)
}
.map_err(|error| format!("cannot remove '{}': {error}", path.display()))
}
fn create_directory(path: &Path) -> Result<(), String> {
let mut builder = fs::DirBuilder::new();
builder.recursive(true);
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder
.create(path)
.map_err(|error| format!("cannot create '{}': {error}", path.display()))?;
ensure_real_directory(path)
}
fn ensure_real_directory(path: &Path) -> Result<(), String> {
let metadata = fs::symlink_metadata(path)
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err(format!("'{}' must be a real directory", path.display()));
}
Ok(())
}
fn copy_regular_file(source: &Path, destination: &Path) -> Result<(), String> {
let metadata = fs::symlink_metadata(source)
.map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?;
if !metadata.file_type().is_file() {
return Err(format!("'{}' must be a regular file", source.display()));
}
fs::copy(source, destination)
.map_err(|error| format!("cannot copy '{}': {error}", source.display()))?;
fs::set_permissions(destination, metadata.permissions())
.map_err(|error| format!("cannot preserve '{}': {error}", destination.display()))
}
fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> {
let metadata = fs::symlink_metadata(path)
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
if !metadata.file_type().is_file() {
return Err(format!("'{}' must be a regular file", path.display()));
}
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
}
fn write_new(path: &Path, value: &[u8]) -> Result<(), String> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|error| format!("cannot create '{}': {error}", path.display()))?;
file.write_all(value)
.and_then(|()| file.sync_all())
.map_err(|error| format!("cannot write '{}': {error}", path.display()))
}
fn sha256_file(path: &Path) -> Result<String, String> {
let mut file =
File::open(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))?;
let mut digest = Sha256::new();
let mut buffer = [0_u8; 64 * 1024];
loop {
let length = file
.read(&mut buffer)
.map_err(|error| format!("cannot hash '{}': {error}", path.display()))?;
if length == 0 {
break;
}
digest.update(&buffer[..length]);
}
Ok(hex(&digest.finalize()))
}
fn sha256(value: &[u8]) -> String {
hex(&Sha256::digest(value))
}
fn sync_directory(path: &Path) -> io::Result<()> {
#[cfg(unix)]
{
File::open(path)?.sync_all()
}
#[cfg(windows)]
{
let _ = path;
Ok(())
}
}
@@ -0,0 +1,326 @@
//! Bounded source-only protocol shared by callers, the service and its worker.
use super::BuildTarget;
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use dever_core::native::EmbeddedResource;
use dever_core::source::SourceMap;
use dever_runtime::config::CompilationBindings;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
// Base64 plus source/metadata must fit while embedding a complete interpreter.
pub const MAX_COMPILE_BYTES: usize = 384 * 1024 * 1024;
const MAX_RESOURCE_BYTES: usize = 128 * 1024 * 1024;
const MAX_RESOURCE_TOTAL: usize = 256 * 1024 * 1024;
pub const WORKER_REQUEST_FILE: &str = "request.json";
pub const WORKER_OUTPUT_FILE: &str = "program";
#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum CompileKind {
Application,
TestSuite,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CompileRequest {
pub version: String,
pub kind: CompileKind,
pub target: BuildTarget,
pub sources: Vec<CompilationSource>,
pub bindings: Option<CompilationBindings>,
pub resources: Vec<CompilationResource>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CompilationSource {
pub logical_path: String,
pub is_test: bool,
pub text: String,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct CompilationResource {
pub path: String,
#[serde(with = "resource_bytes")]
pub bytes: Vec<u8>,
pub sha256: String,
pub executable: bool,
}
impl CompileRequest {
pub fn new(
version: String,
kind: CompileKind,
sources: &SourceMap,
bindings: Option<CompilationBindings>,
resources: &[EmbeddedResource],
target: BuildTarget,
) -> Result<Self, String> {
let request = Self {
version,
kind,
target,
sources: sources
.compilation_sources()?
.into_iter()
.map(|source| CompilationSource {
logical_path: source.logical_path,
is_test: source.is_test,
text: source.text,
})
.collect(),
bindings,
resources: resources
.iter()
.map(|resource| CompilationResource {
path: resource.path.clone(),
bytes: resource.bytes.clone(),
sha256: resource.sha256.clone(),
executable: resource.executable,
})
.collect(),
};
request.validate()?;
Ok(request)
}
pub fn validate(&self) -> Result<(), String> {
super::Version::parse(&self.version)?;
if self.kind == CompileKind::TestSuite && self.target != BuildTarget::host()? {
return Err("test compilation requires the compiler host target".into());
}
if self.sources.is_empty()
|| self.sources.len() > 4096
|| self
.sources
.iter()
.map(|source| source.text.len())
.sum::<usize>()
> 16 * 1024 * 1024
|| self
.sources
.iter()
.any(|source| source.logical_path.len() > 4096)
|| !self.sources.iter().any(|source| !source.is_test)
{
return Err("compilation requires application sources, at most 4096 files and 16 MiB of source text".into());
}
self.source_map()?;
match self.kind {
CompileKind::Application => {
if self.sources.iter().any(|source| source.is_test) {
return Err("application compilation cannot contain test sources".into());
}
self.bindings
.as_ref()
.ok_or("application compilation requires deployment bindings")?
.validate()?;
}
CompileKind::TestSuite if self.bindings.is_some() || !self.resources.is_empty() => {
return Err("test compilation derives case-local bindings and cannot contain deployment bindings or external resources".into());
}
CompileKind::TestSuite => {}
}
if self.resources.len() > 4096
|| self
.resources
.iter()
.map(|resource| resource.bytes.len())
.sum::<usize>()
> MAX_RESOURCE_TOTAL
{
return Err("compilation resources exceed 4096 files or 256 MiB".into());
}
let mut paths = BTreeSet::new();
for resource in &self.resources {
relative_path(&resource.path)?;
if !paths.insert(&resource.path) {
return Err("duplicate compilation resource path".into());
}
if resource.bytes.len() > MAX_RESOURCE_BYTES
|| format!("{:x}", Sha256::digest(&resource.bytes)) != resource.sha256
{
return Err(format!(
"compilation resource '{}' exceeds 128 MiB or has an invalid digest",
resource.path
));
}
}
Ok(())
}
pub fn source_map(&self) -> Result<SourceMap, String> {
SourceMap::from_compilation_sources(
&self
.sources
.iter()
.map(|source| dever_core::source::CompilationSource {
logical_path: source.logical_path.clone(),
is_test: source.is_test,
text: source.text.clone(),
})
.collect::<Vec<_>>(),
)
}
pub fn embedded_resources(&self) -> Vec<EmbeddedResource> {
self.resources
.iter()
.map(|resource| EmbeddedResource {
path: resource.path.clone(),
bytes: resource.bytes.clone(),
sha256: resource.sha256.clone(),
executable: resource.executable,
})
.collect()
}
pub fn encode(&self) -> Result<Vec<u8>, String> {
self.validate()?;
#[derive(Serialize)]
struct Canonical<'a> {
version: &'a str,
kind: CompileKind,
target: BuildTarget,
sources: Vec<&'a CompilationSource>,
bindings: &'a Option<CompilationBindings>,
resources: Vec<&'a CompilationResource>,
}
// Sort references: interpreter payloads must not be cloned to order JSON.
let mut sources = self.sources.iter().collect::<Vec<_>>();
sources.sort_by(|left, right| {
(left.is_test, &left.logical_path).cmp(&(right.is_test, &right.logical_path))
});
let mut resources = self.resources.iter().collect::<Vec<_>>();
resources.sort_by(|left, right| left.path.cmp(&right.path));
let canonical = Canonical {
version: &self.version,
kind: self.kind,
target: self.target,
sources,
bindings: &self.bindings,
resources,
};
let bytes = serde_json::to_vec(&canonical)
.map_err(|error| format!("cannot encode compile request: {error}"))?;
if bytes.len() > MAX_COMPILE_BYTES {
return Err("compile request exceeds 384 MiB".into());
}
Ok(bytes)
}
pub fn decode(bytes: &[u8]) -> Result<Self, String> {
if bytes.len() > MAX_COMPILE_BYTES {
return Err("compile request exceeds 384 MiB".into());
}
let request: Self = serde_json::from_slice(bytes)
.map_err(|error| format!("invalid compile request: {error}"))?;
// Requiring one representation rejects duplicate map keys as well as
// ambiguous ordering/escapes before requests acquire a cache identity.
if request.encode()? != bytes {
return Err("compile request must use canonical JSON without duplicate keys".into());
}
Ok(request)
}
}
fn relative_path(path: &str) -> Result<(), String> {
if path.is_empty()
|| path.len() > 4096
|| path.contains(['\\', ':', '\0'])
|| path
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
{
return Err("compilation resources require normalized relative paths".into());
}
Ok(())
}
/// Workers accept inputs only inside the service owner's private compile stage.
/// This prevents the hidden entry from becoming a client-side managed fallback.
pub fn worker_directory(layout: &super::Layout) -> Result<std::path::PathBuf, String> {
#[cfg(unix)]
{
use std::fs;
use std::os::unix::fs::MetadataExt;
let directory = std::env::current_dir()
.map_err(|error| format!("cannot locate compiler worker directory: {error}"))?;
let staging = layout.native_cache().join("staging");
if directory.parent() != Some(staging.as_path())
|| !directory
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| {
name.strip_prefix("compile-")
.is_some_and(|suffix| !suffix.is_empty())
})
{
return Err(
"compile worker requires a service-owned native cache staging directory".into(),
);
}
let owner = rustix::process::geteuid().as_raw();
for (path, private) in [
(layout.state(), false),
(layout.cache(), false),
(layout.native_cache(), false),
(staging, true),
(directory.clone(), true),
] {
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("cannot inspect compiler worker directory: {error}"))?;
if metadata.file_type().is_symlink()
|| !metadata.is_dir()
|| metadata.uid() != owner
|| metadata.mode() & 0o022 != 0
|| private && metadata.mode() & 0o777 != 0o700
{
return Err("compiler worker directories must be real and service-owned without group/other writes; staging must be private (0700)".into());
}
}
let request = fs::symlink_metadata(directory.join(WORKER_REQUEST_FILE))
.map_err(|error| format!("cannot inspect compiler worker request: {error}"))?;
if request.file_type().is_symlink()
|| !request.is_file()
|| request.uid() != owner
|| request.mode() & 0o777 != 0o600
{
return Err(
"compiler worker request must be a service-owned regular file with mode 0600"
.into(),
);
}
Ok(directory)
}
#[cfg(not(unix))]
{
let _ = layout;
Err("trusted compiler worker directories are unavailable on this platform".into())
}
}
mod resource_bytes {
use super::*;
pub fn serialize<S: serde::Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {
serializer.serialize_str(&STANDARD.encode(bytes))
}
pub fn deserialize<'de, D: serde::Deserializer<'de>>(
deserializer: D,
) -> Result<Vec<u8>, D::Error> {
let encoded = String::deserialize(deserializer)?;
if encoded.len() > MAX_RESOURCE_BYTES * 4 / 3 + 4 {
return Err(serde::de::Error::custom(
"compilation resource exceeds 128 MiB",
));
}
STANDARD.decode(encoded).map_err(serde::de::Error::custom)
}
}
@@ -0,0 +1,212 @@
//! Native compiler loader closure, shared by authoring and signed installation.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use goblin::elf::{Elf, dynamic, header};
use super::{Artifact, runtime_pack};
const MAX_FILES: usize = 64;
const MAX_BYTES: u64 = 512 * 1024 * 1024;
pub(super) fn library_name(name: &str) -> Result<(), String> {
if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0', ':']) {
return Err("compiler dependency must use a plain library name".into());
}
Ok(())
}
fn os_library(name: &str) -> bool {
// These are the declared GNU OS ABI, not host-discovered dependencies.
matches!(
name,
"libc.so.6"
| "libm.so.6"
| "libdl.so.2"
| "libpthread.so.0"
| "librt.so.1"
| "libresolv.so.2"
| "libutil.so.1"
) || name == loader_name()
}
fn loader_name() -> &'static str {
if cfg!(target_arch = "aarch64") {
"ld-linux-aarch64.so.1"
} else {
"ld-linux-x86-64.so.2"
}
}
pub(super) fn validate(root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
validate_entries(root, artifacts, &["dever-core"])
}
/// Both release components keep their complete non-OS loader closure in lib/.
/// Their entry sets differ; the loader and file admission rules do not.
pub(super) fn validate_entries(
root: &Path,
artifacts: &[Artifact],
entries: &[&str],
) -> Result<(), String> {
runtime_pack::host_target()?;
dever_sandbox::require_no_loader_preload()?;
let mut files = BTreeMap::new();
let mut total = 0u64;
for artifact in artifacts {
if !entries.contains(&artifact.path.as_str()) && !artifact.path.starts_with("lib/") {
continue;
}
if let Some(name) = artifact.path.strip_prefix("lib/") {
library_name(name)?;
if os_library(name) {
return Err("compiler pack must not override the GNU OS ABI libraries".into());
}
}
total = total
.checked_add(artifact.bytes)
.ok_or("compiler closure size overflow")?;
if total > MAX_BYTES || files.len() >= MAX_FILES {
return Err("compiler loader closure exceeds its file or byte budget".into());
}
if files.insert(artifact.path.as_str(), artifact).is_some() {
return Err("compiler loader closure has duplicate files".into());
}
}
validate_library_directory(root, &files)?;
let mut remaining = entries
.iter()
.map(|entry| (*entry).to_owned())
.collect::<Vec<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = remaining.pop() {
if !reached.insert(path.clone()) {
continue;
}
let artifact = files.get(path.as_str()).ok_or_else(|| {
format!("compiler dependency '{path}' is not supplied by the signed release")
})?;
let input = runtime_pack::checked_path(root, &path, true)?;
let mut bytes = Vec::new();
File::open(input)
.and_then(|file| file.take(artifact.bytes + 1).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read compiler loader input: {error}"))?;
if bytes.len() as u64 != artifact.bytes {
return Err("compiler loader input size differs from its signed declaration".into());
}
remaining.extend(dependencies(
&path,
&bytes,
entries.contains(&path.as_str()),
)?);
}
if reached.len() != files.len() {
return Err("compiler closure contains libraries unused by the core".into());
}
Ok(())
}
fn dependencies(path: &str, bytes: &[u8], executable: bool) -> Result<Vec<String>, String> {
let elf = Elf::parse(bytes).map_err(|error| format!("invalid compiler ELF input: {error}"))?;
let machine = if cfg!(target_arch = "aarch64") {
header::EM_AARCH64
} else {
header::EM_X86_64
};
if !elf.is_64
|| !elf.little_endian
|| elf.header.e_machine != machine
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
{
return Err("compiler ELF input has an incompatible target or object type".into());
}
// GNU auditing/config and ELF filter tags can load libraries outside
// DT_NEEDED; they are not part of this release's loader contract.
if elf.dynamic.as_ref().is_some_and(|table| {
table.dyns.iter().any(|entry| {
matches!(
entry.d_tag,
dynamic::DT_CONFIG
| dynamic::DT_AUDIT
| dynamic::DT_DEPAUDIT
| 0x7fff_fffd
| 0x7fff_ffff
)
})
}) {
return Err("compiler ELF input contains unsupported dynamic loader hooks".into());
}
if executable {
let interpreter = if cfg!(target_arch = "aarch64") {
"/lib/ld-linux-aarch64.so.1"
} else {
"/lib64/ld-linux-x86-64.so.2"
};
if elf.interpreter != Some(interpreter)
|| elf.rpaths != ["$ORIGIN/lib"]
|| !elf.runpaths.is_empty()
{
return Err("compiler requires the GNU loader and inherited RPATH=$ORIGIN/lib".into());
}
} else {
if elf.header.e_type != header::ET_DYN || elf.soname != path.strip_prefix("lib/") {
return Err("compiler library SONAME must match its signed filename".into());
}
// Upstream LLVM uses $ORIGIN/../lib, which resolves to this same
// private lib directory. No search entry may reach the host tree.
if elf
.rpaths
.iter()
.chain(&elf.runpaths)
.any(|path| !matches!(*path, "$ORIGIN" | "$ORIGIN/../lib"))
{
return Err("compiler library search path leaves its private library directory".into());
}
}
let mut dependencies = Vec::new();
for name in elf.libraries {
library_name(name)?;
if !os_library(name) {
dependencies.push(format!("lib/{name}"));
}
}
Ok(dependencies)
}
fn validate_library_directory(
root: &Path,
files: &BTreeMap<&str, &Artifact>,
) -> Result<(), String> {
let mut expected = files
.keys()
.filter_map(|path| path.strip_prefix("lib/"))
.collect::<BTreeSet<_>>();
match std::fs::symlink_metadata(root.join("lib")) {
Err(error) if expected.is_empty() && error.kind() == std::io::ErrorKind::NotFound => {
return Ok(());
}
Err(error) => return Err(format!("cannot inspect compiler lib directory: {error}")),
Ok(_) => {}
}
let directory = runtime_pack::checked_path(root, "lib", false)?;
for entry in std::fs::read_dir(directory).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let name = entry.file_name();
if !entry
.file_type()
.map_err(|error| error.to_string())?
.is_file()
|| !name.to_str().is_some_and(|name| expected.remove(name))
{
return Err(
"compiler lib directory contains an unsigned file, directory or link".into(),
);
}
}
if !expected.is_empty() {
return Err("compiler lib directory is missing a signed library".into());
}
Ok(())
}
+205
View File
@@ -0,0 +1,205 @@
use std::ffi::{OsStr, OsString};
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use super::release::{Layout, MachineManager, Version, platform_contract};
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
#[derive(Debug)]
pub enum CommandResult {
Exited(i32),
Message(String),
}
pub fn execute(executable: &Path, arguments: &[OsString]) -> Result<CommandResult, String> {
if platform_contract(std::env::consts::OS).is_none() {
return Err(format!(
"Dever does not support platform '{}'",
std::env::consts::OS
));
}
let layout = Layout::from_launcher(executable)?;
execute_in(&layout, arguments)
}
pub fn execute_in(layout: &Layout, arguments: &[OsString]) -> Result<CommandResult, String> {
layout.validate_machine_permissions()?;
#[cfg(target_os = "linux")]
super::bootstrap::require_no_pending(layout)?;
let manager = MachineManager::new(layout.clone());
match arguments {
[command, requested] if command == "install" => {
let requested = utf8(requested, "version")?;
super::release_source::prepare(layout, requested)?;
let version = manager.install_requested(requested)?;
Ok(CommandResult::Message(format!("installed Dever {version}")))
}
[command] if command == "update" => {
super::release_source::prepare(layout, "latest")?;
let version = manager.update()?;
Ok(CommandResult::Message(format!(
"updated Dever and dever-language skill to {version}"
)))
}
[command, version] if command == "use" => {
let version = Version::parse(utf8(version, "version")?)?;
manager.activate(&version)?;
Ok(CommandResult::Message(format!("active Dever {version}")))
}
[command, version] if command == "uninstall" => {
let version = Version::parse(utf8(version, "version")?)?;
manager.uninstall(&version)?;
Ok(CommandResult::Message(format!(
"uninstalled Dever {version}"
)))
}
[command] if command == "version" => {
let version = manager
.active_version()?
.ok_or("no active Dever version; run 'dever install latest'")?;
manager.resolve_core(&version)?;
Ok(CommandResult::Message(version.to_string()))
}
[command, project] if command == "version" => {
let root = PathBuf::from(project);
let version = select_version(&manager, &root)?;
Ok(CommandResult::Message(version.to_string()))
}
[cache, status] if cache == "cache" && status == "status" => {
let status = crate::toolchain::cache_status(layout)?;
Ok(CommandResult::Message(
serde_json::to_string(&status).map_err(|error| error.to_string())?,
))
}
[cache, clean] if cache == "cache" && clean == "clean" => {
let status = crate::toolchain::cache_clean(
layout,
&manager.installed_versions()?.into_iter().collect(),
)?;
Ok(CommandResult::Message(
serde_json::to_string(&status).map_err(|error| error.to_string())?,
))
}
[skill, path] if skill == "skill" && path == "path" => Ok(CommandResult::Message(
super::skill::path(&manager)?.display().to_string(),
)),
[skill, path, project] if skill == "skill" && path == "path" => {
let version = select_version(&manager, Path::new(project))?;
Ok(CommandResult::Message(
manager.resolve_skill(&version)?.display().to_string(),
))
}
[skill, install, directory] if skill == "skill" && install == "install" => {
super::skill::install(&manager, Path::new(directory))?;
Ok(CommandResult::Message(format!(
"installed dever-language skill entry in '{}'",
Path::new(directory).display()
)))
}
[command, ..] if command == "new" => {
let version = manager
.active_version()?
.ok_or("no active Dever version; run 'dever install latest'")?;
dispatch_core(&manager, &version, arguments)
}
[help] if help == "--help" => Ok(CommandResult::Message(format!(
"{USAGE}\n dever new <new-directory> [--markdown]\n dever skill path [project-root]\n dever skill install <new-directory>"
))),
[] => Err(USAGE.into()),
_ => dispatch_project_command(&manager, arguments),
}
}
pub fn project_version(project_root: &Path) -> Result<Option<Version>, String> {
let path = project_root.join("config/setting.json");
let text = fs::read_to_string(&path)
.map_err(|error| format!("cannot read '{}': {error}", path.display()))?;
let root = dever_runtime::wire::parse(&text)
.map_err(|error| format!("invalid '{}': {error}", path.display()))?;
let fields = root
.object()
.map_err(|_| format!("invalid '{}': expected a JSON object", path.display()))?;
let Some(dever) = fields.get("dever") else {
return Ok(None);
};
let fields = dever
.fields(&["version"])
.map_err(|error| format!("invalid '{}': dever: {error}", path.display()))?;
let version = fields
.get("version")
.ok_or_else(|| format!("invalid '{}': dever.version is required", path.display()))?
.text()
.map_err(|error| format!("invalid '{}': dever.version: {error}", path.display()))?;
Version::parse(version)
.map(Some)
.map_err(|error| format!("invalid '{}': {error}", path.display()))
}
fn dispatch_project_command(
manager: &MachineManager,
arguments: &[OsString],
) -> Result<CommandResult, String> {
let root = project_root(arguments).ok_or_else(|| USAGE.to_owned())?;
let version = select_version(manager, &root)?;
dispatch_core(manager, &version, arguments)
}
fn dispatch_core(
manager: &MachineManager,
version: &Version,
arguments: &[OsString],
) -> Result<CommandResult, String> {
let core = manager.resolve_core(version)?;
let status = Command::new(&core)
.args(arguments)
.env_clear()
.status()
.map_err(|error| format!("cannot run Dever {version} '{}': {error}", core.display()))?;
Ok(CommandResult::Exited(status.code().unwrap_or(1)))
}
fn select_version(manager: &MachineManager, project_root: &Path) -> Result<Version, String> {
let version = match project_version(project_root)? {
Some(version) => version,
None => manager
.active_version()?
.ok_or("no active Dever version; run 'dever install latest'")?,
};
manager.resolve_core(&version)?;
Ok(version)
}
fn project_root(arguments: &[OsString]) -> Option<PathBuf> {
match arguments {
[command, _operation, root, ..] if command == "lib" || command == "package" => {
Some(root.into())
}
[command, root, ..]
if matches!(
command.to_str(),
Some("check" | "api" | "fmt" | "test" | "run" | "build" | "clean")
) =>
{
Some(root.into())
}
[tenant, operation, root, ..]
if tenant == "tenant" && matches!(operation.to_str(), Some("migrate" | "owner")) =>
{
Some(root.into())
}
[tenant, component, _operation, root, ..]
if tenant == "tenant" && component == "component" =>
{
Some(root.into())
}
_ => None,
}
}
fn utf8<'a>(value: &'a OsStr, label: &str) -> Result<&'a str, String> {
value
.to_str()
.ok_or_else(|| format!("{label} must be UTF-8"))
}
+25
View File
@@ -0,0 +1,25 @@
#[cfg(target_os = "linux")]
pub mod bootstrap;
mod cache;
pub mod compilation;
mod core_libraries;
mod launcher;
pub mod packaging;
mod release;
mod release_source;
pub mod runtime_pack;
mod service;
mod skill;
mod target;
pub use target::BuildTarget;
pub use cache::{
ArtifactReceipt, BuildIdentity, CacheEntry, CacheStatus, CacheStore, CachedArtifact,
};
pub use launcher::{CommandResult, execute, execute_in, project_version};
pub use release::{
Artifact, DownloadedReleases, Layout, MachineManager, PlatformContract, ReleaseManifest,
Version, platform_contract, platform_identity, sha256_file,
};
pub use service::{artifact_get, artifact_put, cache_clean, cache_status, compile, serve};
+520
View File
@@ -0,0 +1,520 @@
//! Private authoring of signed native releases from explicitly prepared inputs.
//! Compilation and tool discovery do not belong to this entry point.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, File};
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use ring::signature::Ed25519KeyPair;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use super::runtime_pack::{self, Manifest, Profile, Profiles, RuntimeArtifact};
use super::{Artifact, BuildTarget, ReleaseManifest, Version, sha256_file};
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
mod bootstrap;
mod builds;
mod ecosystems;
mod sandbox;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Settings {
format: String,
version: String,
target: String,
signing_key: String,
core: Input,
core_libraries: Vec<LinkInput>,
skill: Vec<LinkInput>,
#[serde(default)]
bootstrap: Option<bootstrap::Inputs>,
profiles: ArchiveInputs,
start: Vec<LinkInput>,
libraries: Vec<LinkInput>,
end: Vec<LinkInput>,
#[serde(default)]
native_targets: BTreeMap<BuildTarget, NativeInputs>,
#[serde(default)]
runtimes: ecosystems::RuntimeInputs,
#[serde(default)]
sandbox: Option<Vec<LinkInput>>,
#[serde(default)]
builds: builds::Inputs,
#[serde(default)]
ecosystem_targets: BTreeMap<BuildTarget, ecosystems::RuntimeInputs>,
#[serde(default)]
build_targets: BTreeMap<BuildTarget, builds::Inputs>,
#[serde(default)]
sandbox_targets: BTreeMap<BuildTarget, Vec<LinkInput>>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct NativeInputs {
profiles: ArchiveInputs,
start: Vec<LinkInput>,
libraries: Vec<LinkInput>,
end: Vec<LinkInput>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Input {
source: String,
sha256: String,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct LinkInput {
source: String,
path: String,
sha256: String,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct ArchiveInputs {
base: Input,
sqlite: Input,
postgres: Input,
both: Input,
}
struct SigningKey {
pair: Ed25519KeyPair,
sha256: String,
}
struct PayloadWriter<'a> {
root: &'a Path,
staging: &'a Path,
key: &'a SigningKey,
destinations: BTreeSet<String>,
}
/// Create a new release directory. Inputs come only from author-root/config/setting.json.
pub fn create(author_root: &Path, output: &Path) -> Result<ReleaseManifest, String> {
let target = runtime_pack::host_target()?;
let metadata = fs::symlink_metadata(author_root)
.map_err(|error| format!("cannot inspect author root: {error}"))?;
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err("author root must be a real directory".into());
}
let root = fs::canonicalize(author_root)
.map_err(|error| format!("cannot resolve author root: {error}"))?;
let config = runtime_pack::checked_path(&root, "config/setting.json", true)?;
let bytes = read_bounded(&config, 16 * 1024 * 1024, "author configuration")?;
// Do not echo configuration values: they include the private signing-key location.
let settings: Settings = serde_json::from_slice(&bytes).map_err(|error| {
format!(
"invalid author configuration at line {}, column {}",
error.line(),
error.column()
)
})?;
if settings.format != "dever-native-release-input-v1" || settings.target != target {
return Err("unsupported native release input format or target".into());
}
let version = Version::parse(&settings.version)?;
let key = read_signing_key(&root, &settings.signing_key)?;
let output = output_path(output)?;
let staging = create_staging(output.parent().expect("resolved output parent"))?;
let result = assemble(&root, &staging, settings, version, &key)
.and_then(|manifest| publish(&staging, &output).map(|()| manifest));
if result.is_err()
&& let Err(error) = fs::remove_dir_all(&staging)
{
return Err(format!(
"{}; cannot remove owned release staging: {error}",
result.expect_err("failed release creation")
));
}
result
}
fn assemble(
root: &Path,
staging: &Path,
settings: Settings,
version: Version,
key: &SigningKey,
) -> Result<ReleaseManifest, String> {
let host = BuildTarget::host()?;
let mut writer = PayloadWriter {
root,
staging,
key,
destinations: BTreeSet::from(["manifest.json".to_owned(), "manifest.sig".to_owned()]),
};
let mut artifacts = Vec::new();
artifacts.push(writer.copy(
&settings.core.source,
"dever-core",
&settings.core.sha256,
true,
)?);
for input in &settings.core_libraries {
super::core_libraries::library_name(&input.path)?;
artifacts.push(writer.copy(
&input.source,
&format!("lib/{}", input.path),
&input.sha256,
false,
)?);
}
super::core_libraries::validate(staging, &artifacts)?;
if settings.skill.len() > 256 || !settings.skill.iter().any(|input| input.path == "SKILL.md") {
return Err("release skill requires SKILL.md and at most 256 files".into());
}
let mut skill_bytes = 0_u64;
for input in settings.skill {
let source = writer.source(&input.source, &input.path, &input.sha256)?;
let bytes = fs::metadata(source)
.map_err(|error| error.to_string())?
.len();
skill_bytes = skill_bytes
.checked_add(bytes)
.ok_or("release skill size overflow")?;
if bytes > 1024 * 1024 || skill_bytes > 8 * 1024 * 1024 {
return Err("release skill exceeds its 1 MiB file or 8 MiB total limit".into());
}
artifacts.push(writer.copy(
&input.source,
&format!("skills/dever-language/{}", input.path),
&input.sha256,
false,
)?);
}
if let Some(input) = settings.bootstrap {
artifacts.extend(bootstrap::assemble(&mut writer, input)?);
}
artifacts.extend(assemble_native(
&mut writer,
&version,
host,
NativeInputs {
profiles: settings.profiles,
start: settings.start,
libraries: settings.libraries,
end: settings.end,
},
)?);
for (target, inputs) in settings.native_targets {
artifacts.extend(assemble_native(&mut writer, &version, target, inputs)?);
}
for (ecosystem, input) in settings.runtimes.entries() {
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, host)?);
}
for (target, inputs) in settings.ecosystem_targets {
for (ecosystem, input) in inputs.entries() {
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, target)?);
}
}
if let Some(inputs) = settings.sandbox {
artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?);
}
for (target, inputs) in settings.sandbox_targets {
artifacts.extend(sandbox::assemble(&mut writer, inputs, target)?);
}
for (ecosystem, input) in settings.builds.entries() {
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, host)?);
}
for (target, inputs) in settings.build_targets {
for (ecosystem, input) in inputs.entries() {
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, target)?);
}
}
artifacts.sort_by(|left, right| left.path.cmp(&right.path));
let manifest = ReleaseManifest::new(version, artifacts);
let bytes = write_metadata(&staging.join("manifest.json"), &manifest)?;
let signature = key.pair.sign(&bytes);
let hex: String = signature
.as_ref()
.iter()
.map(|byte| format!("{byte:02x}"))
.collect();
fs::write(staging.join("manifest.sig"), format!("{hex}\n"))
.map_err(|error| format!("cannot write release signature: {error}"))?;
set_mode(&staging.join("manifest.sig"), false)?;
Ok(manifest)
}
fn assemble_native(
writer: &mut PayloadWriter<'_>,
version: &Version,
target: BuildTarget,
inputs: NativeInputs,
) -> Result<Vec<Artifact>, String> {
let runtime_prefix = format!("runtime/{}", target.platform());
let manifest_path = writer.reserve(&format!("{runtime_prefix}/manifest.json"))?;
let archive_inputs = inputs.profiles;
let mut files = Vec::new();
for (name, input) in [
("base", archive_inputs.base),
("sqlite", archive_inputs.sqlite),
("postgres", archive_inputs.postgres),
("both", archive_inputs.both),
] {
let path = format!("archives/{name}.a");
let artifact = writer.copy(
&input.source,
&format!("{runtime_prefix}/{path}"),
&input.sha256,
false,
)?;
files.push(RuntimeArtifact {
path,
bytes: artifact.bytes,
sha256: artifact.sha256,
});
}
for input in inputs
.start
.iter()
.chain(&inputs.libraries)
.chain(&inputs.end)
{
runtime_pack::relative_path(&input.path)?;
let artifact = writer.copy(
&input.source,
&format!("{runtime_prefix}/{}", input.path),
&input.sha256,
false,
)?;
files.push(RuntimeArtifact {
path: input.path.clone(),
bytes: artifact.bytes,
sha256: artifact.sha256,
});
}
files.sort_by(|left, right| left.path.cmp(&right.path));
let link_paths = |inputs: &[LinkInput]| inputs.iter().map(|input| input.path.clone()).collect();
let profile = |name| {
Some(Profile {
start: link_paths(&inputs.start),
runtime: format!("archives/{name}.a"),
libraries: link_paths(&inputs.libraries),
end: link_paths(&inputs.end),
})
};
let runtime = Manifest {
format: "dever-native-runtime-v1".into(),
compiler_version: version.to_string(),
abi: 1,
platform: target.platform().into(),
target: target.triple().into(),
profiles: Profiles {
base: profile("base"),
sqlite: profile("sqlite"),
postgres: profile("postgres"),
both: profile("both"),
},
files,
};
write_metadata(&manifest_path, &runtime)?;
Ok(
runtime_pack::validate(writer.staging, version.as_str(), target)?
.into_iter()
.map(|artifact| Artifact {
path: artifact.path,
bytes: artifact.bytes,
sha256: artifact.sha256,
})
.collect(),
)
}
impl PayloadWriter<'_> {
fn reserve(&mut self, destination: &str) -> Result<PathBuf, String> {
runtime_pack::relative_path(destination)?;
if self.destinations.iter().any(|path| {
path == destination
|| path.starts_with(&format!("{destination}/"))
|| destination.starts_with(&format!("{path}/"))
}) {
return Err("duplicate release destination or file/directory conflict".into());
}
self.destinations.insert(destination.to_owned());
let output = self.staging.join(destination);
create_directories(self.staging, output.parent().expect("payload parent"))?;
Ok(output)
}
fn source(&self, source: &str, destination: &str, digest: &str) -> Result<PathBuf, String> {
if digest.len() != 64
|| !digest
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err("payload requires a lowercase SHA-256 digest".into());
}
if digest == self.key.sha256 {
return Err("signing key cannot be included as a release payload".into());
}
let source = runtime_pack::checked_path(self.root, source, true)
.map_err(|_| format!("invalid source for release payload '{destination}'"))?;
if sha256_file(&source).map_err(|_| "cannot hash source payload")? != digest {
return Err(format!(
"release payload '{destination}' failed SHA-256 verification"
));
}
Ok(source)
}
fn copy(
&mut self,
source: &str,
destination: &str,
digest: &str,
executable: bool,
) -> Result<Artifact, String> {
let output = self.reserve(destination)?;
let source = self.source(source, destination, digest)?;
fs::copy(&source, &output)
.map_err(|error| format!("cannot copy release payload '{destination}': {error}"))?;
if sha256_file(&output)? != digest {
return Err(format!(
"copied release payload '{destination}' failed SHA-256 verification"
));
}
set_mode(&output, executable)?;
Ok(Artifact {
path: destination.to_owned(),
bytes: fs::metadata(output)
.map_err(|error| error.to_string())?
.len(),
sha256: digest.to_owned(),
})
}
}
fn read_signing_key(root: &Path, relative: &str) -> Result<SigningKey, String> {
let path = runtime_pack::checked_path(root, relative, true)
.map_err(|_| "signing key must be a real file within the author root")?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let metadata = fs::metadata(&path).map_err(|_| "cannot inspect signing key")?;
if metadata.permissions().mode() & 0o077 != 0 {
return Err("signing key permissions must be private to its owner".into());
}
}
let bytes = read_bounded(&path, 4096, "signing key")?;
let pair = Ed25519KeyPair::from_pkcs8(&bytes)
.map_err(|_| "signing key is not a valid Ed25519 PKCS#8 key")?;
Ok(SigningKey {
pair,
sha256: format!("{:x}", Sha256::digest(bytes)),
})
}
fn read_bounded(path: &Path, limit: u64, label: &str) -> Result<Vec<u8>, String> {
let mut bytes = Vec::new();
File::open(path)
.and_then(|file| file.take(limit + 1).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read {label}: {error}"))?;
if bytes.len() as u64 > limit {
return Err(format!("{label} exceeds its size limit"));
}
Ok(bytes)
}
fn write_metadata(path: &Path, value: &impl serde::Serialize) -> Result<Vec<u8>, String> {
let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?;
bytes.push(b'\n');
fs::write(path, &bytes).map_err(|error| format!("cannot write release metadata: {error}"))?;
set_mode(path, false)?;
Ok(bytes)
}
fn output_path(output: &Path) -> Result<PathBuf, String> {
let name = output
.file_name()
.ok_or("release output requires a new directory name")?;
let parent = output
.parent()
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let parent = fs::canonicalize(parent)
.map_err(|error| format!("cannot resolve release output parent: {error}"))?;
let output = parent.join(name);
match fs::symlink_metadata(&output) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(output),
Ok(_) => Err("release output already exists".into()),
Err(error) => Err(format!("cannot inspect release output: {error}")),
}
}
fn create_staging(parent: &Path) -> Result<PathBuf, String> {
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
loop {
let path = parent.join(format!(
".native-release-{}-{}",
std::process::id(),
NEXT_STAGING.fetch_add(1, Ordering::Relaxed)
));
match builder.create(&path) {
Ok(()) => return Ok(path),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(format!("cannot create release staging: {error}")),
}
}
}
fn create_directories(root: &Path, directory: &Path) -> Result<(), String> {
let mut current = root.to_owned();
for component in directory
.strip_prefix(root)
.expect("owned staging path")
.components()
{
current.push(component);
if !current.exists() {
fs::create_dir(&current)
.map_err(|error| format!("cannot create payload directory: {error}"))?;
set_mode(&current, true)?;
}
}
Ok(())
}
fn set_mode(path: &Path, executable: bool) -> Result<(), String> {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(
path,
fs::Permissions::from_mode(if executable { 0o755 } else { 0o644 }),
)
.map_err(|error| format!("cannot set release permissions: {error}"))?;
}
#[cfg(not(unix))]
let _ = (path, executable);
Ok(())
}
fn publish(staging: &Path, output: &Path) -> Result<(), String> {
set_mode(staging, true)?;
#[cfg(target_os = "linux")]
{
use rustix::fs::{CWD, RenameFlags, renameat_with};
renameat_with(CWD, staging, CWD, output, RenameFlags::NOREPLACE)
.map_err(|error| format!("cannot publish new release directory: {error}"))
}
#[cfg(not(target_os = "linux"))]
{
let _ = output;
Err("native release publication is not available for this platform".into())
}
}
@@ -0,0 +1,64 @@
//! The stable Linux entry points share a small, separately checked loader closure.
use serde::Deserialize;
use super::{Artifact, Input, LinkInput, PayloadWriter};
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Inputs {
launcher: Input,
daemon: Input,
libraries: Vec<LinkInput>,
}
pub(super) fn assemble(
writer: &mut PayloadWriter<'_>,
input: Inputs,
) -> Result<Vec<Artifact>, String> {
if !cfg!(target_os = "linux") {
return Err("bootstrap authoring currently requires Linux".into());
}
let mut artifacts = Vec::new();
for (name, input) in [("dever", input.launcher), ("deverd", input.daemon)] {
artifacts.push(writer.copy(
&input.source,
&format!("bootstrap/{name}"),
&input.sha256,
true,
)?);
}
for library in input.libraries {
crate::toolchain::core_libraries::library_name(&library.path)?;
artifacts.push(writer.copy(
&library.source,
&format!("bootstrap/lib/{}", library.path),
&library.sha256,
false,
)?);
}
#[cfg(target_os = "linux")]
{
use sha2::{Digest, Sha256};
for (path, bytes) in [
(
"bootstrap/deverd.service",
crate::toolchain::bootstrap::SERVICE.as_bytes(),
),
(
"bootstrap/dever-sandbox",
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
),
] {
let destination = writer.reserve(path)?;
std::fs::write(&destination, bytes).map_err(|error| error.to_string())?;
super::set_mode(&destination, false)?;
artifacts.push(Artifact {
path: path.into(),
bytes: bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(bytes)),
});
}
crate::toolchain::bootstrap::validate_payload(writer.staging, &artifacts, true)?;
}
Ok(artifacts)
}
@@ -0,0 +1,180 @@
//! Assemble source-build tools against already prepared runtime identities.
use flate2::{Compression, GzBuilder};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::fs::{self, File};
use super::{Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, write_metadata};
use crate::libs::build::pack::{self, Descriptor, Manifest};
use crate::libs::{Ecosystem, InstalledRegistryPack};
#[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Inputs {
pip: Option<Input>,
npm: Option<Input>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Input {
executables: Vec<String>,
files: Vec<LinkInput>,
}
impl Inputs {
pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, Input)> {
[(Ecosystem::Pip, self.pip), (Ecosystem::Npm, self.npm)]
.into_iter()
.filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input)))
}
}
pub(super) fn assemble(
writer: &mut PayloadWriter<'_>,
ecosystem: Ecosystem,
input: Input,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<Artifact>, String> {
let target = target.platform().to_owned();
let runtime = |ecosystem: &Ecosystem| -> Result<InstalledRegistryPack, String> {
let path = writer.staging.join(format!(
"runtime/{}/{target}/manifest.json",
ecosystem.as_str()
));
let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded(
&path,
1024 * 1024,
"build runtime descriptor",
)?)
.map_err(|error| format!("invalid build runtime descriptor: {error}"))?;
manifest.validate(ecosystem, &target)?;
Ok(manifest)
};
let runtime = runtime(&ecosystem)?;
let python = if ecosystem == Ecosystem::Npm {
let path = writer
.staging
.join(format!("runtime/pip/{target}/manifest.json"));
let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded(
&path,
1024 * 1024,
"auxiliary Python runtime descriptor",
)?)
.map_err(|error| format!("invalid auxiliary Python runtime: {error}"))?;
manifest.validate(&Ecosystem::Pip, &target)?;
Some(manifest)
} else {
None
};
let mut descriptor = Descriptor {
format: "dever-registry-build-v1".into(),
ecosystem: ecosystem.clone(),
target: target.clone(),
runtime_sha256: runtime.runtime.pack.sha256.clone(),
python_runtime_sha256: python
.as_ref()
.map(|runtime| runtime.runtime.pack.sha256.clone()),
sha256: "0".repeat(64),
};
descriptor.validate(
&runtime.runtime,
python.as_ref().map(|manifest| &manifest.runtime),
)?;
let manifest = Manifest {
format: "dever-lib-build-v1".into(),
ecosystem: ecosystem.clone(),
target: target.clone(),
runtime_sha256: descriptor.runtime_sha256.clone(),
python_runtime_sha256: descriptor.python_runtime_sha256.clone(),
executables: input.executables,
};
if input.files.len() >= pack::MAX_FILES {
return Err("build pack has too many leaves".into());
}
let metadata = serde_json::to_vec(&manifest).map_err(|error| error.to_string())?;
let mut remaining = pack::MAX_EXPANDED_BYTES
.checked_sub(metadata.len())
.ok_or("build manifest is too large")? as u64;
let mut files = vec![("dever-build.json".into(), metadata)];
for input in input.files {
let path = writer.source(&input.source, &input.path, &input.sha256)?;
let bytes = read_bounded(&path, remaining, "build tool leaf")?;
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
return Err("build tool changed during packaging".into());
}
remaining -= bytes.len() as u64;
files.push((input.path, bytes));
}
files.sort_by(|left, right| left.0.cmp(&right.0));
manifest.validate(&files, &descriptor)?;
let mut assets = Vec::new();
for directory in ["bin", "lib"] {
for entry in fs::read_dir(writer.staging.join(format!("sandbox/{target}/{directory}")))
.map_err(|error| format!("build pack requires prepared sandbox assets: {error}"))?
{
let entry = entry.map_err(|error| error.to_string())?;
let name = entry
.file_name()
.into_string()
.map_err(|_| "sandbox asset name is not UTF-8")?;
assets.push((
format!("{directory}/{name}"),
read_bounded(&entry.path(), 32 * 1024 * 1024, "sandbox build asset")?,
));
}
}
dever_sandbox::validate_assets_for_target(&assets, &target)?;
pack::validate_tools(&manifest, &files, &assets)?;
let prefix = format!("build/{}/{target}", ecosystem.as_str());
let archive_name = format!("{prefix}/build.pack");
let path = writer.reserve(&archive_name)?;
let encoder = GzBuilder::new().mtime(0).operating_system(255).write(
File::create(&path).map_err(|error| error.to_string())?,
Compression::default(),
);
let mut archive = tar::Builder::new(encoder);
for (path, bytes) in files {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(if manifest.executables.contains(&path) {
0o755
} else {
0o644
});
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_cksum();
archive
.append_data(&mut header, path, bytes.as_slice())
.map_err(|error| error.to_string())?;
}
archive
.into_inner()
.and_then(|encoder| encoder.finish())
.map_err(|error| error.to_string())?;
set_mode(&path, false)?;
let bytes = fs::metadata(&path)
.map_err(|error| error.to_string())?
.len();
if bytes > pack::MAX_PACK_BYTES as u64 {
return Err("compressed build pack exceeds 256 MiB".into());
}
descriptor.sha256 = super::sha256_file(&path)?;
let descriptor_name = format!("{prefix}/manifest.json");
let metadata = write_metadata(&writer.reserve(&descriptor_name)?, &descriptor)?;
Ok(vec![
Artifact {
path: archive_name,
bytes,
sha256: descriptor.sha256,
},
Artifact {
path: descriptor_name,
bytes: metadata.len() as u64,
sha256: format!("{:x}", Sha256::digest(&metadata)),
},
])
}
@@ -0,0 +1,152 @@
//! Deterministic ecosystem packs inside the same signed native release.
use std::fs::{self, File};
use flate2::{Compression, GzBuilder};
use pep508_rs::MarkerEnvironment;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use super::{
Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, sha256_file, write_metadata,
};
use crate::libs::{Ecosystem, InstalledRegistryPack, RegistryRuntime, RuntimePack};
use crate::workers::{MAX_TREE_BYTES, MAX_TREE_FILES, runtime};
#[derive(Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct RuntimeInputs {
pip: Option<RuntimeInput>,
npm: Option<RuntimeInput>,
go: Option<RuntimeInput>,
}
impl RuntimeInputs {
pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, RuntimeInput)> {
[
(Ecosystem::Pip, self.pip),
(Ecosystem::Npm, self.npm),
(Ecosystem::Go, self.go),
]
.into_iter()
.filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input)))
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(super) struct RuntimeInput {
name: String,
version: String,
python_markers: Option<MarkerEnvironment>,
#[serde(default)]
python_wheel_tags: Vec<String>,
npm_libc: Option<String>,
files: Vec<LinkInput>,
}
pub(super) fn assemble(
writer: &mut PayloadWriter<'_>,
ecosystem: Ecosystem,
input: RuntimeInput,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<Artifact>, String> {
let target = target.platform().to_owned();
let mut metadata = InstalledRegistryPack {
format: "dever-registry-runtime-v1".into(),
runtime: RegistryRuntime {
pack: RuntimePack {
name: input.name,
version: input.version,
sha256: "0".repeat(64),
},
target: target.clone(),
python_markers: input.python_markers,
python_wheel_tags: input.python_wheel_tags,
npm_libc: input.npm_libc,
},
};
metadata.validate(&ecosystem, &target)?;
if input.files.len() > MAX_TREE_FILES {
return Err("managed runtime pack has too many files".into());
}
let mut files = Vec::with_capacity(input.files.len());
let mut remaining = MAX_TREE_BYTES as u64;
for input in input.files {
super::runtime_pack::relative_path(&input.path)?;
// A runtime archive is built from its leaf files, never another completed pack.
if input.path == "runtime.pack" {
return Err("runtime pack cannot include another runtime.pack".into());
}
let source = writer.source(&input.source, &input.path, &input.sha256)?;
let bytes = read_bounded(&source, remaining, "runtime payload")?;
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
return Err("runtime payload changed during packaging".into());
}
remaining -= bytes.len() as u64;
files.push((input.path, bytes));
}
files.sort_by(|left, right| left.0.cmp(&right.0));
let manifest = runtime::validate(&files, &ecosystem, &target)?;
if manifest.python_wheel_tags != metadata.runtime.python_wheel_tags
|| manifest.python_markers != metadata.runtime.python_markers
{
return Err(
"runtime pack Python metadata differs from its signed registry descriptor".into(),
);
}
let prefix = format!("runtime/{}/{target}", ecosystem.as_str());
let archive_name = format!("{prefix}/runtime.pack");
let archive_path = writer.reserve(&archive_name)?;
let output = File::create(&archive_path)
.map_err(|error| format!("cannot create runtime pack: {error}"))?;
let encoder = GzBuilder::new()
.mtime(0)
.operating_system(255)
.write(output, Compression::default());
let mut archive = tar::Builder::new(encoder);
for (path, bytes) in &files {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(if manifest.executable(path) {
0o755
} else {
0o644
});
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_cksum();
archive
.append_data(&mut header, path, bytes.as_slice())
.map_err(|error| format!("cannot write runtime pack entry: {error}"))?;
}
archive
.into_inner()
.and_then(|encoder| encoder.finish())
.map_err(|error| format!("cannot finish runtime pack: {error}"))?;
set_mode(&archive_path, false)?;
let size = fs::metadata(&archive_path)
.map_err(|error| error.to_string())?
.len();
if size > MAX_TREE_BYTES as u64 {
return Err("compressed runtime pack exceeds 256 MiB".into());
}
metadata.runtime.pack.sha256 = sha256_file(&archive_path)?;
let pack = Artifact {
path: archive_name,
bytes: size,
sha256: metadata.runtime.pack.sha256.clone(),
};
let metadata_name = format!("{prefix}/manifest.json");
let metadata_path = writer.reserve(&metadata_name)?;
let bytes = write_metadata(&metadata_path, &metadata)?;
Ok(vec![
pack,
Artifact {
path: metadata_name,
bytes: bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(bytes)),
},
])
}
@@ -0,0 +1,43 @@
//! Sandbox tools and OS libraries are authenticated by the existing release.
use super::{Artifact, LinkInput, PayloadWriter, read_bounded};
use sha2::{Digest, Sha256};
pub(super) fn assemble(
writer: &mut PayloadWriter<'_>,
inputs: Vec<LinkInput>,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<Artifact>, String> {
if inputs.len() > 128 {
return Err("sandbox pack contains too many files".into());
}
let mut files = Vec::new();
let mut remaining = 32 * 1024 * 1024;
for input in inputs {
super::runtime_pack::relative_path(&input.path)?;
let source = writer.source(&input.source, &input.path, &input.sha256)?;
let bytes = read_bounded(&source, remaining, "sandbox asset")?;
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
return Err("sandbox asset changed during packaging".into());
}
remaining -= bytes.len() as u64;
files.push((input.path, bytes));
}
files.sort_by(|left, right| left.0.cmp(&right.0));
dever_sandbox::validate_assets_for_target(&files, target.platform())?;
// Leaf artifacts avoid a second manifest/signature or extraction protocol.
let mut artifacts = Vec::new();
for (path, bytes) in files {
let destination = format!("sandbox/{}/{path}", target.platform());
let output = writer.reserve(&destination)?;
std::fs::write(&output, &bytes)
.map_err(|error| format!("cannot write sandbox asset: {error}"))?;
super::set_mode(&output, dever_sandbox::asset_executable(&path))?;
artifacts.push(Artifact {
path: destination,
bytes: bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(bytes)),
});
}
Ok(artifacts)
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,387 @@
//! Explicit installation/update downloads. Project commands never enter here.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use super::release::{
InstallLock, Layout, MachineManager, ReleaseManifest, Version, decode_hex_file,
ensure_real_directory, platform_identity, reject_symlink_ancestors,
set_shared_file_permissions, sync_directory, sync_tree, validate_shared_file,
};
const OFFICIAL_RELEASES: &str = "https://github.com/shemic/dever-main/releases";
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300);
const METADATA_LIMIT: u64 = 2 * 1024 * 1024;
const RELEASE_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
static NEXT_DOWNLOAD: AtomicU64 = AtomicU64::new(0);
struct ReleaseSource {
base: String,
agent: ureq::Agent,
}
impl ReleaseSource {
fn official() -> Self {
Self::new(OFFICIAL_RELEASES.to_owned())
}
fn new(base: String) -> Self {
Self {
base,
agent: ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.proxy(None)
.max_redirects(0)
.timeout_global(Some(DOWNLOAD_TIMEOUT))
.build(),
),
}
}
fn asset(&self, requested: &str, suffix: &str) -> String {
let selection = if requested == "latest" {
"latest/download".to_owned()
} else {
format!("download/v{requested}")
};
format!(
"{}/{selection}/dever-{}.{suffix}",
self.base,
platform_identity()
)
}
fn response(
&self,
mut address: String,
started: Instant,
) -> Result<ureq::http::Response<ureq::Body>, String> {
for _ in 0..=5 {
let remaining = DOWNLOAD_TIMEOUT
.checked_sub(started.elapsed())
.filter(|duration| !duration.is_zero())
.ok_or("Dever release download exceeded its five-minute deadline")?;
let response = self
.agent
.get(&address)
.config()
.timeout_global(Some(remaining))
.build()
.call()
.map_err(|error| match error {
ureq::Error::StatusCode(404) => {
"the official Dever release is not published for this version/platform; no installed version was changed".to_owned()
}
ureq::Error::StatusCode(code) => {
format!("official Dever release download returned HTTP {code}")
}
_ => "cannot download the official Dever release".to_owned(),
})?;
if response.status().is_success() {
return Ok(response);
}
if !response.status().is_redirection() {
return Err("unexpected official release HTTP status".into());
}
let mut locations = response.headers().get_all("location").iter();
let location = locations
.next()
.and_then(|value| value.to_str().ok())
.ok_or("release redirect is missing a valid Location")?;
if locations.next().is_some() {
return Err("release redirect has multiple Locations".into());
}
address = redirect(&address, location)?;
}
Err("official release download exceeded its redirect limit".into())
}
fn metadata(
&self,
requested: &str,
suffix: &str,
path: &Path,
started: Instant,
) -> Result<(), String> {
let mut response = self.response(self.asset(requested, suffix), started)?;
let mut bytes = Vec::new();
response
.body_mut()
.as_reader()
.take(METADATA_LIMIT + 1)
.read_to_end(&mut bytes)
.map_err(|_| "cannot read official release metadata")?;
if bytes.len() as u64 > METADATA_LIMIT {
return Err("official release metadata exceeds its size limit".into());
}
write_new(path, &bytes)
}
}
fn redirect(current: &str, location: &str) -> Result<String, String> {
let next = url::Url::parse(current)
.and_then(|base| base.join(location))
.map_err(|_| "invalid release redirect")?;
if next.scheme() != "https"
|| !next.username().is_empty()
|| next.password().is_some()
|| next.port_or_known_default() != Some(443)
|| next.fragment().is_some()
|| !matches!(
next.host_str(),
Some(
"github.com"
| "release-assets.githubusercontent.com"
| "objects.githubusercontent.com"
)
)
{
return Err("release redirect must remain on the official HTTPS asset hosts".into());
}
Ok(next.into())
}
pub(super) fn prepare(layout: &Layout, requested: &str) -> Result<Version, String> {
prepare_from(layout, requested, &ReleaseSource::official())
}
fn prepare_from(
layout: &Layout,
requested: &str,
source: &ReleaseSource,
) -> Result<Version, String> {
if requested != "latest" {
Version::parse(requested)?;
}
layout.validate_machine_permissions()?;
for directory in [layout.downloads(), layout.staging()] {
reject_symlink_ancestors(&directory)?;
ensure_real_directory(&directory, "release download directory")?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let metadata = fs::metadata(&directory).map_err(|error| error.to_string())?;
if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 {
return Err(
"release download directories must be root-owned and not publicly writable"
.into(),
);
}
}
}
let _lock = InstallLock::acquire(layout)?;
#[cfg(target_os = "linux")]
super::bootstrap::require_no_pending(layout)?;
let key_path = layout.state().join("trusted-release-key");
validate_shared_file(&key_path, &layout.state(), "trusted release key", false)?;
let key = decode_hex_file(&key_path, 32)?;
let staging = Download::new(&layout.staging())?;
let started = Instant::now();
source.metadata(
requested,
"manifest.json",
&staging.0.join("manifest.json"),
started,
)?;
source.metadata(
requested,
"manifest.sig",
&staging.0.join("manifest.sig"),
started,
)?;
let manifest = MachineManager::signed_manifest(&staging.0, &key)?;
if !manifest
.artifacts
.iter()
.any(|artifact| artifact.path == "skills/dever-language/SKILL.md")
{
return Err("official releases must include the matching dever-language skill".into());
}
if requested != "latest" && manifest.version.as_str() != requested {
return Err("downloaded release version does not match the requested version".into());
}
validate_catalog(&manifest)?;
let version = manifest.version.clone();
let destination = layout.downloads().join(version.as_str());
match fs::symlink_metadata(&destination) {
Ok(_) => {
MachineManager::verify_package(&destination, Some(&version), &key)?;
if fs::read(destination.join("manifest.json")).map_err(|error| error.to_string())?
!= fs::read(staging.0.join("manifest.json")).map_err(|error| error.to_string())?
{
return Err("published Dever version changed its signed manifest".into());
}
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {
let mut response =
source.response(source.asset(version.as_str(), "tar.gz"), started)?;
let compressed = response.body_mut().as_reader().take(RELEASE_LIMIT + 1);
unpack(
flate2::read::GzDecoder::new(compressed),
&staging.0,
&manifest,
)?;
MachineManager::verify_package(&staging.0, Some(&version), &key)?;
sync_tree(&staging.0)?;
publish(&staging.0, &destination)?;
sync_directory(&layout.downloads()).map_err(|error| error.to_string())?;
}
Err(error) => return Err(format!("cannot inspect downloaded release: {error}")),
}
// Latest is only a catalog hint; the signed version and active journal own trust.
if requested == "latest" {
let latest = layout.downloads().join("latest");
match fs::symlink_metadata(&latest) {
Ok(_) => validate_shared_file(&latest, &layout.downloads(), "release catalog", false)?,
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(format!("cannot inspect release catalog: {error}")),
}
let temporary = Download::new(&layout.staging())?;
let catalog = temporary.0.join("latest");
write_new(&catalog, format!("{version}\n").as_bytes())?;
fs::rename(catalog, latest)
.map_err(|error| format!("cannot publish release catalog: {error}"))?;
sync_directory(&layout.downloads()).map_err(|error| error.to_string())?;
}
Ok(version)
}
fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
if manifest.artifacts.is_empty() || manifest.artifacts.len() > 4096 {
return Err("release artifact count is outside 1..=4096".into());
}
let mut paths = BTreeSet::new();
let mut size = 0_u64;
for artifact in &manifest.artifacts {
if artifact
.path
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
|| artifact.path.contains(['\\', ':', '\0'])
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|| !paths.insert(&artifact.path)
{
return Err("release has an unsafe or duplicate artifact path".into());
}
size = size
.checked_add(artifact.bytes)
.ok_or("release size overflow")?;
if size > RELEASE_LIMIT {
return Err("release exceeds its two-GiB unpacked size limit".into());
}
}
Ok(())
}
fn unpack(reader: impl Read, root: &Path, manifest: &ReleaseManifest) -> Result<(), String> {
validate_catalog(manifest)?;
let expected: BTreeMap<_, _> = manifest
.artifacts
.iter()
.map(|artifact| (artifact.path.as_str(), artifact))
.collect();
let mut seen = BTreeSet::new();
// Raw entries reject PAX/GNU extensions before tar can buffer arbitrary metadata.
let mut archive = tar::Archive::new(reader.take(RELEASE_LIMIT + 4096 * 1024));
for entry in archive
.entries()
.map_err(|error| format!("invalid release archive: {error}"))?
.raw(true)
{
let mut entry = entry.map_err(|error| format!("invalid release archive entry: {error}"))?;
if !entry.header().entry_type().is_file() {
return Err("release archives may contain only regular files".into());
}
let bytes = entry.path_bytes();
let name = std::str::from_utf8(&bytes)
.map_err(|_| "release archive path is not UTF-8")?
.to_owned();
let artifact = expected
.get(name.as_str())
.ok_or("release archive contains an unsigned file")?;
if !seen.insert(name.clone()) || entry.size() != artifact.bytes {
return Err("release archive contains a duplicate or wrong-sized file".into());
}
let path = root.join(&name);
fs::create_dir_all(path.parent().expect("artifact has a parent"))
.map_err(|error| format!("cannot create release artifact directory: {error}"))?;
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.map_err(|error| format!("cannot create release artifact: {error}"))?;
let bytes = io::copy(&mut entry, &mut file)
.map_err(|error| format!("cannot extract release artifact: {error}"))?;
if bytes != artifact.bytes {
return Err("release archive contains a truncated file".into());
}
set_shared_file_permissions(&path, false)?;
file.sync_all().map_err(|error| error.to_string())?;
}
if seen.len() != expected.len() {
return Err("release archive is missing signed artifacts".into());
}
Ok(())
}
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), String> {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(path)
.map_err(|error| format!("cannot create release metadata: {error}"))?;
file.write_all(bytes)
.and_then(|()| file.sync_all())
.map_err(|error| error.to_string())?;
set_shared_file_permissions(path, false)
}
fn publish(source: &Path, destination: &Path) -> Result<(), String> {
#[cfg(target_os = "linux")]
{
use rustix::fs::{CWD, RenameFlags, renameat_with};
renameat_with(CWD, source, CWD, destination, RenameFlags::NOREPLACE)
.map_err(|error| format!("cannot publish downloaded release: {error}"))
}
#[cfg(not(target_os = "linux"))]
{
let _ = (source, destination);
Err("official release downloads currently require Linux".into())
}
}
struct Download(PathBuf);
impl Download {
fn new(parent: &Path) -> Result<Self, String> {
let path = parent.join(format!(
"download-{}-{}",
std::process::id(),
NEXT_DOWNLOAD.fetch_add(1, Ordering::Relaxed)
));
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
builder
.create(&path)
.map_err(|error| format!("cannot create release download staging: {error}"))?;
Ok(Self(path))
}
}
impl Drop for Download {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
#[cfg(test)]
#[path = "../../../../test/dever-cli-tests/release_source.rs"]
mod tests;
@@ -0,0 +1,464 @@
//! Compiler-relative, explicitly supplied native link inputs. This is not a
//! package resolver or a substitute for the managed compiler service.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::io::{Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use dever_runtime::config::RuntimeProfile;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::{BuildTarget, sha256_file};
const MANIFEST_LIMIT: u64 = 1024 * 1024;
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Manifest {
pub(super) format: String,
pub(super) compiler_version: String,
pub(super) abi: u32,
pub(super) platform: String,
pub(super) target: String,
pub(super) profiles: Profiles,
pub(super) files: Vec<RuntimeArtifact>,
}
#[derive(Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Profiles {
pub(super) base: Option<Profile>,
pub(super) sqlite: Option<Profile>,
pub(super) postgres: Option<Profile>,
pub(super) both: Option<Profile>,
}
impl Profiles {
fn all(&self) -> [(&'static str, Option<&Profile>); 4] {
[
("base", self.base.as_ref()),
("sqlite", self.sqlite.as_ref()),
("postgres", self.postgres.as_ref()),
("both", self.both.as_ref()),
]
}
}
#[derive(Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub(super) struct Profile {
pub(super) start: Vec<String>,
pub(super) runtime: String,
pub(super) libraries: Vec<String>,
pub(super) end: Vec<String>,
}
impl Profile {
fn inputs(&self) -> impl Iterator<Item = &str> {
self.start
.iter()
.chain(std::iter::once(&self.runtime))
.chain(&self.libraries)
.chain(&self.end)
.map(String::as_str)
}
}
#[derive(Clone, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct RuntimeArtifact {
pub path: String,
pub bytes: u64,
pub sha256: String,
}
pub struct RuntimePack {
root: PathBuf,
pub target: BuildTarget,
pub identity: String,
profile: Profile,
files: BTreeMap<String, RuntimeArtifact>,
}
pub struct LinkInputs {
pub start: Vec<PathBuf>,
pub libraries: Vec<PathBuf>,
pub end: Vec<PathBuf>,
}
struct LoadedManifest {
root: PathBuf,
target: BuildTarget,
bytes: Vec<u8>,
profiles: Profiles,
files: BTreeMap<String, RuntimeArtifact>,
}
fn load_manifest(
directory: &Path,
compiler_version: &str,
selected: BuildTarget,
) -> Result<LoadedManifest, String> {
let target = selected.triple();
let platform = selected.platform();
let root = checked_path(directory, &format!("runtime/{platform}"), false)?;
let path = checked_path(&root, "manifest.json", true)?;
let mut bytes = Vec::new();
fs::File::open(&path)
.and_then(|file| file.take(MANIFEST_LIMIT + 1).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read runtime pack manifest: {error}"))?;
if bytes.len() as u64 > MANIFEST_LIMIT {
return Err("runtime pack manifest exceeds 1 MiB".into());
}
let manifest: Manifest = serde_json::from_slice(&bytes)
.map_err(|error| format!("invalid runtime pack manifest: {error}"))?;
if manifest.format != "dever-native-runtime-v1"
|| manifest.compiler_version != compiler_version
// A v1 manifest describes v1 runtime symbols. Even bridge metadata
// references would link the native SDK into the launcher and daemon.
|| manifest.abi != 1
|| manifest.platform != platform
|| manifest.target != target
{
return Err(
"runtime pack format, compiler version, ABI or target does not match this compiler"
.into(),
);
}
let files = validate_files(manifest.files)?;
validate_profiles(&manifest.profiles, &files)?;
Ok(LoadedManifest {
root,
target: selected,
bytes,
profiles: manifest.profiles,
files,
})
}
/// Return the entire runtime closure that must be covered by the signed release.
pub fn validate(
directory: &Path,
compiler_version: &str,
target: BuildTarget,
) -> Result<Vec<RuntimeArtifact>, String> {
let manifest = load_manifest(directory, compiler_version, target)?;
let prefix = format!("runtime/{}", target.platform());
let mut artifacts = vec![RuntimeArtifact {
path: format!("{prefix}/manifest.json"),
bytes: manifest.bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(&manifest.bytes)),
}];
for (path, artifact) in manifest.files {
verify_file(
&checked_path(&manifest.root, &path, true)?,
&artifact,
target,
)?;
artifacts.push(RuntimeArtifact {
path: format!("{prefix}/{path}"),
..artifact
});
}
Ok(artifacts)
}
pub(super) fn host_target() -> Result<&'static str, String> {
Ok(BuildTarget::host()?.triple())
}
impl RuntimePack {
pub fn load(
directory: &Path,
profile: RuntimeProfile,
selected: BuildTarget,
) -> Result<Self, String> {
let LoadedManifest {
root,
target,
bytes,
profiles,
files,
} = load_manifest(directory, env!("CARGO_PKG_VERSION"), selected)?;
let name = match (profile.sqlite, profile.postgres) {
(false, false) => "base",
(true, false) => "sqlite",
(false, true) => "postgres",
(true, true) => "both",
};
let profile = profiles
.all()
.into_iter()
.find_map(|(key, value)| (key == name).then_some(value).flatten())
.ok_or_else(|| format!("runtime pack is missing the required '{name}' profile"))?
.clone();
// Validate even on a cache hit: a corrupt installed pack is never
// silently replaced with an old cached executable.
for input in profile.inputs() {
verify_file(&checked_path(&root, input, true)?, &files[input], selected)?;
}
Ok(Self {
root,
target,
identity: format!("{name}:{:x}", Sha256::digest(bytes)),
profile,
files,
})
}
pub fn snapshot(&self, directory: &Path) -> Result<LinkInputs, String> {
let destination = directory.join("runtime");
fs::create_dir(&destination)
.map_err(|error| format!("cannot create runtime input snapshot: {error}"))?;
let mut inputs = Vec::new();
for (index, input) in self.profile.inputs().enumerate() {
let source = checked_path(&self.root, input, true)?;
let extension = Path::new(input)
.extension()
.expect("validated input extension");
let output = destination
.join(index.to_string())
.with_extension(extension);
fs::copy(&source, &output)
.map_err(|error| format!("cannot snapshot runtime input '{input}': {error}"))?;
verify_file(&output, &self.files[input], self.target)?;
inputs.push(output);
}
let start = inputs.drain(..self.profile.start.len()).collect();
let end = inputs.split_off(1 + self.profile.libraries.len());
Ok(LinkInputs {
start,
libraries: inputs,
end,
})
}
}
fn validate_files(
artifacts: Vec<RuntimeArtifact>,
) -> Result<BTreeMap<String, RuntimeArtifact>, String> {
if artifacts.is_empty() || artifacts.len() > 4096 {
return Err("runtime pack must declare 1..4096 files".into());
}
let mut files = BTreeMap::new();
for artifact in artifacts {
relative_path(&artifact.path)?;
if artifact.bytes == 0
|| artifact.bytes > 1024 * 1024 * 1024
|| artifact.sha256.len() != 64
|| !artifact
.sha256
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err(format!(
"invalid runtime file digest or size for '{}'",
artifact.path
));
}
if files.insert(artifact.path.clone(), artifact).is_some() {
return Err("duplicate runtime file path".into());
}
}
Ok(files)
}
fn validate_profiles(
profiles: &Profiles,
files: &BTreeMap<String, RuntimeArtifact>,
) -> Result<(), String> {
let mut referenced = BTreeSet::new();
for (_, profile) in profiles.all() {
let Some(profile) = profile else { continue };
if profile.start.is_empty() || profile.end.is_empty() || profile.libraries.is_empty() {
return Err("runtime profile requires CRT start/end and system libraries".into());
}
let mut unique = BTreeSet::new();
for input in profile.inputs() {
if !files.contains_key(input) || !unique.insert(input) {
return Err(format!(
"runtime profile has an undeclared or duplicate input '{input}'"
));
}
referenced.insert(input);
}
for (paths, extension) in [
(profile.start.as_slice(), "o"),
(profile.end.as_slice(), "o"),
(profile.libraries.as_slice(), "a"),
(std::slice::from_ref(&profile.runtime), "a"),
] {
if paths.iter().any(|path| {
Path::new(path).extension().and_then(|value| value.to_str()) != Some(extension)
}) {
return Err(
"runtime profiles accept only explicit object files and static archives".into(),
);
}
}
}
if referenced.len() != files.len() {
return Err("runtime pack contains unreferenced files".into());
}
Ok(())
}
pub(super) fn relative_path(path: &str) -> Result<(), String> {
if path.is_empty()
|| path.len() > 4096
|| path.contains(['\\', ':', '\0'])
|| path
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
{
return Err("runtime pack paths must be normalized relative paths".into());
}
Ok(())
}
pub(super) fn checked_path(root: &Path, relative: &str, file: bool) -> Result<PathBuf, String> {
relative_path(relative)?;
let mut path = root.to_owned();
let mut segments = relative.split('/').peekable();
while let Some(segment) = segments.next() {
path.push(segment);
let metadata = fs::symlink_metadata(&path).map_err(|error| {
format!("cannot inspect runtime pack '{}': {error}", path.display())
})?;
let expect_file = file && segments.peek().is_none();
if metadata.file_type().is_symlink()
|| if expect_file {
!metadata.is_file()
} else {
!metadata.is_dir()
}
{
return Err(format!(
"runtime pack '{}' must be a real {}",
path.display(),
if expect_file { "file" } else { "directory" }
));
}
}
Ok(path)
}
fn verify_file(path: &Path, artifact: &RuntimeArtifact, target: BuildTarget) -> Result<(), String> {
let length = fs::metadata(path)
.map_err(|error| format!("cannot inspect runtime file '{}': {error}", path.display()))?
.len();
if length != artifact.bytes || sha256_file(path)? != artifact.sha256 {
return Err(format!(
"runtime file '{}' failed size or SHA-256 verification",
artifact.path
));
}
let mut prefix = [0_u8; 8];
fs::File::open(path)
.and_then(|mut file| file.read_exact(&mut prefix))
.map_err(|error| format!("cannot read runtime input header: {error}"))?;
let valid = match Path::new(&artifact.path)
.extension()
.and_then(|value| value.to_str())
{
Some("a") => &prefix == b"!<arch>\n",
Some("o") => &prefix[..4] == b"\x7fELF",
_ => false,
};
if !valid {
return Err(format!(
"runtime input '{}' is not an ELF object or self-contained archive; linker scripts and thin archives are forbidden",
artifact.path
));
}
validate_target(path, length, target).map_err(|error| {
format!(
"runtime input '{}' failed target verification: {error}",
artifact.path
)
})?;
Ok(())
}
fn validate_target(path: &Path, length: u64, target: BuildTarget) -> Result<(), String> {
let mut file = fs::File::open(path).map_err(|error| error.to_string())?;
if path.extension().is_some_and(|extension| extension == "o") {
return validate_object(&mut file, length, target);
}
// Walk archive members without loading large runtime archives into memory.
let mut offset = 8_u64;
while offset < length {
file.seek(SeekFrom::Start(offset))
.map_err(|error| error.to_string())?;
let mut header = [0; 60];
file.read_exact(&mut header)
.map_err(|_| "truncated runtime archive member")?;
if &header[58..] != b"`\n" {
return Err("invalid runtime archive member header".into());
}
let size = std::str::from_utf8(&header[48..58])
.ok()
.and_then(|size| size.trim().parse::<u64>().ok())
.ok_or("invalid runtime archive member size")?;
let start = offset
.checked_add(60)
.ok_or("runtime archive size overflow")?;
let end = start
.checked_add(size)
.filter(|end| *end <= length)
.ok_or("truncated runtime archive member")?;
let name = std::str::from_utf8(&header[..16])
.map_err(|_| "invalid runtime archive member name")?
.trim();
if !matches!(
name,
"/" | "//" | "/SYM64/" | "__.SYMDEF" | "__.SYMDEF SORTED"
) {
let name_bytes = match name.strip_prefix("#1/") {
Some(bytes) => bytes
.parse::<u64>()
.map_err(|_| "invalid archive extended name")?,
None => 0,
};
let object_size = size
.checked_sub(name_bytes)
.ok_or("invalid archive extended name size")?;
file.seek(SeekFrom::Start(start + name_bytes))
.map_err(|error| error.to_string())?;
validate_object(&mut file, object_size, target)?;
}
offset = end
.checked_add(size % 2)
.filter(|end| *end <= length)
.ok_or("truncated runtime archive padding")?;
}
Ok(())
}
fn validate_object(file: &mut fs::File, length: u64, target: BuildTarget) -> Result<(), String> {
let mut header = [0; 20];
if length < header.len() as u64 {
return Err("runtime input requires a complete target ELF object header".into());
}
file.read_exact(&mut header)
.map_err(|error| format!("cannot read runtime object: {error}"))?;
let machine = match target {
BuildTarget::LinuxX86_64 => 62,
BuildTarget::LinuxAarch64 => 183,
};
if &header[..4] != b"\x7fELF"
|| header[4] != 2
|| header[5] != 1
|| header[6] != 1
|| u16::from_le_bytes([header[16], header[17]]) != 1
|| u16::from_le_bytes([header[18], header[19]]) != machine
{
return Err(format!(
"runtime input is not a relocatable ELF object for {}",
target.platform()
));
}
Ok(())
}
+732
View File
@@ -0,0 +1,732 @@
//! The small authenticated boundary around the machine cache.
//!
//! `CacheStore` remains an on-disk engine owned by this process. Clients use
//! this module's length-delimited local IPC instead of constructing the store
//! in the launcher. Platform installers can replace the fixture socket with
//! the documented system endpoint without changing the request contract.
use std::collections::BTreeSet;
#[cfg(unix)]
use std::fs::File;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
use std::path::Path;
#[cfg(unix)]
use std::sync::{Arc, Mutex, mpsc};
#[cfg(unix)]
use fs2::FileExt;
use ring::rand::{SecureRandom, SystemRandom};
use serde::{Deserialize, Serialize};
use super::cache::{ArtifactReceipt, CacheStatus, CacheStore};
use super::release::{Layout, Version};
use sha2::{Digest, Sha256};
#[cfg(unix)]
mod compile;
pub use self::compile_request::compile;
mod compile_request;
const TOKEN_BYTES: usize = 32;
const MAX_MESSAGE: usize = 1024 * 1024;
#[cfg(unix)]
const CLIENT_WORKERS: usize = 8;
#[cfg(unix)]
struct Client {
stream: std::os::unix::net::UnixStream,
caller: Option<u32>,
#[cfg(target_os = "linux")]
_admission: Option<ForeignAdmission>,
}
#[cfg(target_os = "linux")]
struct ForeignAdmission {
uid: u32,
users: Arc<Mutex<std::collections::BTreeMap<u32, usize>>>,
}
#[cfg(target_os = "linux")]
impl ForeignAdmission {
fn acquire(
users: &Arc<Mutex<std::collections::BTreeMap<u32, usize>>>,
uid: u32,
) -> Option<Self> {
let mut counts = users.lock().unwrap();
// One stalled caller cannot fill the worker pool. Two service slots
// stay available to the owner even when foreign users are saturated.
if counts.values().sum::<usize>() >= CLIENT_WORKERS - 2
|| counts.get(&uid).copied().unwrap_or(0) >= 2
{
return None;
}
*counts.entry(uid).or_default() += 1;
Some(Self {
uid,
users: Arc::clone(users),
})
}
}
#[cfg(target_os = "linux")]
impl Drop for ForeignAdmission {
fn drop(&mut self) {
let mut counts = self.users.lock().unwrap();
if let Some(count) = counts.get_mut(&self.uid) {
*count -= 1;
if *count == 0 {
counts.remove(&self.uid);
}
}
}
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Request {
#[serde(default)]
token: String,
operation: Operation,
#[serde(default)]
installed: Vec<Version>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
enum Operation {
Status,
Clean,
ArtifactPut(ArtifactReceipt),
ArtifactGet(ArtifactReceipt),
Compile { bytes: u64 },
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Response {
ok: bool,
status: Option<CacheStatus>,
error: Option<String>,
#[serde(default)]
artifact: Option<ArtifactReceipt>,
}
/// Ensure a per-machine authentication token exists. The token is not an
/// environment variable and is never accepted from a command line argument.
pub fn ensure_token(layout: &Layout) -> Result<(), String> {
if layout.service_token().exists() {
validate_token_file(layout)?;
return Ok(());
}
let mut token = [0_u8; TOKEN_BYTES];
SystemRandom::new()
.fill(&mut token)
.map_err(|_| "cannot generate deverd authentication token".to_owned())?;
let mut options = OpenOptions::new();
options.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let mut file = options
.open(layout.service_token())
.map_err(|error| format!("cannot create deverd authentication token: {error}"))?;
file.write_all(hex(&token).as_bytes())
.and_then(|_| file.write_all(b"\n"))
.and_then(|_| file.sync_all())
.map_err(|error| format!("cannot write deverd authentication token: {error}"))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(layout.service_token(), fs::Permissions::from_mode(0o600))
.map_err(|error| format!("cannot protect deverd authentication token: {error}"))?;
}
Ok(())
}
/// Serve requests until the process is stopped. This is intentionally a
/// local service primitive; OS service registration and privilege adapters
/// belong to the distribution layer.
pub fn serve(layout: &Layout) -> Result<(), String> {
#[cfg(unix)]
let _service_lock = acquire_service_lock(layout)?;
ensure_token(layout)?;
#[cfg(unix)]
{
use std::os::unix::fs::FileTypeExt;
use std::os::unix::net::UnixListener;
if let Some(parent) = layout.service_socket().parent() {
fs::create_dir_all(parent)
.map_err(|error| format!("cannot create deverd state directory: {error}"))?;
}
match fs::symlink_metadata(layout.service_socket()) {
Ok(metadata)
if metadata.file_type().is_symlink() || !metadata.file_type().is_socket() =>
{
return Err("deverd IPC endpoint must be a real Unix socket".into());
}
Ok(_) => fs::remove_file(layout.service_socket())
.map_err(|error| format!("cannot replace deverd IPC endpoint: {error}"))?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(format!("cannot inspect deverd IPC endpoint: {error}")),
}
let listener = UnixListener::bind(layout.service_socket())
.map_err(|error| format!("cannot bind deverd IPC endpoint: {error}"))?;
use std::os::unix::fs::PermissionsExt;
// Linux obtains the calling uid from the kernel. The cache directories
// remain private; only the socket is usable by other machine users.
let mode = if cfg!(target_os = "linux") {
0o666
} else {
0o600
};
fs::set_permissions(layout.service_socket(), fs::Permissions::from_mode(mode))
.map_err(|error| format!("cannot protect deverd IPC endpoint: {error}"))?;
let (sender, receiver) = mpsc::sync_channel(CLIENT_WORKERS);
let receiver = Arc::new(Mutex::new(receiver));
#[cfg(target_os = "linux")]
let (owner, foreign_users) = {
use std::os::unix::fs::MetadataExt;
let owner = fs::symlink_metadata(layout.state())
.map_err(|error| format!("cannot inspect deverd state: {error}"))?
.uid();
if owner != rustix::process::geteuid().as_raw() {
return Err("deverd must run as the state directory owner".into());
}
(
owner,
Arc::new(Mutex::new(std::collections::BTreeMap::new())),
)
};
std::thread::scope(|scope| {
for _ in 0..CLIENT_WORKERS {
let receiver = Arc::clone(&receiver);
scope.spawn(move || {
loop {
let next = { receiver.lock().unwrap().recv() };
match next {
Ok(client) => {
let Client {
stream,
caller,
#[cfg(target_os = "linux")]
_admission,
} = client;
serve_client(layout, stream, caller);
}
Err(_) => break,
}
}
});
}
let result = loop {
match listener.accept() {
Ok((stream, _)) => {
#[cfg(target_os = "linux")]
let caller = match rustix::net::sockopt::socket_peercred(&stream) {
Ok(credentials) => Some(credentials.uid.as_raw()),
Err(_) => continue,
};
#[cfg(not(target_os = "linux"))]
let caller = None;
#[cfg(target_os = "linux")]
let admission = if caller != Some(owner) {
match ForeignAdmission::acquire(
&foreign_users,
caller.expect("checked kernel uid"),
) {
Some(admission) => Some(admission),
None => continue,
}
} else {
None
};
let client = Client {
stream,
caller,
#[cfg(target_os = "linux")]
_admission: admission,
};
let _ = sender.try_send(client);
}
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) => break Err(format!("deverd IPC accept failed: {error}")),
}
};
drop(sender);
result
})
}
#[cfg(not(unix))]
{
let _ = layout;
Err("deverd authenticated IPC is not implemented for this platform yet".into())
}
}
pub fn cache_status(layout: &Layout) -> Result<CacheStatus, String> {
request(layout, Operation::Status, &[]).and_then(|response| {
response.status.ok_or_else(|| {
response
.error
.unwrap_or_else(|| "deverd returned no cache status".into())
})
})
}
pub fn cache_clean(layout: &Layout, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> {
request(
layout,
Operation::Clean,
&installed.iter().cloned().collect::<Vec<_>>(),
)
.and_then(|response| {
response.status.ok_or_else(|| {
response
.error
.unwrap_or_else(|| "deverd returned no cache status".into())
})
})
}
pub fn artifact_put(layout: &Layout, bytes: &[u8]) -> Result<ArtifactReceipt, String> {
let expected = ArtifactReceipt {
sha256: hex(&Sha256::digest(bytes)),
bytes: bytes.len() as u64,
};
expected.validate()?;
#[cfg(unix)]
{
let (mut stream, token) = connect(layout)?;
write_frame(
&mut stream,
&Request {
token,
operation: Operation::ArtifactPut(expected.clone()),
installed: Vec::new(),
},
)?;
require_artifact(read_frame(&mut stream)?, &expected)?;
stream
.write_all(bytes)
.map_err(|error| format!("deverd artifact upload failed: {error}"))?;
require_artifact(read_frame(&mut stream)?, &expected)?;
Ok(expected)
}
#[cfg(not(unix))]
{
let _ = layout;
Err("deverd artifact IPC is not implemented for this platform".into())
}
}
pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>, String> {
let expected = ArtifactReceipt {
sha256: sha256.to_owned(),
bytes,
};
expected.validate()?;
#[cfg(unix)]
{
let (mut stream, token) = connect(layout)?;
write_frame(
&mut stream,
&Request {
token,
operation: Operation::ArtifactGet(expected.clone()),
installed: Vec::new(),
},
)?;
require_artifact(read_frame(&mut stream)?, &expected)?;
let mut bytes = vec![0; expected.bytes as usize];
stream
.read_exact(&mut bytes)
.map_err(|error| format!("deverd artifact download is incomplete: {error}"))?;
if hex(&Sha256::digest(&bytes)) != expected.sha256 {
return Err("deverd artifact download failed SHA-256 verification".into());
}
Ok(bytes)
}
#[cfg(not(unix))]
{
let _ = layout;
Err("deverd artifact IPC is not implemented for this platform".into())
}
}
fn require_artifact(response: Response, expected: &ArtifactReceipt) -> Result<(), String> {
if !response.ok {
return Err(response
.error
.unwrap_or_else(|| "deverd artifact request failed".into()));
}
if response.artifact.as_ref() != Some(expected) {
return Err("deverd returned a different artifact identity".into());
}
Ok(())
}
#[cfg(unix)]
fn connect(layout: &Layout) -> Result<(std::os::unix::net::UnixStream, String), String> {
use std::os::unix::fs::{FileTypeExt, MetadataExt};
use std::os::unix::net::UnixStream;
let state = fs::symlink_metadata(layout.state())
.map_err(|error| format!("cannot inspect deverd state directory: {error}"))?;
if !state.is_dir() || state.file_type().is_symlink() || state.mode() & 0o022 != 0 {
return Err("deverd state directory must not be writable by other users".into());
}
let endpoint = fs::symlink_metadata(layout.service_socket())
.map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?;
if !endpoint.file_type().is_socket() || endpoint.uid() != state.uid() {
return Err("deverd IPC endpoint does not belong to the service owner".into());
}
let stream = UnixStream::connect(layout.service_socket())
.map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?;
#[cfg(target_os = "linux")]
if rustix::net::sockopt::socket_peercred(&stream)
.map_err(|error| format!("cannot authenticate deverd OS peer: {error}"))?
.uid
.as_raw()
!= state.uid()
{
return Err("deverd OS peer does not match the service owner".into());
}
stream
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
.and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5))))
.map_err(|error| format!("cannot configure deverd IPC timeout: {error}"))?;
#[cfg(target_os = "linux")]
if rustix::process::geteuid().as_raw() != state.uid() {
return Ok((stream, String::new()));
}
validate_token_file(layout)?;
let token = String::from_utf8(read_file(&layout.service_token())?)
.map_err(|_| "invalid deverd authentication token".to_owned())?;
Ok((stream, token.trim().to_owned()))
}
fn request(
layout: &Layout,
operation: Operation,
installed: &[Version],
) -> Result<Response, String> {
#[cfg(unix)]
{
let (mut stream, token) = connect(layout)?;
let request = Request {
token,
operation,
installed: installed.to_owned(),
};
write_frame(&mut stream, &request)?;
read_frame(&mut stream)
}
#[cfg(not(unix))]
{
let _ = (layout, operation, installed);
Err(
"authenticated deverd cache service unavailable: platform IPC is not implemented"
.into(),
)
}
}
#[cfg(unix)]
fn handle(
layout: &Layout,
stream: &mut std::os::unix::net::UnixStream,
caller: Option<u32>,
) -> Response {
let request: Result<Request, _> = read_frame(stream);
let request = match request {
Ok(request) => request,
Err(error) => return failure(error),
};
if let Err(error) = validate_token_file(layout) {
return failure(error);
}
let bytes = match read_file(&layout.service_token()) {
Ok(bytes) => bytes,
Err(error) => return failure(error),
};
let expected = match String::from_utf8(bytes) {
Ok(token) => token.trim().as_bytes().to_vec(),
Err(_) => return failure("invalid deverd authentication token".into()),
};
#[cfg(target_os = "linux")]
let foreign = match caller {
Some(uid) => {
use std::os::unix::fs::MetadataExt;
match fs::symlink_metadata(layout.state()) {
Ok(state) => uid != state.uid(),
Err(error) => return failure(format!("cannot inspect deverd state: {error}")),
}
}
None => return failure("deverd OS caller credentials are unavailable".into()),
};
#[cfg(not(target_os = "linux"))]
let foreign = {
let _ = caller;
false
};
if foreign {
if matches!(request.operation, Operation::Clean) {
return failure("deverd cache clean requires the service owner".into());
}
} else if !constant_time_equal(request.token.as_bytes(), &expected) {
return failure("deverd authentication failed".into());
}
let store = CacheStore::new(layout);
let result = match request.operation {
Operation::Status => {
if foreign {
store.summary()
} else {
store.status()
}
}
Operation::Clean => {
if request.installed.len() > 256 {
return failure("too many installed Dever versions".into());
}
store.clean(&request.installed.into_iter().collect())
}
Operation::ArtifactPut(expected) => {
if let Err(error) = expected.validate() {
return failure(error);
}
if let Err(error) = write_frame(stream, &artifact_response(expected.clone())) {
return failure(error);
}
let mut input = TransferReader {
input: stream,
started: std::time::Instant::now(),
};
return match store.publish_artifact(&expected, &mut input) {
Ok(receipt) => artifact_response(receipt),
Err(error) => failure(error),
};
}
Operation::ArtifactGet(expected) => {
let mut artifact = match store.artifact(&expected) {
Ok(artifact) => artifact,
Err(error) => return failure(error),
};
if let Err(error) = write_frame(stream, &artifact_response(artifact.receipt)) {
return failure(error);
}
if let Err(error) = std::io::copy(&mut artifact.file, stream) {
return failure(format!("deverd artifact transfer failed: {error}"));
}
return Response {
ok: true,
status: None,
artifact: None,
error: None,
};
}
Operation::Compile { bytes } => {
return match compile::serve(layout, stream, bytes) {
Ok(()) => Response {
ok: true,
status: None,
artifact: None,
error: None,
},
Err(error) => failure(error),
};
}
};
match result {
Ok(status) => Response {
ok: true,
status: Some(status),
error: None,
artifact: None,
},
Err(error) => failure(error),
}
}
fn failure(error: String) -> Response {
Response {
ok: false,
status: None,
error: Some(error),
artifact: None,
}
}
fn artifact_response(artifact: ArtifactReceipt) -> Response {
Response {
ok: true,
status: None,
error: None,
artifact: Some(artifact),
}
}
struct TransferReader<'a, R> {
input: &'a mut R,
started: std::time::Instant,
}
impl<R: Read> Read for TransferReader<'_, R> {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
if self.started.elapsed() >= std::time::Duration::from_secs(60) {
return Err(std::io::Error::new(
std::io::ErrorKind::TimedOut,
"deverd artifact transfer deadline exceeded",
));
}
self.input.read(buffer)
}
}
#[cfg(unix)]
fn serve_client(layout: &Layout, mut stream: std::os::unix::net::UnixStream, caller: Option<u32>) {
if stream
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
.and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5))))
.is_err()
{
return;
}
let response = handle(layout, &mut stream, caller);
let _ = write_frame(&mut stream, &response);
}
#[cfg(unix)]
fn acquire_service_lock(layout: &Layout) -> Result<File, String> {
use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
let path = layout.state().join("deverd.lock");
let mut options = OpenOptions::new();
options.read(true).write(true);
match fs::symlink_metadata(&path) {
Ok(_) => {
validate_owned_file(layout, &path, "deverd lock")?;
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
options.create_new(true).mode(0o600);
}
Err(error) => return Err(format!("cannot inspect deverd lock: {error}")),
}
let file = options
.open(&path)
.map_err(|error| format!("cannot open deverd lock: {error}"))?;
let opened = file
.metadata()
.map_err(|error| format!("cannot inspect deverd lock: {error}"))?;
let current = validate_owned_file(layout, &path, "deverd lock")?;
if opened.dev() != current.dev() || opened.ino() != current.ino() {
return Err("deverd lock changed while opening".into());
}
FileExt::try_lock_exclusive(&file).map_err(|error| {
if error.kind() == std::io::ErrorKind::WouldBlock {
"deverd is already running for this machine root".into()
} else {
format!("cannot lock deverd state: {error}")
}
})?;
Ok(file)
}
fn validate_token_file(layout: &Layout) -> Result<(), String> {
let path = layout.service_token();
validate_owned_file(layout, &path, "deverd authentication token")?;
let token = read_file(&path)?;
let token = token.strip_suffix(b"\n").unwrap_or(&token);
if token.len() != TOKEN_BYTES * 2
|| !token
.iter()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
{
return Err("invalid deverd authentication token".into());
}
Ok(())
}
fn validate_owned_file(layout: &Layout, path: &Path, name: &str) -> Result<fs::Metadata, String> {
let metadata =
fs::symlink_metadata(path).map_err(|error| format!("cannot inspect {name}: {error}"))?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
return Err(format!("{name} must be a regular file"));
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let state = fs::symlink_metadata(layout.state())
.map_err(|error| format!("cannot inspect deverd state directory: {error}"))?;
if !state.is_dir() || state.uid() != metadata.uid() {
return Err(format!("{name} must have the state directory owner"));
}
if metadata.mode() & 0o077 != 0 {
return Err(format!("{name} must be accessible only to its owner"));
}
}
Ok(metadata)
}
fn write_frame<T: Serialize>(stream: &mut impl Write, value: &T) -> Result<(), String> {
let payload =
serde_json::to_vec(value).map_err(|error| format!("deverd IPC encode failed: {error}"))?;
if payload.len() > MAX_MESSAGE {
return Err("deverd IPC message is too large".into());
}
stream
.write_all(&(payload.len() as u32).to_be_bytes())
.and_then(|_| stream.write_all(&payload))
.and_then(|_| stream.flush())
.map_err(|error| format!("deverd IPC write failed: {error}"))
}
fn read_frame<T: for<'de> Deserialize<'de>>(stream: &mut impl Read) -> Result<T, String> {
let mut length = [0_u8; 4];
stream
.read_exact(&mut length)
.map_err(|error| format!("deverd IPC read failed: {error}"))?;
let length = u32::from_be_bytes(length) as usize;
if length > MAX_MESSAGE {
return Err("deverd IPC message is too large".into());
}
let mut payload = vec![0; length];
stream
.read_exact(&mut payload)
.map_err(|error| format!("deverd IPC read failed: {error}"))?;
let text =
std::str::from_utf8(&payload).map_err(|_| "deverd IPC payload must be UTF-8 JSON")?;
dever_runtime::wire::parse(text)
.map_err(|error| format!("deverd IPC decode failed: {error}"))?;
serde_json::from_slice(&payload).map_err(|error| format!("deverd IPC decode failed: {error}"))
}
fn read_file(path: &Path) -> Result<Vec<u8>, String> {
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
}
fn constant_time_equal(left: &[u8], right: &[u8]) -> bool {
if left.len() != right.len() {
return false;
}
let mut difference = 0_u8;
for (left, right) in left.iter().zip(right) {
difference |= left ^ right;
}
difference == 0
}
fn hex(bytes: &[u8]) -> String {
const DIGITS: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
output.push(DIGITS[usize::from(byte >> 4)] as char);
output.push(DIGITS[usize::from(byte & 0x0f)] as char);
}
output
}
@@ -0,0 +1,273 @@
//! Only this service path may publish compiled entries. Artifact uploads remain
//! opaque bytes and can never satisfy a compilation lookup.
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Write};
use std::os::unix::net::UnixStream;
use std::path::Path;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use super::super::cache::{ArtifactReceipt, BuildIdentity, CacheStore, MAX_COMPILED_BYTES};
use super::super::compilation::{
CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE,
};
use super::super::release::{InstalledCompiler, Layout, MachineManager, Version};
use super::{Response, artifact_response, hex, write_frame};
const COMPILE_TIMEOUT: Duration = Duration::from_secs(180);
const MAX_DIAGNOSTIC_BYTES: u64 = 64 * 1024;
static ACTIVE_COMPILATIONS: AtomicUsize = AtomicUsize::new(0);
struct Admission;
impl Admission {
fn acquire() -> Result<Self, String> {
ACTIVE_COMPILATIONS
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {
(count < 2).then_some(count + 1)
})
.map_err(|_| "deverd compilation capacity is busy; retry later".to_owned())?;
Ok(Self)
}
}
impl Drop for Admission {
fn drop(&mut self) {
ACTIVE_COMPILATIONS.fetch_sub(1, Ordering::Release);
}
}
pub(super) fn serve(layout: &Layout, stream: &mut UnixStream, bytes: u64) -> Result<(), String> {
if bytes == 0 || bytes > MAX_COMPILE_BYTES as u64 {
return Err("deverd compilation request exceeds its transfer limit".into());
}
let _admission = Admission::acquire()?;
let started = Instant::now();
let store = CacheStore::new(layout);
let _operation = store.operation()?;
write_frame(
stream,
&Response {
ok: true,
status: None,
artifact: None,
error: None,
},
)?;
let mut payload = vec![0; bytes as usize];
super::TransferReader {
input: stream,
started,
}
.read_exact(&mut payload)
.map_err(|error| format!("incomplete compilation request: {error}"))?;
let request = CompileRequest::decode(&payload)?;
let version = Version::parse(&request.version)?;
let target = request.target;
// Canonical bytes own the cache identity; decoded interpreter copies are no
// longer needed by the service while the worker rereads its private request.
drop(request);
let compiler = MachineManager::new(layout.clone()).compilation(&version)?;
verify_pack(&compiler, target)?;
let identity = BuildIdentity {
version: &version,
compiler: compiler.identity.as_bytes(),
runtime: compiler.identity.as_bytes(),
target: target.platform(),
generated: &payload,
options: b"dever-trusted-compilation-v1",
environment: b"",
};
stream
.set_read_timeout(Some(Duration::from_millis(1)))
.map_err(|error| format!("cannot monitor compilation caller: {error}"))?;
require_connected(stream, started)?;
if let Some(artifact) = store.restore(&identity)? {
return send_output(stream, artifact.as_bytes(), started);
}
let directory = store.compilation_directory()?;
write_private(&directory.0.join(WORKER_REQUEST_FILE), &payload)?;
run_worker(&compiler.core, &directory.0, stream, started)?;
require_connected(stream, started)?;
let output = directory.0.join(WORKER_OUTPUT_FILE);
let metadata = fs::symlink_metadata(&output)
.map_err(|error| format!("compiler produced no executable: {error}"))?;
if !metadata.is_file()
|| metadata.file_type().is_symlink()
|| metadata.len() == 0
|| metadata.len() > MAX_COMPILED_BYTES
{
return Err("compiler output must be a regular executable of at most 320 MiB".into());
}
store.publish(&identity, &output)?;
let artifact = store
.restore(&identity)?
.ok_or("compiled output was not published")?;
send_output(stream, artifact.as_bytes(), started)
}
fn verify_pack(
compiler: &InstalledCompiler,
target: super::super::BuildTarget,
) -> Result<(), String> {
let root = compiler
.core
.parent()
.ok_or("installed compiler has no directory")?;
let artifacts =
super::super::runtime_pack::validate(root, compiler.manifest.version.as_str(), target)?;
for input in artifacts {
if !compiler.manifest.artifacts.iter().any(|signed| {
signed.path == input.path
&& signed.bytes == input.bytes
&& signed.sha256 == input.sha256
}) {
return Err(format!(
"native runtime input '{}' is not covered by the signed release",
input.path
));
}
}
// This is the embedded compiler's fixed Linux LLVM dependency, not a PATH lookup.
if cfg!(target_os = "linux")
&& !compiler
.manifest
.artifacts
.iter()
.any(|artifact| artifact.path == "lib/libLLVM.so.18.1")
{
return Err("signed release is missing lib/libLLVM.so.18.1".into());
}
Ok(())
}
fn send_output(stream: &mut UnixStream, bytes: &[u8], started: Instant) -> Result<(), String> {
let receipt = ArtifactReceipt {
sha256: hex(ring::digest::digest(&ring::digest::SHA256, bytes).as_ref()),
bytes: bytes.len() as u64,
};
receipt.validate_compiled()?;
let timeout = || {
COMPILE_TIMEOUT
.checked_sub(started.elapsed())
.filter(|remaining| !remaining.is_zero())
.map(|remaining| remaining.min(Duration::from_secs(5)))
.ok_or_else(|| "deverd compilation transfer deadline exceeded".to_owned())
};
stream
.set_write_timeout(Some(timeout()?))
.map_err(|error| error.to_string())?;
write_frame(stream, &artifact_response(receipt))?;
let mut remaining = bytes;
while !remaining.is_empty() {
stream
.set_write_timeout(Some(timeout()?))
.map_err(|error| error.to_string())?;
match stream.write(&remaining[..remaining.len().min(64 * 1024)]) {
Ok(0) => return Err("compiled output transfer closed".into()),
Ok(written) => remaining = &remaining[written..],
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
Err(error) => return Err(format!("compiled output transfer failed: {error}")),
}
}
Ok(())
}
fn require_connected(stream: &mut UnixStream, started: Instant) -> Result<(), String> {
if started.elapsed() >= COMPILE_TIMEOUT {
return Err("deverd compilation deadline exceeded".into());
}
let mut signal = [0_u8; 1];
match stream.read(&mut signal) {
// Any byte after the exact request body explicitly cancels this request.
Ok(_) => Err("deverd compilation cancelled by caller".into()),
Err(error)
if matches!(
error.kind(),
std::io::ErrorKind::WouldBlock
| std::io::ErrorKind::TimedOut
| std::io::ErrorKind::Interrupted
) =>
{
Ok(())
}
Err(error) => Err(format!("deverd compilation caller disconnected: {error}")),
}
}
struct Worker(Child);
impl Drop for Worker {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
fn run_worker(
core: &Path,
directory: &Path,
stream: &mut UnixStream,
started: Instant,
) -> Result<(), String> {
let diagnostic = directory.join("diagnostic");
let error_output = private_file(&diagnostic)?;
let mut worker = Worker(
Command::new(core)
.arg("--dever-compile-worker")
.env_clear()
.current_dir(directory)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(error_output)
.spawn()
.map_err(|error| format!("cannot start trusted compile worker: {error}"))?,
);
loop {
require_connected(stream, started)?;
if fs::metadata(&diagnostic)
.map_err(|error| error.to_string())?
.len()
> MAX_DIAGNOSTIC_BYTES
{
return Err("compile worker diagnostic exceeded 64 KiB".into());
}
if let Some(status) = worker
.0
.try_wait()
.map_err(|error| format!("cannot wait for compile worker: {error}"))?
{
if status.success() {
return Ok(());
}
let mut bytes = Vec::new();
File::open(&diagnostic)
.map_err(|error| error.to_string())?
.take(MAX_DIAGNOSTIC_BYTES)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
return Err(format!(
"trusted compile worker failed ({status}): {}",
String::from_utf8_lossy(&bytes)
));
}
std::thread::sleep(Duration::from_millis(10));
}
}
fn private_file(path: &Path) -> Result<File, String> {
use std::os::unix::fs::OpenOptionsExt;
OpenOptions::new()
.write(true)
.create_new(true)
.mode(0o600)
.open(path)
.map_err(|error| format!("cannot create compilation input: {error}"))
}
fn write_private(path: &Path, bytes: &[u8]) -> Result<(), String> {
private_file(path)?
.write_all(bytes)
.map_err(|error| format!("cannot write compilation input: {error}"))
}
@@ -0,0 +1,87 @@
use super::super::compilation::CompileRequest;
use super::super::release::Layout;
/// Submit source and compile-only bindings, never a client-chosen cache key or IR.
pub fn compile(layout: &Layout, request: &CompileRequest) -> Result<Vec<u8>, String> {
#[cfg(unix)]
{
use super::{Operation, Request, Response, connect, read_frame, write_frame};
use std::io::{Read, Write};
use std::time::{Duration, Instant};
let payload = request.encode()?;
let (mut stream, token) = connect(layout)?;
write_frame(
&mut stream,
&Request {
token,
operation: Operation::Compile {
bytes: payload.len() as u64,
},
installed: Vec::new(),
},
)?;
let ready: Response = read_frame(&mut stream)?;
if !ready.ok {
return Err(ready
.error
.unwrap_or_else(|| "deverd rejected compilation".into()));
}
stream
.write_all(&payload)
.map_err(|error| format!("deverd compilation upload failed: {error}"))?;
drop(payload);
let mut download = CompilationDownload {
stream: &mut stream,
deadline: Instant::now() + Duration::from_secs(190),
};
let response: Response = read_frame(&mut download)?;
if !response.ok {
return Err(response
.error
.unwrap_or_else(|| "deverd compilation failed".into()));
}
let receipt = response
.artifact
.ok_or("deverd returned no compiled output identity")?;
receipt.validate_compiled()?;
let mut output = vec![0; receipt.bytes as usize];
download
.read_exact(&mut output)
.map_err(|error| format!("deverd compilation download is incomplete: {error}"))?;
if super::hex(ring::digest::digest(&ring::digest::SHA256, &output).as_ref())
!= receipt.sha256
{
return Err("deverd compiled output failed SHA-256 verification".into());
}
Ok(output)
}
#[cfg(not(unix))]
{
let _ = (layout, request);
Err("trusted deverd compilation is not implemented for this platform".into())
}
}
#[cfg(unix)]
struct CompilationDownload<'a> {
stream: &'a mut std::os::unix::net::UnixStream,
deadline: std::time::Instant,
}
#[cfg(unix)]
impl std::io::Read for CompilationDownload<'_> {
fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> {
let remaining = self
.deadline
.checked_duration_since(std::time::Instant::now())
.filter(|duration| !duration.is_zero())
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::TimedOut,
"compiled output download deadline exceeded",
)
})?;
self.stream.set_read_timeout(Some(remaining))?;
self.stream.read(bytes)
}
}
+91
View File
@@ -0,0 +1,91 @@
//! A stable AI entry resolves the same active-version journal as the launcher.
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use super::release::MachineManager;
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
pub(super) fn path(manager: &MachineManager) -> Result<PathBuf, String> {
let version = manager
.active_version()?
.ok_or("no active Dever version; install Dever first")?;
// Like launcher dispatch, this is read-only and available to every machine user.
manager.resolve_skill(&version)
}
pub(super) fn install(manager: &MachineManager, destination: &Path) -> Result<(), String> {
path(manager)?;
let launcher = manager.layout().bin().join("dever");
let launcher = launcher
.to_str()
.ok_or("launcher path must be UTF-8 for an AI skill")?;
if launcher.chars().any(char::is_control) || launcher.contains('`') {
return Err("launcher path cannot contain control characters or backticks".into());
}
let quoted = format!("'{}'", launcher.replace('\'', "'\\''"));
let content = format!(
"---\nname: dever-language\ndescription: Develop, review and debug Dever language applications (.dever and .dever.md), manage Dever Packages and Libs, and install or update Dever. Not the Go Dever framework.\n---\n\n# Dever language\n\nBefore every Dever task, run this read-only command:\n\n```sh\n{quoted} skill path\n```\n\nRead `SKILL.md` in the returned directory completely, then follow its instructions. Resolve its reference and asset paths against that directory. This entry follows the machine's active Dever version, so `dever update` and `dever use` update the compiler and development guidance together. If resolution fails, report the error; do not use stale guidance or infer language syntax. This entry grants no permission to install, update or change a project.\n"
);
let name = destination
.file_name()
.ok_or("skill destination requires a new directory name")?;
let parent = destination
.parent()
.filter(|path| !path.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let parent = fs::canonicalize(parent)
.map_err(|error| format!("cannot resolve skill destination parent: {error}"))?;
let destination = parent.join(name);
let staging = stage(&parent)?;
let entry = staging.join("SKILL.md");
let result = (|| {
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&entry)
.map_err(|error| format!("cannot create skill entry: {error}"))?;
file.write_all(content.as_bytes())
.and_then(|()| file.sync_all())
.map_err(|error| format!("cannot persist skill entry: {error}"))?;
#[cfg(target_os = "linux")]
{
use rustix::fs::{CWD, RenameFlags, renameat_with};
renameat_with(CWD, &staging, CWD, &destination, RenameFlags::NOREPLACE)
.map_err(|error| format!("cannot publish new skill directory: {error}"))
}
#[cfg(not(target_os = "linux"))]
{
Err("skill installation is not available for this platform".into())
}
})();
if result.is_err() {
let _ = fs::remove_file(&entry);
let _ = fs::remove_dir(&staging);
}
result
}
fn stage(parent: &Path) -> Result<PathBuf, String> {
loop {
let path = parent.join(format!(
".dever-skill-{}-{}",
std::process::id(),
NEXT_STAGING.fetch_add(1, Ordering::Relaxed)
));
let mut builder = fs::DirBuilder::new();
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o700);
}
match builder.create(&path) {
Ok(()) => return Ok(path),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(format!("cannot stage skill entry: {error}")),
}
}
}
+50
View File
@@ -0,0 +1,50 @@
//! Application deployment targets, independent of the compiler host identity.
use serde::{Deserialize, Serialize};
use std::str::FromStr;
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)]
pub enum BuildTarget {
#[serde(rename = "linux-x86_64")]
LinuxX86_64,
#[serde(rename = "linux-aarch64")]
LinuxAarch64,
}
impl BuildTarget {
pub fn host() -> Result<Self, String> {
match (std::env::consts::OS, std::env::consts::ARCH) {
("linux", "x86_64") => Ok(Self::LinuxX86_64),
("linux", "aarch64") => Ok(Self::LinuxAarch64),
_ => Err("native executable runtime packs are not available for this platform".into()),
}
}
pub fn platform(self) -> &'static str {
match self {
Self::LinuxX86_64 => "linux-x86_64",
Self::LinuxAarch64 => "linux-aarch64",
}
}
pub fn triple(self) -> &'static str {
match self {
Self::LinuxX86_64 => "x86_64-unknown-linux-gnu",
Self::LinuxAarch64 => "aarch64-unknown-linux-gnu",
}
}
}
impl FromStr for BuildTarget {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"linux-x86_64" => Ok(Self::LinuxX86_64),
"linux-aarch64" => Ok(Self::LinuxAarch64),
_ => Err(format!(
"unsupported build target '{value}'; expected linux-x86_64 or linux-aarch64"
)),
}
}
}
+97
View File
@@ -0,0 +1,97 @@
//! Hidden source compiler. It never prepares dependencies or executes a project.
use dever_cli::toolchain::compilation::{
CompileKind, CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE,
worker_directory,
};
use std::fs;
use std::io::Read;
pub(super) fn execute() -> Result<(), String> {
resource_limits()?;
let (layout, version) = crate::compile::managed_compiler()?
.ok_or("compile workers require an installed managed compiler")?;
let directory = worker_directory(&layout)?;
let path = directory.join(WORKER_REQUEST_FILE);
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("cannot inspect compile request: {error}"))?;
if !metadata.is_file()
|| metadata.file_type().is_symlink()
|| metadata.len() > MAX_COMPILE_BYTES as u64
{
return Err("compile request must be a bounded regular file".into());
}
let mut bytes = Vec::new();
fs::File::open(&path)
.and_then(|file| {
file.take(MAX_COMPILE_BYTES as u64 + 1)
.read_to_end(&mut bytes)
})
.map_err(|error| format!("cannot read compile request: {error}"))?;
let request = CompileRequest::decode(&bytes)?;
drop(bytes);
if request.version != version {
return Err("compile request version does not match worker".into());
}
let sources = request.source_map()?;
let checked = match &request.bindings {
Some(bindings) => dever_core::check_with_bindings(&sources, bindings),
None => dever_core::check(&sources),
};
let render = |errors: Vec<dever_core::diagnostic::Diagnostic>| {
errors
.iter()
.map(|error| error.render(&sources))
.collect::<String>()
};
let program = checked.map_err(render)?;
if request.kind == CompileKind::Application {
if request.sources.iter().any(|source| {
source
.logical_path
.rsplit('/')
.next()
.is_some_and(|name| name == "main.dever" || name == "main.dever.md")
}) {
return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into());
}
let errors = program.application_errors()?;
if !errors.is_empty() {
return Err(render(errors));
}
}
crate::compile::worker_compile(
&program,
&sources,
&request,
&directory,
&directory.join(WORKER_OUTPUT_FILE),
)
}
#[cfg(unix)]
fn resource_limits() -> Result<(), String> {
use rustix::process::{Resource, Rlimit, setrlimit};
for (resource, limit) in [
(Resource::Cpu, 120),
(Resource::As, 2 * 1024 * 1024 * 1024),
(Resource::Fsize, 1024 * 1024 * 1024),
(Resource::Nofile, 128),
(Resource::Core, 0),
] {
setrlimit(
resource,
Rlimit {
current: Some(limit),
maximum: Some(limit),
},
)
.map_err(|error| format!("cannot restrict compiler worker resources: {error}"))?;
}
Ok(())
}
#[cfg(not(unix))]
fn resource_limits() -> Result<(), String> {
Err("managed compiler worker resource limits are unavailable on this platform".into())
}
+833
View File
@@ -0,0 +1,833 @@
//! Offline, per-Adapter execution trees for managed external Workers.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use std::path::{Component, Path};
use dever_core::hir::{ExternalWorkerContract, Program};
use dever_core::native::EmbeddedResource;
use serde_json::json;
use sha2::{Digest, Sha256};
use crate::libs::{self, Ecosystem, LibSpec, LockFile, LockedLib};
mod command;
mod elf;
mod go_build;
pub mod python_wheel;
pub(crate) mod runtime;
pub(crate) const MAX_TREE_BYTES: usize = 256 * 1024 * 1024;
pub(crate) const MAX_TREE_FILES: usize = 65_536;
const IGNORED_SOURCE_DIRECTORIES: &[&str] = &["node_modules", "__pycache__", ".git", ".dever"];
const PYTHON_SDK: &[u8] = include_bytes!("../../../sdk/python/dever_component.py");
const JAVASCRIPT_SDK: &[u8] = include_bytes!("../../../sdk/javascript/dever_component.mjs");
const PYTHON_RUNNER: &str = r#"import importlib.util
import json
from pathlib import Path
import sys
base = Path(__file__).resolve().parent
sys.stdout = sys.stderr
sys.path.insert(0, str(base))
from dever_component import Worker, main
sys.path.insert(0, str(base / "runtime" / "lib" / "python__PYTHON_VERSION__" / "site-packages"))
sys.path.insert(0, str(base / "source"))
spec = importlib.util.spec_from_file_location("dever_worker_entry", base / "source" / "__ENTRY__")
if spec is None or spec.loader is None:
raise RuntimeError("managed Worker entry cannot be loaded")
entry = importlib.util.module_from_spec(spec)
spec.loader.exec_module(entry)
manifest = json.loads((base / "contract.json").read_text(encoding="utf-8"))
handlers = {operation: getattr(entry, operation) for operation in manifest["operations"]}
main(Worker.from_manifest(manifest, handlers))
"#;
const JAVASCRIPT_RUNNER: &str = r#"import { Console } from 'node:console';
import { readFileSync } from 'node:fs';
import { Worker, main } from './dever_component.mjs';
globalThis.console = new Console({ stdout: process.stderr, stderr: process.stderr });
const manifest = JSON.parse(readFileSync(new URL('./contract.json', import.meta.url), 'utf8'));
const entry = await import(new URL('./source/__ENTRY__', import.meta.url));
const handlers = Object.fromEntries(manifest.operations.map(operation => [operation, entry[operation] ?? entry.default?.[operation]]));
await main(Worker.fromManifest(manifest, handlers));
"#;
pub fn prepare(
project_root: &Path,
program: &Program,
resources: &[EmbeddedResource],
) -> Result<Vec<EmbeddedResource>, String> {
prepare_for_target(
project_root,
program,
resources,
crate::toolchain::BuildTarget::host()?,
resources,
)
}
pub fn prepare_for_target(
project_root: &Path,
program: &Program,
resources: &[EmbeddedResource],
target: crate::toolchain::BuildTarget,
build_assets: &[EmbeddedResource],
) -> Result<Vec<EmbeddedResource>, String> {
let workers = program.external_worker_contracts();
let commands = program.external_command_contracts();
if workers.is_empty() && commands.is_empty() {
return Ok(Vec::new());
}
let lock = if workers.iter().any(|worker| worker.ecosystem != "exec") {
let lock_path = project_root.join("dever.lock");
if fs::symlink_metadata(&lock_path)
.map_err(|error| format!("cannot inspect dever.lock: {error}"))?
.file_type()
.is_symlink()
{
return Err("managed Worker lock file must not be a symbolic link".into());
}
let lock = LockFile::decode(
&fs::read(&lock_path).map_err(|error| format!("cannot read dever.lock: {error}"))?,
)?;
libs::doctor(&lock)?;
Some(lock)
} else {
None
};
let mut supplied = BTreeMap::new();
for resource in resources {
if supplied.insert(resource.path.as_str(), resource).is_some() {
return Err(format!(
"duplicate locked Worker resource '{}'",
resource.path
));
}
if digest(&resource.bytes) != resource.sha256 {
return Err(format!(
"locked Worker resource '{}' has a digest mismatch",
resource.path
));
}
}
let mut output = Vec::new();
let package_files = crate::packages::owned_files(project_root)?;
for contract in commands {
output.extend(command::prepare(
project_root,
&contract,
&package_files,
&supplied,
target,
)?);
}
for contract in workers {
if contract.ecosystem == "exec" {
output.extend(prepare_exec(&contract, &supplied, target)?);
continue;
}
let lock = lock.as_ref().expect("managed Worker has a lock");
let locked = lock
.workers
.iter()
.find(|worker| worker.port == contract.port && worker.adapter == contract.adapter)
.ok_or_else(|| {
format!(
"managed Worker '{}.{}' is not locked",
contract.port, contract.adapter
)
})?;
if locked.entry != contract.entry
|| locked.schema != contract.schema
|| locked.ecosystem != contract.ecosystem
|| locked.operations != contract.operations
|| locked.capabilities != contract.capabilities
{
return Err(format!(
"managed Worker '{}.{}' differs from its checked Port",
contract.port, contract.adapter
));
}
let requested = contract
.libs
.iter()
.map(|value| value.parse::<LibSpec>())
.collect::<Result<BTreeSet<_>, _>>()?;
if requested != locked.libs.iter().cloned().collect() {
return Err("managed Worker Lib roots differ from its checked Adapter".into());
}
output.extend(prepare_worker(
project_root,
&contract,
lock,
&supplied,
&package_files,
target,
build_assets,
)?);
}
Ok(output)
}
fn prepare_exec(
contract: &ExternalWorkerContract,
supplied: &BTreeMap<&str, &EmbeddedResource>,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<EmbeddedResource>, String> {
let entry = supplied
.get(contract.entry.as_str())
.ok_or("exec Worker is missing from verified resources")?;
if !entry.executable || entry.bytes.is_empty() {
return Err("exec Worker entry must be a nonempty executable resource".into());
}
runtime::validate_binary(&contract.entry, &entry.bytes, target.platform())?;
let identity =
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?;
let tree = format!("workers/{}", digest(&identity));
let executable = format!("{tree}/worker");
let manifest = json!({
"format": "dever-worker-launch-v1", "ecosystem": "exec", "entry": contract.entry,
"executable": executable, "arguments": [], "working_directory": tree,
});
Ok(vec![
resource(executable, entry.bytes.clone(), true),
resource(
format!("{}.dever-worker.json", contract.entry),
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
false,
),
])
}
fn prepare_worker(
project_root: &Path,
contract: &ExternalWorkerContract,
lock: &LockFile,
supplied: &BTreeMap<&str, &EmbeddedResource>,
package_files: &BTreeMap<String, Vec<u8>>,
target: crate::toolchain::BuildTarget,
build_assets: &[EmbeddedResource],
) -> Result<Vec<EmbeddedResource>, String> {
let ecosystem: Ecosystem = contract.ecosystem.parse()?;
let identity =
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?;
let tree = format!("workers/{}", digest(&identity));
let target = target.platform();
let runtime_path = format!("lib/runtime/{}/{target}/runtime.pack", ecosystem.as_str());
let runtime = supplied
.get(runtime_path.as_str())
.ok_or_else(|| format!("managed {} runtime pack is missing", ecosystem.as_str()))?;
let locked_worker = lock
.workers
.iter()
.find(|worker| worker.port == contract.port && worker.adapter == contract.adapter)
.expect("validated Worker lock identity");
let locked_runtime = locked_worker
.runtime
.as_ref()
.ok_or("managed Worker lacks a locked runtime pack")?;
if runtime.sha256 != locked_runtime.sha256 {
return Err("managed Worker runtime pack differs from its lock".into());
}
let files = libs::archive_files("tgz", &runtime.bytes)?;
let manifest = runtime::validate(&files, &ecosystem, target)?;
let mut tree_files = Tree::new(tree.clone());
let runtime_executable = if ecosystem == Ecosystem::Go {
None
} else {
let executable = manifest
.executable
.as_deref()
.expect("validated interpreter executable");
for (path, bytes) in &files {
tree_files.add(
&format!("runtime/{path}"),
bytes.clone(),
path == executable,
)?;
}
let path = format!("{tree}/runtime/{executable}");
Some(path)
};
add_adapter_source(project_root, contract, package_files, &mut tree_files)?;
let runner = match ecosystem {
Ecosystem::Pip => {
add_python_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?;
tree_files.add("dever_component.py", PYTHON_SDK.to_vec(), false)?;
tree_files.add(
"runner.py",
PYTHON_RUNNER
.replace("__ENTRY__", entry_name(&contract.entry)?)
.replace(
"__PYTHON_VERSION__",
&manifest
.python_markers
.as_ref()
.expect("validated Python markers")
.python_version()
.to_string(),
)
.into_bytes(),
false,
)?;
"runner.py"
}
Ecosystem::Npm => {
ensure_javascript_format(&contract.entry, &mut tree_files)?;
add_javascript_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?;
tree_files.add("dever_component.mjs", JAVASCRIPT_SDK.to_vec(), false)?;
tree_files.add(
"runner.mjs",
JAVASCRIPT_RUNNER
.replace("__ENTRY__", entry_name(&contract.entry)?)
.into_bytes(),
false,
)?;
"runner.mjs"
}
Ecosystem::Go => {
tree_files.add("contract.json", identity.clone(), false)?;
let binary = go_build::compile_worker(
contract,
lock,
supplied,
go_build::Inputs {
files: &files,
manifest: manifest
.build
.as_ref()
.expect("validated Go build descriptor"),
sandbox: build_assets,
},
&mut tree_files,
target,
)?;
tree_files.remove_prefix("source/");
let launch = json!({
"format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem,
"entry": contract.entry, "executable": format!("{tree}/worker{}", std::env::consts::EXE_SUFFIX),
"arguments": [], "working_directory": tree,
});
let mut output = tree_files.finish();
output.push(resource(
format!("{}.dever-worker.json", contract.entry),
serde_json::to_vec(&launch).map_err(|error| error.to_string())?,
false,
));
debug_assert_eq!(binary, format!("worker{}", std::env::consts::EXE_SUFFIX));
return Ok(output);
}
};
tree_files.add("contract.json", identity, false)?;
let runner_path = format!("{tree}/{runner}");
let arguments = manifest
.arguments
.iter()
.map(|value| json!({"literal": value}))
.chain(std::iter::once(json!({"resource": runner_path})))
.collect::<Vec<_>>();
let launch = json!({
"format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem,
"entry": contract.entry, "executable": runtime_executable.expect("interpreter branch"),
"arguments": arguments, "working_directory": tree,
});
let mut output = tree_files.finish();
output.push(resource(
format!("{}.dever-worker.json", contract.entry),
serde_json::to_vec(&launch).map_err(|error| error.to_string())?,
false,
));
Ok(output)
}
fn digest(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn resource(path: String, bytes: Vec<u8>, executable: bool) -> EmbeddedResource {
EmbeddedResource {
sha256: digest(&bytes),
path,
bytes,
executable,
}
}
pub(crate) fn valid_path(path: &str) -> Result<(), String> {
let relative = Path::new(path);
if path.is_empty()
|| path.contains(['\\', ':', '\0'])
|| relative.is_absolute()
|| relative
.components()
.any(|part| !matches!(part, Component::Normal(_)))
|| path.split('/').any(str::is_empty)
{
return Err(format!(
"managed Worker path '{path}' is not portable and relative"
));
}
Ok(())
}
fn entry_name(entry: &str) -> Result<&str, String> {
valid_path(entry)?;
let name = Path::new(entry)
.file_name()
.and_then(|name| name.to_str())
.ok_or("invalid managed Worker entry name")?;
if !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
{
return Err("managed Worker entry name contains an unsafe character".into());
}
Ok(name)
}
fn ensure_javascript_format(entry: &str, tree: &mut Tree) -> Result<(), String> {
if !entry_name(entry)?.ends_with(".js") {
return Ok(());
}
let path = format!("{}/source/package.json", tree.prefix);
if let Some(package) = tree.files.get(&path) {
let metadata: serde_json::Value = serde_json::from_slice(&package.bytes)
.map_err(|error| format!("invalid Adapter package.json: {error}"))?;
if !matches!(
metadata.get("type").and_then(|value| value.as_str()),
Some("module" | "commonjs")
) {
return Err(
"managed .js Worker entry requires package.json type module or commonjs".into(),
);
}
} else {
tree.add(
"source/package.json",
b"{\"type\":\"module\"}\n".to_vec(),
false,
)?;
}
Ok(())
}
fn add_adapter_source(
project_root: &Path,
contract: &ExternalWorkerContract,
package_files: &BTreeMap<String, Vec<u8>>,
tree: &mut Tree,
) -> Result<(), String> {
let name = entry_name(&contract.entry)?;
let allowed: &[&str] = match contract.ecosystem.as_str() {
"pip" if name.ends_with(".py") => &["py", "pyi", "json"],
"npm" if name.ends_with(".js") || name.ends_with(".mjs") || name.ends_with(".cjs") => {
&["js", "mjs", "cjs", "json"]
}
"go" if name.ends_with(".go") => &[],
_ => return Err("managed Worker entry has an unsupported language extension".into()),
};
let package_owned = crate::packages::owns_worker(project_root, &contract.entry)?;
if add_package_adapter_source(&contract.entry, allowed, package_files, tree, package_owned)? {
return Ok(());
}
let entry = libs::owned_worker(project_root, &contract.entry)?;
let parent = entry
.parent()
.ok_or("managed Worker entry has no source directory")?;
let mut pending = vec![parent.to_path_buf()];
while let Some(directory) = pending.pop() {
let mut children = fs::read_dir(&directory)
.map_err(|error| format!("cannot read Adapter source: {error}"))?
.map(|result| {
result
.map(|entry| entry.path())
.map_err(|error| error.to_string())
})
.collect::<Result<Vec<_>, _>>()?;
children.sort();
for path in children {
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("cannot inspect Adapter source: {error}"))?;
if metadata.file_type().is_symlink() {
return Err("Adapter source contains a symbolic link".into());
}
let relative = path
.strip_prefix(parent)
.map_err(|error| error.to_string())?
.to_str()
.ok_or("Adapter source path is not UTF-8")?
.replace(std::path::MAIN_SEPARATOR, "/");
valid_path(&relative)?;
if metadata.is_dir() {
if !path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| IGNORED_SOURCE_DIRECTORIES.contains(&name))
{
pending.push(path);
}
} else if metadata.is_file() {
if allowed.is_empty()
|| path
.extension()
.and_then(|value| value.to_str())
.is_some_and(|extension| allowed.contains(&extension))
{
tree.add(
&format!("source/{relative}"),
fs::read(&path)
.map_err(|error| format!("cannot read Adapter source: {error}"))?,
false,
)?;
}
} else {
return Err("Adapter source contains a non-regular file".into());
}
}
}
Ok(())
}
fn add_package_adapter_source(
entry: &str,
allowed: &[&str],
package_files: &BTreeMap<String, Vec<u8>>,
tree: &mut Tree,
package_owned: bool,
) -> Result<bool, String> {
// Ownership comes from the checked Package lock, not from file presence:
// a missing entire source directory must never enable a local fallback.
if !package_owned {
return Ok(false);
}
if !package_files.contains_key(entry) {
return Err(format!("locked Package Worker entry '{entry}' is missing"));
}
let parent = Path::new(entry)
.parent()
.and_then(|path| path.to_str())
.ok_or("Package Worker entry has no source directory")?;
let prefix = format!("{parent}/");
for (path, bytes) in package_files {
let Some(relative) = path.strip_prefix(&prefix) else {
continue;
};
valid_path(relative)?;
if relative
.split('/')
.any(|part| IGNORED_SOURCE_DIRECTORIES.contains(&part))
{
continue;
}
if allowed.is_empty()
|| Path::new(relative)
.extension()
.and_then(|extension| extension.to_str())
.is_some_and(|extension| allowed.contains(&extension))
{
tree.add(&format!("source/{relative}"), bytes.clone(), false)?;
}
}
Ok(true)
}
fn dependency_closure<'a>(
contract: &ExternalWorkerContract,
lock: &'a LockFile,
) -> Result<Vec<&'a LockedLib>, String> {
let mut remaining = contract
.libs
.iter()
.map(|value| value.parse::<LibSpec>())
.collect::<Result<Vec<_>, _>>()?;
let mut visited = BTreeSet::new();
let mut closure = Vec::new();
while let Some(spec) = remaining.pop() {
if !visited.insert(spec.clone()) {
continue;
}
if spec.ecosystem.as_str() != contract.ecosystem {
return Err("Worker Lib ecosystem differs from Adapter".into());
}
let lib = lock
.libs
.iter()
.find(|lib| lib.spec == spec)
.ok_or_else(|| format!("locked Worker Lib {} is missing", spec.key()))?;
remaining.extend(
lib.dependencies
.iter()
.map(|dependency| dependency.spec.clone()),
);
closure.push(lib);
}
closure.sort_by(|left, right| left.spec.cmp(&right.spec));
Ok(closure)
}
fn locked_archive<'a>(
lib: &LockedLib,
extension: &str,
supplied: &'a BTreeMap<&str, &EmbeddedResource>,
target: &str,
) -> Result<&'a [u8], String> {
let artifact = lib
.artifacts
.iter()
.find(|artifact| {
artifact.target == target && artifact.path.ends_with(&format!(".{extension}"))
})
.ok_or_else(|| format!("locked {} archive for {target} is missing", lib.spec.key()))?;
let path = format!(
"lib/{}/{}/{}/archive.{extension}",
lib.spec.ecosystem.as_str(),
digest(lib.spec.distribution_name().as_bytes()),
lib.spec.version
);
if artifact.path != path {
return Err(format!(
"locked Worker archive path differs from the {} installation layout",
lib.spec.key()
));
}
let resource = supplied
.get(path.as_str())
.ok_or_else(|| format!("locked Worker archive '{path}' is missing"))?;
if resource.sha256 != artifact.sha256 || resource.bytes.len() as u64 != artifact.bytes {
return Err(format!(
"locked Worker archive '{path}' differs from its lock"
));
}
Ok(&resource.bytes)
}
fn add_python_dependencies(
contract: &ExternalWorkerContract,
lock: &LockFile,
supplied: &BTreeMap<&str, &EmbeddedResource>,
runtime: &runtime::Manifest,
tree: &mut Tree,
) -> Result<(), String> {
let mut installed_distributions = BTreeMap::<&str, &LockedLib>::new();
let mut wheel_files = BTreeSet::new();
for lib in dependency_closure(contract, lock)? {
let wheel = python_wheel::Wheel::parse(
&lib.spec,
None,
&runtime.python_wheel_tags,
locked_archive(lib, "whl", supplied, &runtime.target)?,
)?;
wheel.validate_dependencies(
lib,
runtime
.python_markers
.as_ref()
.expect("validated Python markers"),
)?;
libs::build::validate_python_policy(
lock,
lib,
runtime
.python_markers
.as_ref()
.expect("validated Python markers"),
)?;
if let Some(previous) = installed_distributions.insert(lib.spec.distribution_name(), lib) {
if previous.spec.version != lib.spec.version || previous.artifacts != lib.artifacts {
return Err("Python extras refer to conflicting distribution artifacts".into());
}
continue;
}
for file in wheel.install(
&format!(
"runtime/{}",
runtime
.executable
.as_deref()
.expect("validated Python interpreter")
),
&runtime
.python_markers
.as_ref()
.expect("validated Python markers")
.python_version()
.to_string(),
)? {
wheel_files.insert(file.path.clone());
tree.add(&file.path, file.bytes, file.executable)?;
}
}
validate_worker_native(
runtime,
tree,
supplied,
&wheel_files,
&runtime.python_extension_suffixes,
)
}
fn validate_worker_native(
runtime: &runtime::Manifest,
tree: &Tree,
supplied: &BTreeMap<&str, &EmbeddedResource>,
entries: &BTreeSet<String>,
suffixes: &[String],
) -> Result<(), String> {
let prefix = format!("{}/", tree.prefix);
let mut files = tree
.files
.iter()
.map(|(path, resource)| {
(
path.strip_prefix(&prefix)
.expect("owned Worker tree")
.to_owned(),
resource.bytes.as_slice(),
)
})
.collect::<BTreeMap<_, _>>();
files.extend(
supplied
.iter()
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
.map(|(path, resource)| ((*path).to_owned(), resource.bytes.as_slice())),
);
python_wheel::validate_native(
&runtime.target,
suffixes,
&format!(
"runtime/{}",
runtime
.executable
.as_deref()
.expect("validated interpreter")
),
&files,
entries,
)?;
Ok(())
}
fn add_javascript_dependencies(
contract: &ExternalWorkerContract,
lock: &LockFile,
supplied: &BTreeMap<&str, &EmbeddedResource>,
runtime: &runtime::Manifest,
tree: &mut Tree,
) -> Result<(), String> {
let requested = contract
.libs
.iter()
.map(|value| value.parse())
.collect::<Result<Vec<LibSpec>, _>>()?;
let Some(environment) = libs::npm::environment(lock, &requested)? else {
return Ok(());
};
if let Some(receipt) = libs::build::npm_receipt(lock, environment)? {
if receipt.output.target != runtime.target || receipt.tools.target != runtime.target {
return Err("locked npm build receipt differs from the selected Worker target".into());
}
let resource = supplied
.get(receipt.output.path.as_str())
.ok_or("locked npm installation resource is missing")?;
for (path, bytes, executable) in
libs::build::npm_installation(lock, environment, &resource.bytes)?
{
tree.add(&path, bytes, executable)?;
}
let entries = receipt
.native_entries
.iter()
.filter(|path| {
libs::npm::installed_owner(path)
.is_ok_and(|owner| environment.instances.iter().any(|node| node.path == owner))
})
.cloned()
.collect();
return validate_worker_native(runtime, tree, supplied, &entries, &[]);
}
for (path, bytes) in libs::npm::files(environment, lock, |lib| {
locked_archive(lib, "tgz", supplied, &runtime.target)
})? {
tree.add(&path, bytes, false)?;
}
Ok(())
}
struct Tree {
prefix: String,
files: BTreeMap<String, EmbeddedResource>,
bytes: usize,
}
impl Tree {
fn new(prefix: String) -> Self {
Self {
prefix,
files: BTreeMap::new(),
bytes: 0,
}
}
fn add(&mut self, relative: &str, bytes: Vec<u8>, executable: bool) -> Result<(), String> {
valid_path(relative)?;
let total = self
.bytes
.checked_add(bytes.len())
.ok_or("managed Worker tree size overflow")?;
if total > MAX_TREE_BYTES || self.files.len() >= MAX_TREE_FILES {
return Err("managed Worker tree exceeds file or byte budget".into());
}
let path = format!("{}/{relative}", self.prefix);
if self.files.contains_key(&path) {
return Err(format!("managed Worker tree repeats '{path}'"));
}
let mut ancestor = path.as_str();
while let Some((parent, _)) = ancestor.rsplit_once('/') {
if self.files.contains_key(parent) {
return Err(format!(
"managed Worker tree has a file/directory conflict at '{parent}'"
));
}
ancestor = parent;
}
let descendant_prefix = format!("{path}/");
if self
.files
.range(descendant_prefix.clone()..)
.next()
.is_some_and(|(existing, _)| existing.starts_with(&descendant_prefix))
{
return Err(format!(
"managed Worker tree has a file/directory conflict at '{path}'"
));
}
self.bytes = total;
self.files
.insert(path.clone(), resource(path, bytes, executable));
Ok(())
}
fn finish(self) -> Vec<EmbeddedResource> {
self.files.into_values().collect()
}
fn remove_prefix(&mut self, relative: &str) {
let prefix = format!("{}/{relative}", self.prefix);
self.files.retain(|path, resource| {
if path.starts_with(&prefix) {
self.bytes -= resource.bytes.len();
false
} else {
true
}
});
}
}
+239
View File
@@ -0,0 +1,239 @@
//! Offline native command trees, including their explicitly supplied libraries.
use std::collections::BTreeMap;
use std::fs::{self, File};
use std::io::Read;
use std::path::Path;
use dever_core::hir::ExternalWorkerContract;
use dever_core::native::EmbeddedResource;
use goblin::elf::{dynamic, header};
use serde_json::json;
use super::{MAX_TREE_BYTES, MAX_TREE_FILES, Tree, digest, elf, resource, runtime, valid_path};
use crate::toolchain::BuildTarget;
pub(super) fn prepare(
root: &Path,
contract: &ExternalWorkerContract,
package_files: &BTreeMap<String, Vec<u8>>,
supplied: &BTreeMap<&str, &EmbeddedResource>,
target: BuildTarget,
) -> Result<Vec<EmbeddedResource>, String> {
let prefix = format!(
"commands/{}",
digest(&serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?)
);
let mut tree = Tree::new(prefix.clone());
let source = command_files(root, &contract.entry, package_files)?;
let binary = &source["program/tool"];
runtime::validate_binary(&contract.entry, binary, target.platform())?;
let machine = match target {
BuildTarget::LinuxX86_64 => header::EM_X86_64,
BuildTarget::LinuxAarch64 => header::EM_AARCH64,
};
let executable = elf::object("program/tool", binary, machine)?;
let interpreter = executable.interpreter.map(str::to_owned);
if interpreter.is_none() && !executable.libraries.is_empty() {
return Err("command executable has dynamic dependencies without an interpreter".into());
}
let mut files = source
.iter()
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
files.extend(
supplied
.iter()
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
.map(|(path, entry)| ((*path).to_owned(), entry.bytes.as_slice())),
);
let closure = elf::Closure {
files: &files,
machine,
directories: vec!["program/lib".into(), "sandbox/lib".into()],
};
for (path, bytes) in &source {
let object = elf::object(path, bytes, machine)?;
reject_loader_hooks(&object)?;
// The logical program root becomes /worker in the sandbox. A path
// escaping it could otherwise resolve a library from a file grant.
for fields in [&object.rpaths, &object.runpaths] {
for directory in elf::origin_directories(path, fields)? {
if directory != "program" && !directory.starts_with("program/") {
return Err("command library search path escapes its packaged tree".into());
}
}
}
if let Some(name) = path.strip_prefix("program/lib/") {
if supplied.contains_key(format!("sandbox/lib/{name}").as_str()) {
return Err(format!(
"command library '{name}' overrides a signed sandbox library"
));
}
if object.header.e_type != header::ET_DYN
|| object.interpreter.is_some()
|| object.soname.is_some_and(|soname| soname != name)
{
return Err(format!(
"command library '{name}' is not a matching SONAME shared object"
));
}
}
closure.follow(path, Vec::new(), &BTreeMap::new())?;
}
for (path, bytes) in source {
let name = path.strip_prefix("program/").expect("owned command tree");
tree.add(name, bytes, name == "tool")?;
}
let (executable, arguments) = if let Some(interpreter) = interpreter {
let name = interpreter
.rsplit('/')
.next()
.expect("validated GNU loader");
let loader = supplied
.get(format!("sandbox/lib/{name}").as_str())
.ok_or("command requires the signed target GNU loader")?;
runtime::validate_binary(name, &loader.bytes, target.platform())?;
tree.add("loader", loader.bytes.clone(), true)?;
(
format!("{prefix}/loader"),
vec![
json!({"literal": "--inhibit-cache"}),
json!({"literal": "--library-path"}),
json!({"literal": "lib:/lib"}),
json!({"resource": format!("{prefix}/tool")}),
],
)
} else {
(format!("{prefix}/tool"), Vec::new())
};
let manifest = json!({
"format": "dever-command-launch-v1", "ecosystem": "command",
"entry": contract.entry, "executable": executable,
"arguments": arguments, "working_directory": prefix,
});
let mut output = tree.finish();
output.push(resource(
format!("{}.dever-command.json", contract.entry),
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
false,
));
Ok(output)
}
fn reject_loader_hooks(elf: &goblin::elf::Elf<'_>) -> Result<(), String> {
if elf.dynamic.as_ref().is_some_and(|table| {
table.dyns.iter().any(|entry| {
matches!(
entry.d_tag,
dynamic::DT_CONFIG
| dynamic::DT_AUDIT
| dynamic::DT_DEPAUDIT
| 0x7fff_fffd
| 0x7fff_ffff
)
})
}) {
return Err("command ELF contains unsupported dynamic loader hooks".into());
}
Ok(())
}
fn command_files(
root: &Path,
entry: &str,
package_files: &BTreeMap<String, Vec<u8>>,
) -> Result<BTreeMap<String, Vec<u8>>, String> {
valid_path(entry)?;
let parent = entry
.rsplit_once('/')
.map(|(parent, _)| parent)
.ok_or("command entry must belong to a domain")?;
let library_prefix = format!("{parent}/lib/");
let packaged = crate::packages::owns_worker(root, entry)?;
let binary = if packaged {
package_files
.get(entry)
.cloned()
.ok_or("Package command executable is missing")?
} else {
read_file(root, entry)?
};
let mut files = BTreeMap::from([("program/tool".to_owned(), binary)]);
let mut bytes = files["program/tool"].len();
if bytes == 0 || bytes > MAX_TREE_BYTES {
return Err("command executable is empty or exceeds the byte budget".into());
}
let names = if packaged {
package_files
.keys()
.filter_map(|path| path.strip_prefix(&library_prefix))
.map(str::to_owned)
.collect::<Vec<_>>()
} else {
library_names(&root.join(&library_prefix))?
};
for name in names {
valid_path(&name)?;
if name.contains('/') || files.len() >= MAX_TREE_FILES {
return Err("command lib directory must contain bounded regular library files".into());
}
let path = format!("{library_prefix}{name}");
let library = if packaged {
package_files[&path].clone()
} else {
read_file(root, &path)?
};
bytes = bytes
.checked_add(library.len())
.ok_or("command library size overflow")?;
if bytes > MAX_TREE_BYTES {
return Err("command libraries exceed the byte budget".into());
}
files.insert(format!("program/lib/{name}"), library);
}
Ok(files)
}
fn library_names(directory: &Path) -> Result<Vec<String>, String> {
let metadata = match fs::symlink_metadata(directory) {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(error) => return Err(format!("cannot inspect command lib directory: {error}")),
};
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err("command lib directory must be a real directory".into());
}
let mut names = Vec::new();
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
if names.len() >= MAX_TREE_FILES
|| !entry
.file_type()
.map_err(|error| error.to_string())?
.is_file()
{
return Err("command lib directory must contain bounded regular library files".into());
}
names.push(
entry
.file_name()
.into_string()
.map_err(|_| "command library name is not UTF-8")?,
);
}
names.sort();
Ok(names)
}
fn read_file(root: &Path, entry: &str) -> Result<Vec<u8>, String> {
let path = crate::libs::owned_worker(root, entry)?;
let mut bytes = Vec::new();
File::open(path)
.and_then(|file| file.take(MAX_TREE_BYTES as u64 + 1).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read command resource '{entry}': {error}"))?;
if bytes.len() > MAX_TREE_BYTES {
return Err("command resource exceeds the byte budget".into());
}
Ok(bytes)
}
+176
View File
@@ -0,0 +1,176 @@
//! Shared ELF architecture, loader search and versioned dependency closure.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use goblin::elf::{Elf, header};
pub(crate) fn object<'a>(path: &str, bytes: &'a [u8], machine: u16) -> Result<Elf<'a>, String> {
let elf = Elf::parse(bytes).map_err(|error| format!("invalid native ELF '{path}': {error}"))?;
if !elf.is_64
|| !elf.little_endian
|| elf.header.e_machine != machine
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
{
return Err(format!(
"native ELF '{path}' has an incompatible target or object type"
));
}
Ok(elf)
}
pub(crate) fn origin_directories(path: &str, fields: &[&str]) -> Result<Vec<String>, String> {
let parent = path.rsplit_once('/').map_or("", |(parent, _)| parent);
let mut result = Vec::new();
for field in fields {
for directory in field.split(':') {
let relative = directory
.strip_prefix("$ORIGIN")
.or_else(|| directory.strip_prefix("${ORIGIN}"))
.ok_or("native library search path must be relative to $ORIGIN")?;
if !relative.is_empty() && !relative.starts_with('/')
|| relative.contains(['$', '\\', '\0'])
{
return Err("native library search path has an unsupported token".into());
}
let combined = format!("{parent}{relative}");
let mut normalized = Vec::new();
for part in combined.split('/') {
match part {
"" | "." => {}
".." => {
normalized
.pop()
.ok_or("native library path escapes the installation tree")?;
}
part => normalized.push(part),
}
}
let normalized = normalized.join("/");
if normalized.is_empty() || normalized.starts_with("sandbox/") {
return Err("native library path escapes its Worker installation".into());
}
if !result.contains(&normalized) {
result.push(normalized);
}
}
}
Ok(result)
}
fn plain_library(name: &str) -> Result<(), String> {
if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0']) {
return Err("native DT_NEEDED must be a plain library name".into());
}
Ok(())
}
fn validate_versions(consumer: &Elf<'_>, library: &str, provider: &Elf<'_>) -> Result<(), String> {
let mut provided = BTreeSet::new();
if let Some(definitions) = &provider.verdef {
for definition in definitions.iter() {
for name in definition.iter() {
provided.insert(
provider
.dynstrtab
.get_at(name.vda_name)
.ok_or("invalid ELF version definition")?
.to_owned(),
);
}
}
}
if let Some(requirements) = &consumer.verneed {
for requirement in requirements.iter() {
if consumer.dynstrtab.get_at(requirement.vn_file) != Some(library) {
continue;
}
for version in requirement.iter() {
let name = consumer
.dynstrtab
.get_at(version.vna_name)
.ok_or("invalid ELF version requirement")?;
if version.vna_flags & 2 == 0 && !provided.contains(name) {
return Err(format!(
"packaged library '{library}' does not provide native ABI version '{name}'"
));
}
}
}
}
Ok(())
}
pub(crate) struct Closure<'a> {
pub(crate) files: &'a BTreeMap<String, &'a [u8]>,
pub(crate) machine: u16,
pub(crate) directories: Vec<String>,
}
impl Closure<'_> {
pub(crate) fn follow(
&self,
start: &str,
inherited: Vec<String>,
globals: &BTreeMap<String, String>,
) -> Result<(BTreeMap<String, String>, BTreeSet<String>), String> {
let mut loaded = globals.clone();
let mut pending = VecDeque::from([(start.to_owned(), inherited)]);
let mut seen = BTreeSet::new();
while let Some((path, inherited)) = pending.pop_front() {
if !seen.insert(path.clone()) {
continue;
}
let bytes = self
.files
.get(&path)
.ok_or_else(|| format!("native dependency '{path}' is absent"))?;
let elf = object(&path, bytes, self.machine)?;
let rpaths = if elf.runpaths.is_empty() {
origin_directories(&path, &elf.rpaths)?
} else {
Vec::new()
};
let runpaths = origin_directories(&path, &elf.runpaths)?;
let mut inherited_next = rpaths.clone();
inherited_next.extend(inherited.clone());
let inherited_search = if elf.runpaths.is_empty() {
inherited.as_slice()
} else {
&[]
};
for name in &elf.libraries {
plain_library(name)?;
let supplied = loaded
.get(*name)
.cloned()
.or_else(|| {
rpaths
.iter()
.chain(inherited_search.iter())
.chain(runpaths.iter())
.map(|directory| format!("{directory}/{name}"))
.chain(
self.directories
.iter()
.map(|directory| format!("{directory}/{name}")),
)
.find(|candidate| self.files.contains_key(candidate))
})
.ok_or_else(|| {
format!("native library '{name}' is not in the packaged dependency closure")
})?;
let provider = object(&supplied, self.files[&supplied], self.machine)?;
validate_versions(&elf, name, &provider)?;
loaded.insert((*name).to_owned(), supplied.clone());
if let Some(soname) = provider.soname {
plain_library(soname)?;
if soname != *name {
return Err("native library SONAME differs from DT_NEEDED".into());
}
}
pending.push_back((supplied, inherited_next.clone()));
}
}
Ok((loaded, seen))
}
}
+574
View File
@@ -0,0 +1,574 @@
//! Compile a checked Go Adapter with the exact tools and archives in its locked pack.
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsString;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use dever_core::hir::ExternalWorkerContract;
use dever_core::native::EmbeddedResource;
use serde::Deserialize;
use serde_json::json;
use super::{Tree, dependency_closure, digest, entry_name, locked_archive, valid_path};
use crate::libs::build::process::Stage;
use crate::libs::{self, LockFile};
const SDK: &[u8] = include_bytes!("../../../../sdk/go/component.go");
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct GoBuildManifest {
host: crate::toolchain::BuildTarget,
compiler: String,
linker: String,
analyzer: String,
stdlib_importcfg: String,
goos: String,
goarch: String,
go_version: String,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct SelectedPackage {
name: String,
files: Vec<String>,
imports: Vec<String>,
embed: BTreeMap<String, Vec<String>>,
}
struct Builder {
stage: Stage,
input: PathBuf,
assets: PathBuf,
grants: Vec<dever_sandbox::Grant>,
tools: GoBuildManifest,
entry: String,
stdlib: BTreeMap<String, PathBuf>,
modules: BTreeMap<String, PathBuf>,
archives: BTreeMap<String, PathBuf>,
visiting: BTreeSet<String>,
}
pub(super) struct Inputs<'a> {
pub files: &'a [(String, Vec<u8>)],
pub manifest: &'a GoBuildManifest,
pub sandbox: &'a [EmbeddedResource],
}
pub(super) fn compile_worker(
contract: &ExternalWorkerContract,
lock: &LockFile,
supplied: &BTreeMap<&str, &EmbeddedResource>,
inputs: Inputs<'_>,
tree: &mut Tree,
target: &str,
) -> Result<String, String> {
let Inputs {
files: pack_files,
manifest: build,
sandbox: build_assets,
} = inputs;
let references = build.validate_files(pack_files, target)?;
if build.host != crate::toolchain::BuildTarget::host()? {
return Err("Go build tools require a different build host".into());
}
let stage = Stage::new()?;
let input = stage.0.join("input");
let assets = stage.0.join("sandbox");
for resource in build_assets
.iter()
.filter(|resource| resource.path.starts_with("sandbox/"))
{
if digest(&resource.bytes) != resource.sha256 {
return Err("Go build sandbox asset has a digest mismatch".into());
}
stage.write(&resource.path, &resource.bytes, resource.executable)?;
}
let build_directory = stage.0.join("build");
fs::create_dir(&build_directory)
.map_err(|error| format!("cannot create Go build output: {error}"))?;
let grants = vec![dever_sandbox::Grant {
source: build_directory,
destination: "/data/build".into(),
writable: true,
}];
for (path, bytes) in pack_files {
stage.write(
&format!("input/{path}"),
bytes,
path == &build.compiler || path == &build.linker || path == &build.analyzer,
)?;
}
let stdlib = references
.into_iter()
.map(|(name, path)| (name, input.join(path)))
.collect();
let source_prefix = format!("{}/source/", tree.prefix);
let mut entry_present = false;
for (path, resource) in &tree.files {
if let Some(relative) = path.strip_prefix(&source_prefix) {
if relative == entry_name(&contract.entry)? {
entry_present = true;
}
stage.write(&format!("input/source/{relative}"), &resource.bytes, false)?;
}
}
if !entry_present {
return Err("managed Go Worker entry source is missing".into());
}
if tree
.files
.contains_key(&format!("{}/source/dever_worker_runner.go", tree.prefix))
{
return Err("Adapter source conflicts with the generated Go Worker runner".into());
}
stage.write("input/sdk/component.go", SDK, false)?;
let runner = runner_source(contract)?;
stage.write(
"input/source/dever_worker_runner.go",
runner.as_bytes(),
false,
)?;
let mut modules = BTreeMap::new();
let authenticated =
libs::sumdb::verify_chain(&lock.go_sumdb, &libs::sumdb::Verifier::official())?;
for lib in dependency_closure(contract, lock)? {
let root = format!("{}@v{}/", lib.spec.name, lib.spec.version);
let directory = format!("modules/{}", digest(lib.spec.key().as_bytes()));
let archive = locked_archive(lib, "zip", supplied, target)?;
authenticated.verify_zip(&lib.spec.name, &lib.spec.version, archive)?;
let files = libs::archive_files("zip", archive)?;
if files.is_empty() {
return Err(format!("locked Go module {} is empty", lib.spec.key()));
}
for (path, bytes) in files {
let relative = path.strip_prefix(&root).ok_or_else(|| {
format!(
"locked Go module {} has a mismatched archive root",
lib.spec.key()
)
})?;
stage.write(&format!("input/{directory}/{relative}"), &bytes, false)?;
}
if modules
.insert(lib.spec.name.clone(), input.join(directory))
.is_some()
{
return Err("locked Go module graph selects multiple versions of one module".into());
}
}
let mut builder = Builder {
stage,
input,
assets,
grants,
tools: build.clone(),
entry: entry_name(&contract.entry)?.into(),
stdlib,
modules,
archives: BTreeMap::new(),
visiting: BTreeSet::new(),
};
builder.verify_tools()?;
builder.compile("dever-component")?;
builder.compile("main")?;
let binary_name = format!("worker{}", std::env::consts::EXE_SUFFIX);
builder.link(&binary_name)?;
let path = builder.stage.0.join("build").join(&binary_name);
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
}
let file = options
.open(path)
.map_err(|error| format!("cannot open linked Go Worker: {error}"))?;
let metadata = file.metadata().map_err(|error| error.to_string())?;
if !metadata.is_file() || metadata.len() > 128 * 1024 * 1024 {
return Err("linked Go Worker is not a bounded regular file".into());
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if metadata.nlink() != 1 {
return Err("linked Go Worker must not have hard links".into());
}
}
let mut binary = Vec::new();
file.take(128 * 1024 * 1024 + 1)
.read_to_end(&mut binary)
.map_err(|error| format!("cannot read linked Go Worker: {error}"))?;
if binary.len() > 128 * 1024 * 1024 {
return Err("linked Go Worker exceeds the byte limit".into());
}
if binary.is_empty() {
return Err("linked Go Worker is empty".into());
}
super::runtime::validate_binary("Go Worker", &binary, target)?;
tree.add(&binary_name, binary, true)?;
Ok(binary_name)
}
impl GoBuildManifest {
pub(super) fn executable(&self, path: &str) -> bool {
[&self.compiler, &self.linker, &self.analyzer]
.iter()
.any(|tool| tool.as_str() == path)
}
pub(super) fn validate_files(
&self,
files: &[(String, Vec<u8>)],
target: &str,
) -> Result<BTreeMap<String, String>, String> {
self.validate(target)?;
let mut pack = BTreeMap::new();
for (path, bytes) in files {
if pack.insert(path.as_str(), bytes.as_slice()).is_some() {
return Err("Go build pack repeats a file".into());
}
}
for tool in [&self.compiler, &self.linker, &self.analyzer] {
if !tool.starts_with("bin/")
|| !pack
.get(tool.as_str())
.is_some_and(|bytes| !bytes.is_empty())
{
return Err(format!("locked Go build tool '{tool}' is missing"));
}
super::runtime::validate_binary(tool, pack[tool.as_str()], self.host.platform())?;
}
if self.stdlib_importcfg != "stdlib/importcfg" {
return Err("locked Go standard library importcfg path is invalid".into());
}
let importcfg = pack
.get(self.stdlib_importcfg.as_str())
.ok_or("locked Go standard library importcfg is missing")?;
let importcfg = std::str::from_utf8(importcfg)
.map_err(|_| "locked Go standard library importcfg is not UTF-8")?;
let mut stdlib = BTreeMap::new();
for line in importcfg.lines() {
let (name, path) = line
.strip_prefix("packagefile ")
.and_then(|line| line.split_once('='))
.ok_or("invalid locked Go standard library importcfg")?;
valid_import(name)?;
if path != format!("stdlib/{name}.a")
|| !pack.get(path).is_some_and(|bytes| !bytes.is_empty())
|| stdlib.insert(name.into(), path.into()).is_some()
{
return Err(
"locked Go standard library importcfg has a missing or repeated archive".into(),
);
}
let archive = pack[path];
let version = self
.go_version
.split_whitespace()
.nth(2)
.ok_or("invalid Go version")?;
let object_identity = format!("go object {} {} {version} ", self.goos, self.goarch);
if !archive.starts_with(b"!<arch>\n__.PKGDEF")
|| !archive
.get(68..)
.is_some_and(|bytes| bytes.starts_with(object_identity.as_bytes()))
{
return Err(format!(
"locked Go standard library archive '{name}' differs from its target or compiler version"
));
}
}
if !["runtime", "context", "encoding/json"]
.iter()
.all(|name| stdlib.contains_key(*name))
{
return Err("locked Go standard library importcfg is incomplete".into());
}
Ok(stdlib)
}
fn validate(&self, target: &str) -> Result<(), String> {
let expected = match target {
"linux-x86_64" => ("linux", "amd64"),
"linux-aarch64" => ("linux", "arm64"),
_ => return Err("managed Go build target is unsupported".into()),
};
if (self.goos.as_str(), self.goarch.as_str()) != expected
|| !self.go_version.starts_with("go version go")
|| !self.go_version.ends_with(match self.host {
crate::toolchain::BuildTarget::LinuxX86_64 => " linux/amd64",
crate::toolchain::BuildTarget::LinuxAarch64 => " linux/arm64",
})
{
return Err("managed Go build tools do not match the selected target".into());
}
for path in [
&self.compiler,
&self.linker,
&self.analyzer,
&self.stdlib_importcfg,
] {
valid_path(path)?;
}
Ok(())
}
}
impl Builder {
fn verify_tools(&self) -> Result<(), String> {
let version = self
.tools
.go_version
.split_whitespace()
.nth(2)
.ok_or("locked Go build version is invalid")?;
for (kind, path) in [
("compile", &self.tools.compiler),
("link", &self.tools.linker),
] {
let output = self.run(&self.input.join(path), &["-V"])?;
if output != format!("{kind} version {version}\n").as_bytes() {
return Err(format!(
"locked Go {kind} tool version differs from its pack"
));
}
}
Ok(())
}
fn package_dir(&self, name: &str) -> Result<PathBuf, String> {
if name == "main" {
return Ok(self.input.join("source"));
}
if name == "dever-component" {
return Ok(self.input.join("sdk"));
}
if let Some(relative) = name.strip_prefix("dever-worker/") {
valid_path(relative)?;
return Ok(self.input.join("source").join(relative));
}
let (module, root) = self
.modules
.iter()
.filter(|(module, _)| {
name == module.as_str() || name.starts_with(&format!("{module}/"))
})
.max_by_key(|(module, _)| module.len())
.ok_or_else(|| format!("Go import '{name}' is not in the locked module graph"))?;
let suffix = name
.strip_prefix(module)
.expect("selected module prefix")
.trim_start_matches('/');
if !suffix.is_empty() {
valid_path(suffix)?;
}
Ok(root.join(suffix))
}
fn compile(&mut self, name: &str) -> Result<(), String> {
if self.stdlib.contains_key(name) || name == "unsafe" || self.archives.contains_key(name) {
return Ok(());
}
valid_import(name)?;
if !self.visiting.insert(name.to_owned()) {
return Err(format!("Go import cycle includes '{name}'"));
}
let directory = self.package_dir(name)?;
let analyzer = self.input.join(&self.tools.analyzer);
let output = self.run(
&analyzer,
&[
self.tools.goos.as_str(),
self.tools.goarch.as_str(),
directory.to_str().ok_or("Go package path is not UTF-8")?,
],
)?;
let selected: SelectedPackage = serde_json::from_slice(&output)
.map_err(|error| format!("invalid locked Go analyzer output: {error}"))?;
if selected.files.is_empty() || (name == "main") != (selected.name == "main") {
return Err(format!(
"Go package '{name}' has an invalid package declaration"
));
}
if name == "main" && !selected.files.contains(&self.entry) {
// The generated runner may be selected while the checked entry is excluded by tags.
return Err("checked Go Worker entry is excluded by target build constraints".into());
}
for import in &selected.imports {
self.compile(import)?;
}
let cfg = self.importcfg()?;
let id = digest(name.as_bytes());
let cfg_path = self.stage.write(
&format!("input/config/{id}.importcfg"),
cfg.as_bytes(),
false,
)?;
let archive = self.stage.0.join(format!("build/{id}.a"));
let mut args = vec![
"-pack".to_owned(),
"-complete".into(),
"-nolocalimports".into(),
// Staging is invocation-owned; recorded source paths must not make
// identical locked Workers differ across run/build or projects.
"-trimpath".into(),
self.input.to_string_lossy().into_owned(),
"-p".into(),
name.into(),
"-importcfg".into(),
cfg_path.to_string_lossy().into_owned(),
"-o".into(),
archive.to_string_lossy().into_owned(),
];
if !selected.embed.is_empty() {
let mut files = BTreeMap::new();
for paths in selected.embed.values() {
for path in paths {
valid_path(path)?;
let file = directory.join(path);
if !file.is_file() {
return Err("Go embed file is absent from locked source".into());
}
files.insert(path.clone(), self.visible_path(&file)?);
}
}
let embed = json!({"Patterns":selected.embed,"Files":files});
let path = self.stage.write(
&format!("input/config/{id}.embedcfg"),
&serde_json::to_vec(&embed).map_err(|error| error.to_string())?,
false,
)?;
args.extend(["-embedcfg".into(), path.to_string_lossy().into_owned()]);
}
for file in selected.files {
valid_path(&file)?;
args.push(directory.join(file).to_string_lossy().into_owned());
}
let compiler = self.input.join(&self.tools.compiler);
let arguments = args.iter().map(String::as_str).collect::<Vec<_>>();
self.run(&compiler, &arguments)?;
self.archives.insert(name.into(), archive);
self.visiting.remove(name);
Ok(())
}
fn importcfg(&self) -> Result<String, String> {
let mut cfg = String::new();
for (name, path) in self.stdlib.iter().chain(&self.archives) {
cfg.push_str(&format!(
"packagefile {name}={}\n",
self.visible_path(path)?
));
}
Ok(cfg)
}
fn link(&self, output: &str) -> Result<(), String> {
let cfg = self.stage.write(
"input/config/link.importcfg",
self.importcfg()?.as_bytes(),
false,
)?;
let main = self
.archives
.get("main")
.ok_or("Go Worker main archive is missing")?;
let linker = self.input.join(&self.tools.linker);
let output_path = self.stage.0.join("build").join(output);
let temporary_directory = self.stage.0.join("build");
self.run(
&linker,
&[
"-linkmode=internal",
"-buildmode=exe",
"-tmpdir",
temporary_directory
.to_str()
.ok_or("Go build path is not UTF-8")?,
"-importcfg",
cfg.to_str().ok_or("Go importcfg path is not UTF-8")?,
"-o",
output_path.to_str().ok_or("Go output path is not UTF-8")?,
main.to_str().ok_or("Go archive path is not UTF-8")?,
],
)?;
Ok(())
}
fn run(&self, executable: &Path, args: &[&str]) -> Result<Vec<u8>, String> {
let arguments = args.iter().map(OsString::from).collect::<Vec<_>>();
self.stage.run(
self.sandbox(executable, &arguments).command()?,
"locked Go build tool",
&self.stage.0.join("build"),
)
}
fn sandbox<'a>(
&'a self,
executable: &'a Path,
arguments: &'a [OsString],
) -> dever_sandbox::Launch<'a> {
dever_sandbox::Launch {
assets: &self.assets,
worker: &self.input,
executable,
arguments,
working_directory: &self.input,
capabilities: dever_sandbox::Capabilities::default(),
grants: &self.grants,
}
}
fn visible_path(&self, path: &Path) -> Result<String, String> {
self.sandbox(&self.input, &[])
.argument_path(path)?
.into_os_string()
.into_string()
.map_err(|_| "Go sandbox path is not UTF-8".into())
}
}
fn valid_import(name: &str) -> Result<(), String> {
if name.is_empty()
|| name.split('/').any(str::is_empty)
|| !name.bytes().all(|byte| {
byte.is_ascii_alphanumeric() || matches!(byte, b'/' | b'.' | b'_' | b'-' | b'~')
})
{
return Err(format!("invalid locked Go import '{name}'"));
}
Ok(())
}
fn runner_source(contract: &ExternalWorkerContract) -> Result<String, String> {
let mut handlers = String::new();
let mut names = BTreeSet::new();
for operation in &contract.operations {
let mut function = String::new();
for part in operation.split(['.', '_', '-']) {
if part.is_empty() || !part.bytes().all(|byte| byte.is_ascii_alphanumeric()) {
return Err(format!(
"Go operation '{operation}' cannot bind to an exported function"
));
}
let mut letters = part.chars();
function.push(letters.next().expect("nonempty part").to_ascii_uppercase());
function.extend(letters);
}
if !names.insert(function.clone())
|| !function.starts_with(|character: char| character.is_ascii_uppercase())
{
return Err("Go operation names map to duplicate or invalid exported functions".into());
}
handlers.push_str(&format!("\t\t{operation:?}: {function},\n"));
}
Ok(format!(
"package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\tcomponent \"dever-component\"\n)\n\nfunc main() {{\n\tmanifest, err := os.ReadFile(\"contract.json\")\n\tif err == nil {{\n\t\terr = component.ServeManifest(manifest, map[string]func(context.Context, any, any) (any, error){{\n{handlers}\t\t}}, os.Stdin, os.Stdout)\n\t}}\n\tif err != nil {{\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}}\n}}\n"
))
}
@@ -0,0 +1,588 @@
//! One wheel admission and installation contract for registry downloads,
//! offline Workers and isolated build dependencies.
use std::collections::{BTreeMap, BTreeSet};
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
use pep440_rs::Version;
use pep508_rs::Requirement;
use sha2::{Digest, Sha256, Sha384, Sha512};
use crate::libs::{LibSpec, archive_files};
pub(crate) mod native;
mod scripts;
pub use native::validate_native;
pub struct Wheel {
files: Vec<(String, Vec<u8>)>,
stem: String,
pub requirements: Vec<String>,
pub extras: Vec<String>,
pub requires_python: Option<String>,
}
pub struct InstalledFile {
pub path: String,
pub bytes: Vec<u8>,
pub executable: bool,
}
/// Filename tags can be compressed; runtime and WHEEL tags must be expanded.
pub fn validate_tags(tags: &[String]) -> Result<(), String> {
let mut unique = BTreeSet::new();
if tags.is_empty() || tags.len() > 4096 {
return Err("Python wheel compatibility tags are missing or exceed their bound".into());
}
for tag in tags {
if tag.split('-').count() != 3
|| tag.split('-').any(|part| {
part.is_empty()
|| !part
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
})
|| !unique.insert(tag)
{
return Err("Python wheel compatibility tags are invalid or repeated".into());
}
}
Ok(())
}
pub(crate) fn validate_target(
target: &str,
tags: &[String],
markers: &pep508_rs::MarkerEnvironment,
) -> Result<(), String> {
validate_tags(tags)?;
let architecture = match target.parse::<crate::toolchain::BuildTarget>()? {
crate::toolchain::BuildTarget::LinuxX86_64 => "x86_64",
crate::toolchain::BuildTarget::LinuxAarch64 => "aarch64",
};
if markers.sys_platform() != "linux" || markers.platform_machine() != architecture {
return Err("Python marker environment differs from the selected target".into());
}
for tag in tags {
let platform = tag.rsplit('-').next().expect("validated wheel tag");
if platform != "any"
&& (!(platform.starts_with("linux_")
|| platform.starts_with("manylinux")
|| platform.starts_with("musllinux"))
|| !platform.ends_with(&format!("_{architecture}")))
{
return Err("Python wheel platform tag differs from the selected target".into());
}
}
Ok(())
}
fn expanded_tags(python: &str, abi: &str, platform: &str) -> Result<BTreeSet<String>, String> {
let mut tags = BTreeSet::new();
for python in python.split('.') {
for abi in abi.split('.') {
for platform in platform.split('.') {
if tags.len() >= 4096 || !tags.insert(format!("{python}-{abi}-{platform}")) {
return Err("wheel filename tags are repeated or exceed their bound".into());
}
}
}
}
validate_tags(&tags.iter().cloned().collect::<Vec<_>>())?;
Ok(tags)
}
fn filename(
filename: &str,
spec: Option<&LibSpec>,
) -> Result<(BTreeSet<String>, Option<String>), String> {
super::valid_path(filename)?;
if filename.contains('/') {
return Err("wheel filename must be a leaf name".into());
}
let fields = filename
.strip_suffix(".whl")
.ok_or("wheel filename lacks .whl")?
.split('-')
.collect::<Vec<_>>();
if !matches!(fields.len(), 5 | 6) {
return Err("invalid wheel filename fields".into());
}
if let Some(spec) = spec {
validate_identity(fields[0], fields[1], spec)?;
}
let build = if fields.len() == 6 {
validate_build(fields[2])?;
Some(fields[2].to_owned())
} else {
None
};
let tail = &fields[fields.len() - 3..];
Ok((expanded_tags(tail[0], tail[1], tail[2])?, build))
}
pub fn matches(filename_value: &str, selected_tag: &str) -> bool {
filename(filename_value, None).is_ok_and(|(tags, _)| tags.contains(selected_tag))
}
fn validate_build(value: &str) -> Result<(), String> {
if !value.as_bytes().first().is_some_and(u8::is_ascii_digit)
|| !value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
{
return Err("invalid wheel build tag".into());
}
Ok(())
}
fn validate_identity(name: &str, version: &str, spec: &LibSpec) -> Result<(), String> {
name.parse::<pep508_rs::PackageName>()
.map_err(|error| format!("invalid wheel distribution name: {error}"))?;
let identity: LibSpec = format!("pip:{name}@{version}").parse()?;
if identity.distribution_name() != spec.distribution_name()
|| version
.parse::<Version>()
.map_err(|error| error.to_string())?
!= spec
.version
.parse::<Version>()
.map_err(|error| error.to_string())?
{
return Err("wheel distribution identity differs from its lock".into());
}
Ok(())
}
fn headers(bytes: &[u8]) -> Result<BTreeMap<String, Vec<String>>, String> {
if bytes.len() > 16 * 1024 * 1024 {
return Err("wheel metadata exceeds 16 MiB".into());
}
let text = std::str::from_utf8(bytes).map_err(|_| "wheel metadata is not UTF-8")?;
let mut result = BTreeMap::<String, Vec<String>>::new();
let mut previous: Option<String> = None;
for line in text.lines() {
if line.is_empty() {
break;
}
if line.starts_with([' ', '\t']) {
let key = previous
.as_ref()
.ok_or("wheel metadata starts with a folded line")?;
let value = result.get_mut(key).unwrap().last_mut().unwrap();
value.push(' ');
value.push_str(line.trim());
continue;
}
let (key, value) = line
.split_once(':')
.ok_or("invalid wheel metadata header")?;
if key.is_empty()
|| !key
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|| value.contains('\0')
{
return Err("invalid wheel metadata header".into());
}
let key = key.to_ascii_lowercase();
result
.entry(key.clone())
.or_default()
.push(value.trim().into());
previous = Some(key);
}
Ok(result)
}
fn optional_header<'a>(
headers: &'a BTreeMap<String, Vec<String>>,
key: &str,
) -> Result<Option<&'a str>, String> {
match headers.get(key) {
None => Ok(None),
Some(values) if values.len() == 1 => Ok(Some(values[0].as_str())),
_ => Err(format!("wheel metadata repeats {key}")),
}
}
fn required_header<'a>(
headers: &'a BTreeMap<String, Vec<String>>,
key: &str,
) -> Result<&'a str, String> {
optional_header(headers, key)?.ok_or_else(|| format!("wheel metadata lacks {key}"))
}
impl Wheel {
pub(crate) fn validate_dependencies(
&self,
lib: &crate::libs::LockedLib,
markers: &pep508_rs::MarkerEnvironment,
) -> Result<(), String> {
let extras = lib
.spec
.name
.split_once('[')
.map(|(_, extras)| extras.trim_end_matches(']'))
.unwrap_or("")
.split(',')
.filter(|extra| !extra.is_empty())
.map(str::parse::<pep508_rs::ExtraName>)
.collect::<Result<Vec<_>, _>>()
.map_err(|error| error.to_string())?;
let supplied_extras = self
.extras
.iter()
.map(|extra| extra.parse::<pep508_rs::ExtraName>())
.collect::<Result<BTreeSet<_>, _>>()
.map_err(|error| error.to_string())?;
if extras.iter().any(|extra| !supplied_extras.contains(extra)) {
return Err("wheel does not provide a requested extra".into());
}
let document = serde_json::json!({"info":{"requires_dist":self.requirements}});
let expected = crate::libs::python_dependencies(&document, markers, &extras)?;
let mut names = BTreeSet::new();
for (name, rule) in expected {
if name == lib.spec.name
&& rule.contains(
&lib.spec
.version
.parse::<Version>()
.map_err(|error| error.to_string())?,
)
{
continue;
}
let dependency = lib
.dependencies
.iter()
.find(|dependency| dependency.spec.name == name)
.ok_or("wheel dependency is missing from the exact lock")?;
if !rule.contains(
&dependency
.spec
.version
.parse::<Version>()
.map_err(|error| error.to_string())?,
) {
return Err("locked wheel dependency does not satisfy METADATA".into());
}
names.insert(name);
}
if lib
.dependencies
.iter()
.any(|dependency| !names.contains(&dependency.spec.name))
{
return Err("wheel lock has an undeclared dependency".into());
}
if let Some(rule) = &self.requires_python
&& !rule
.parse::<pep440_rs::VersionSpecifiers>()
.map_err(|error| error.to_string())?
.contains(markers.python_full_version())
{
return Err("wheel Requires-Python excludes the signed runtime".into());
}
Ok(())
}
pub fn parse(
spec: &LibSpec,
filename_value: Option<&str>,
runtime_tags: &[String],
bytes: &[u8],
) -> Result<Self, String> {
validate_tags(runtime_tags)?;
let files = archive_files("whl", bytes)?;
let mut info = BTreeSet::new();
let mut data = BTreeSet::new();
for (path, _) in &files {
super::valid_path(path)?;
if let Some((root, _)) = path.split_once('/') {
if root.ends_with(".dist-info") {
info.insert(root);
}
if root.ends_with(".data") {
data.insert(root);
}
}
}
if info.len() != 1 {
return Err("wheel must contain exactly one dist-info directory".into());
}
let info = info.into_iter().next().unwrap();
let stem = info.strip_suffix(".dist-info").unwrap();
let (name, version) = stem
.rsplit_once('-')
.ok_or("invalid wheel dist-info directory")?;
validate_identity(name, version, spec)?;
if data.iter().any(|root| *root != format!("{stem}.data")) {
return Err("wheel data directory differs from its distribution identity".into());
}
let read = |name: &str| {
files
.iter()
.find(|(path, _)| path == &format!("{info}/{name}"))
.map(|(_, bytes)| bytes.as_slice())
.ok_or_else(|| format!("wheel lacks {name}"))
};
let metadata = headers(read("METADATA")?)?;
let metadata_version = required_header(&metadata, "metadata-version")?;
if metadata_version
.split_once('.')
.is_none_or(|(major, minor)| {
!matches!(major, "1" | "2") || minor.parse::<u32>().is_err()
})
{
return Err("unsupported wheel core metadata version".into());
}
validate_identity(
required_header(&metadata, "name")?,
required_header(&metadata, "version")?,
spec,
)?;
let wheel = headers(read("WHEEL")?)?;
let wheel_version = required_header(&wheel, "wheel-version")?;
if wheel_version
.split_once('.')
.is_none_or(|(major, minor)| major != "1" || minor.parse::<u32>().is_err())
{
return Err("unsupported Wheel-Version".into());
}
if !matches!(
required_header(&wheel, "root-is-purelib")?,
"true" | "false"
) {
return Err("invalid wheel Root-Is-Purelib".into());
}
let tags = wheel.get("tag").ok_or("wheel has no compatibility Tag")?;
validate_tags(tags)?;
if !tags.iter().any(|tag| runtime_tags.contains(tag)) {
return Err("wheel tags do not match the signed Python runtime".into());
}
if tags.iter().all(|tag| tag.ends_with("-any"))
&& files.iter().any(|(path, bytes)| {
bytes.starts_with(b"\x7fELF")
|| path.ends_with(".pyd")
|| path.ends_with(".dll")
|| path.ends_with(".dylib")
})
{
return Err("platform-independent wheel contains a native binary".into());
}
let build = optional_header(&wheel, "build")?;
if let Some(build) = build {
validate_build(build)?;
}
if let Some(value) = filename_value {
let (file_tags, file_build) = filename(value, Some(spec))?;
if file_tags != tags.iter().cloned().collect() || file_build.as_deref() != build {
return Err("wheel filename tags/build differ from WHEEL metadata".into());
}
}
validate_record(&files, info, read("RECORD")?)?;
let requirements = metadata.get("requires-dist").cloned().unwrap_or_default();
for requirement in &requirements {
requirement
.parse::<Requirement>()
.map_err(|error| format!("invalid wheel Requires-Dist: {error}"))?;
}
let extras = metadata.get("provides-extra").cloned().unwrap_or_default();
for extra in &extras {
extra
.parse::<pep508_rs::ExtraName>()
.map_err(|error| format!("invalid wheel Provides-Extra: {error}"))?;
}
let requires_python = optional_header(&metadata, "requires-python")?.map(str::to_owned);
if let Some(rule) = &requires_python {
rule.parse::<pep440_rs::VersionSpecifiers>()
.map_err(|error| error.to_string())?;
}
let stem = stem.to_owned();
Ok(Self {
files,
stem,
requirements,
extras,
requires_python,
})
}
/// Install into the interpreter's real prefix so sys.prefix, sysconfig,
/// standalone scripts and the isolated runner agree on resource locations.
pub fn install(
self,
interpreter: &str,
python_version: &str,
) -> Result<Vec<InstalledFile>, String> {
super::valid_path(interpreter)?;
let prefix = interpreter
.rsplit_once("/bin/")
.map(|(prefix, _)| prefix)
.ok_or("Python installation executable must live in its prefix/bin")?;
super::valid_path(prefix)?;
if python_version.split('.').count() != 2
|| python_version
.split('.')
.any(|part| part.parse::<u32>().is_err())
{
return Err("Python installation requires a major.minor version".into());
}
let library = format!("{prefix}/lib/python{python_version}/site-packages");
let name = self.stem.rsplit_once('-').unwrap().0;
let data_prefix = format!("{}.data/", self.stem);
let mut installed = BTreeSet::new();
let mut output = Vec::with_capacity(self.files.len());
let entry_points = self
.files
.iter()
.find(|(path, _)| path == &format!("{}.dist-info/entry_points.txt", self.stem))
.map(|(_, bytes)| scripts::entry_points(bytes, prefix, interpreter))
.transpose()?
.unwrap_or_default();
for (path, mut bytes) in self.files {
let (destination, executable) = if let Some(relative) = path.strip_prefix(&data_prefix)
{
let (scheme, relative) = relative
.split_once('/')
.ok_or("wheel data scheme has no file")?;
let destination = match scheme {
"purelib" | "platlib" => format!("{library}/{relative}"),
"headers" => {
format!("{prefix}/include/python{python_version}/{name}/{relative}")
}
"scripts" if !relative.contains('/') => format!("{prefix}/bin/{relative}"),
"data" => format!("{prefix}/{relative}"),
_ => return Err("invalid wheel data installation scheme".into()),
};
if scheme == "scripts"
&& (bytes.starts_with(b"#!python\n")
|| bytes.starts_with(b"#!pythonw\n")
|| bytes.starts_with(b"#!python\r\n")
|| bytes.starts_with(b"#!pythonw\r\n"))
{
let newline = bytes.iter().position(|byte| *byte == b'\n').unwrap();
bytes = [
format!("#!/worker/{interpreter} -I\n").as_bytes(),
&bytes[newline + 1..],
]
.concat();
}
(destination, scheme == "scripts")
} else {
(format!("{library}/{path}"), false)
};
super::valid_path(&destination)?;
if !installed.insert(destination.clone()) {
return Err("wheel installation schemes produce conflicting files".into());
}
output.push(InstalledFile {
path: destination,
bytes,
executable,
});
}
for file in entry_points {
if !installed.insert(file.path.clone()) {
return Err("wheel entry point conflicts with another installed script".into());
}
output.push(file);
}
let record_path = format!("{library}/{}.dist-info/RECORD", self.stem);
let mut record = csv::WriterBuilder::new()
.has_headers(false)
.from_writer(Vec::new());
for file in &output {
let base = library.split('/').collect::<Vec<_>>();
let target = file.path.split('/').collect::<Vec<_>>();
let common = base
.iter()
.zip(&target)
.take_while(|(left, right)| left == right)
.count();
let relative = format!(
"{}{}",
"../".repeat(base.len() - common),
target[common..].join("/")
);
let (hash, size) = if file.path == record_path {
(String::new(), String::new())
} else {
(
format!(
"sha256={}",
URL_SAFE_NO_PAD.encode(Sha256::digest(&file.bytes))
),
file.bytes.len().to_string(),
)
};
record
.write_record([relative, hash, size])
.map_err(|error| error.to_string())?;
}
output
.iter_mut()
.find(|file| file.path == record_path)
.expect("validated RECORD")
.bytes = record.into_inner().map_err(|error| error.to_string())?;
Ok(output)
}
}
fn validate_record(files: &[(String, Vec<u8>)], info: &str, bytes: &[u8]) -> Result<(), String> {
let entries = files
.iter()
.map(|(path, bytes)| (path.as_str(), bytes))
.collect::<BTreeMap<_, _>>();
let record_path = format!("{info}/RECORD");
let mut seen = BTreeSet::new();
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.flexible(false)
.from_reader(bytes);
for row in reader.records() {
let row = row.map_err(|error| format!("invalid wheel RECORD CSV: {error}"))?;
if row.len() != 3 {
return Err("wheel RECORD requires three fields".into());
}
let path = &row[0];
super::valid_path(path)?;
if !seen.insert(path.to_owned()) {
return Err("wheel RECORD repeats a file".into());
}
let content = entries
.get(path)
.ok_or("wheel RECORD references an absent file")?;
if path == record_path {
if !row[1].is_empty() || !row[2].is_empty() {
return Err("wheel RECORD cannot hash itself".into());
}
continue;
}
if path == format!("{info}/RECORD.jws") || path == format!("{info}/RECORD.p7s") {
return Err("wheel signatures must not be listed in RECORD".into());
}
let (algorithm, expected) = row[1]
.split_once('=')
.ok_or("wheel RECORD lacks a secure hash")?;
let actual = match algorithm {
"sha256" => URL_SAFE_NO_PAD.encode(Sha256::digest(content)),
"sha384" => URL_SAFE_NO_PAD.encode(Sha384::digest(content)),
"sha512" => URL_SAFE_NO_PAD.encode(Sha512::digest(content)),
_ => return Err("wheel RECORD hash must use SHA-256 or stronger SHA-2".into()),
};
if actual != expected || row[2].parse::<usize>().ok() != Some(content.len()) {
return Err("wheel RECORD hash or size does not match its file".into());
}
}
if !seen.contains(&record_path)
|| entries.keys().any(|path| {
!seen.contains(*path)
&& **path != format!("{info}/RECORD.jws")
&& **path != format!("{info}/RECORD.p7s")
})
{
return Err("wheel RECORD does not cover every file".into());
}
Ok(())
}
@@ -0,0 +1,154 @@
//! Validate extension ABI and the dynamic loader's closed, packaged search tree.
use super::super::elf::{Closure, object, origin_directories};
use goblin::elf::header;
use std::collections::{BTreeMap, BTreeSet};
/// `files` contains Worker-relative installation files and separately verified
/// OS ABI files at sandbox/lib/*. No host directory is searched or executed.
pub fn validate_native(
target: &str,
suffixes: &[String],
interpreter: &str,
files: &BTreeMap<String, &[u8]>,
wheel_files: &BTreeSet<String>,
) -> Result<(), String> {
let native = files
.iter()
.filter(|(path, _)| wheel_files.contains(*path))
.filter(|(path, bytes)| {
bytes.starts_with(b"\x7fELF")
|| path.ends_with(".so")
|| path.contains(".so.")
|| path.ends_with(".pyd")
|| path.ends_with(".dll")
|| path.ends_with(".dylib")
})
.collect::<Vec<_>>();
if native.is_empty() {
return Ok(());
}
let machine = match target {
"linux-x86_64" => header::EM_X86_64,
"linux-aarch64" => header::EM_AARCH64,
_ => return Err("native Python wheel target is unsupported".into()),
};
let closure = Closure {
files,
machine,
directories: vec!["sandbox/lib".into()],
};
let interpreter_elf = object(
interpreter,
files
.get(interpreter)
.ok_or("native wheel requires its packaged Python executable")?,
machine,
)?;
let inherited = if interpreter_elf.runpaths.is_empty() {
origin_directories(interpreter, &interpreter_elf.rpaths)?
} else {
vec![]
};
let (globals, _) = closure.follow(interpreter, vec![], &BTreeMap::new())?;
let mut entries = Vec::new();
for (path, bytes) in native {
if path.ends_with(".pyd") || path.ends_with(".dll") || path.ends_with(".dylib") {
return Err("native Python wheel contains a binary for a different OS".into());
}
let elf = object(path, bytes, machine)?;
if elf.header.e_type != header::ET_DYN || elf.interpreter.is_some() {
return Err("native wheel content must be a shared library, not an executable".into());
}
let module = elf.dynsyms.iter().any(|symbol| {
symbol.st_shndx != 0
&& elf
.dynstrtab
.get_at(symbol.st_name)
.is_some_and(|name| name.starts_with("PyInit_"))
});
let module = module || path.contains(".cpython-") || path.ends_with(".abi3.so");
if module {
let compatible = suffixes.iter().any(|suffix| {
path.ends_with(suffix)
&& (suffix != ".so"
|| !path.contains(".cpython-") && !path.ends_with(".abi3.so"))
});
if !compatible {
return Err(
"native Python extension suffix differs from the signed interpreter ABI".into(),
);
}
}
entries.push((path, module));
}
entries.sort_by_key(|(_, module)| !*module);
let mut reached = BTreeSet::new();
for (path, module) in entries {
// Vendored helpers inherit the importing extension's loader context;
// reaching one does not imply an independent direct dlopen entry.
if !module && reached.contains(path) {
continue;
}
let (_, closure_paths) = closure.follow(path, inherited.clone(), &globals)?;
reached.extend(closure_paths);
}
Ok(())
}
/// Compiler processes use the signed sysroot's standard library locations.
/// Reuse the ELF version/SONAME closure checks used by installed extensions.
pub(crate) fn validate_build_tools(
target: &str,
executables: &[String],
files: &BTreeMap<String, &[u8]>,
) -> Result<(), String> {
let (machine, multiarch, interpreter) = match target {
"linux-x86_64" => (
header::EM_X86_64,
"x86_64-linux-gnu",
"/lib64/ld-linux-x86-64.so.2",
),
"linux-aarch64" => (
header::EM_AARCH64,
"aarch64-linux-gnu",
"/lib/ld-linux-aarch64.so.1",
),
_ => return Err("build tools target is unsupported".into()),
};
let closure = Closure {
files,
machine,
directories: vec![
format!("rootfs/usr/lib/{multiarch}"),
"sandbox/lib".into(),
"rootfs/usr/lib".into(),
],
};
for path in executables {
let bytes = files.get(path).ok_or("build executable is missing")?;
if bytes.starts_with(b"#!") {
let first = bytes
.split(|byte| *byte == b'\n')
.next()
.unwrap_or_default();
if !matches!(
first,
b"#!/bin/sh"
| b"#!/usr/bin/sh"
| b"#!/usr/bin/env sh"
| b"#!/usr/bin/env node"
| b"#!/usr/bin/env python3"
) {
return Err("build tool script must name the signed shell directly".into());
}
continue;
}
let elf = object(path, bytes, machine)?;
if elf.interpreter.is_some_and(|value| value != interpreter) {
return Err("build tool interpreter differs from signed sandbox loader".into());
}
closure.follow(path, vec![], &BTreeMap::new())?;
}
Ok(())
}
@@ -0,0 +1,88 @@
//! Wheel console/gui entry points share the installed interpreter prefix.
use super::InstalledFile;
use std::collections::{BTreeMap, BTreeSet};
fn identifier(path: &str) -> bool {
!path.is_empty()
&& path.split('.').all(|part| {
let mut bytes = part.bytes();
bytes
.next()
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
})
}
pub(super) fn entry_points(
bytes: &[u8],
prefix: &str,
interpreter: &str,
) -> Result<Vec<InstalledFile>, String> {
let text = std::str::from_utf8(bytes).map_err(|_| "wheel entry points must be UTF-8")?;
let mut active = false;
let mut sections = BTreeSet::new();
let mut entries = BTreeMap::<String, String>::new();
let mut previous: Option<String> = None;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.is_empty() || trimmed.starts_with(['#', ';']) {
continue;
}
if let Some(section) = trimmed
.strip_prefix('[')
.and_then(|value| value.strip_suffix(']'))
{
active = matches!(section, "console_scripts" | "gui_scripts");
if active && !sections.insert(section.to_owned()) {
return Err("wheel repeats an entry-point group".into());
}
previous = None;
continue;
}
if !active {
continue;
}
if line.starts_with(char::is_whitespace) && !trimmed.contains('=') {
let name = previous
.as_ref()
.ok_or("wheel entry point has an orphan continuation")?;
let value = entries.get_mut(name).expect("previous entry");
value.push(' ');
value.push_str(trimmed);
continue;
}
let (name, value) = trimmed
.split_once('=')
.ok_or("invalid wheel script entry point")?;
let name = name.trim();
if name.is_empty()
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte))
|| matches!(name, "." | "..")
{
return Err("wheel script name must be a portable file name".into());
}
if entries
.insert(name.to_owned(), value.trim().to_owned())
.is_some()
{
return Err("wheel repeats a script name".into());
}
previous = Some(name.to_owned());
}
entries.into_iter().map(|(name,value)| {
let value = if let Some((target,extras))=value.split_once('[') {
let extras=extras.strip_suffix(']').ok_or("invalid entry-point extras")?;
for extra in extras.split(',') {
extra.trim().parse::<pep508_rs::ExtraName>().map_err(|error|format!("invalid entry-point extra: {error}"))?;
}
target.trim()
}else{value.trim()};
let (module,function)=value.split_once(':').ok_or("wheel console script must name a callable")?;
if !identifier(module) || !identifier(function) { return Err("wheel script has an invalid Python entry point".into()); }
let source=format!("#!/worker/{interpreter} -I\nimport importlib\nimport re\nimport sys\nentry = importlib.import_module({module:?})\nfor part in {function:?}.split('.'):\n entry = getattr(entry, part)\nsys.argv[0] = re.sub(r'(-script\\.pyw|\\.exe)?$', '', sys.argv[0])\nsys.exit(entry())\n");
Ok(InstalledFile {path:format!("{prefix}/bin/{name}"),bytes:source.into_bytes(),executable:true})
}).collect()
}
+183
View File
@@ -0,0 +1,183 @@
//! One runtime descriptor and file-closure contract for authoring and Worker preparation.
use std::collections::BTreeSet;
use std::path::Path;
use serde::Deserialize;
use super::{MAX_TREE_BYTES, MAX_TREE_FILES, go_build, valid_path};
use crate::libs::Ecosystem;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct Manifest {
format: String,
ecosystem: String,
pub(crate) target: String,
pub(crate) executable: Option<String>,
#[serde(default)]
pub(super) arguments: Vec<String>,
pub(super) build: Option<go_build::GoBuildManifest>,
#[serde(default)]
pub(crate) python_wheel_tags: Vec<String>,
#[serde(default)]
pub(crate) python_extension_suffixes: Vec<String>,
pub(crate) python_markers: Option<pep508_rs::MarkerEnvironment>,
}
impl Manifest {
pub(crate) fn executable(&self, path: &str) -> bool {
self.executable.as_deref() == Some(path)
|| self
.build
.as_ref()
.is_some_and(|build| build.executable(path))
}
}
pub(crate) fn validate(
files: &[(String, Vec<u8>)],
ecosystem: &Ecosystem,
target: &str,
) -> Result<Manifest, String> {
if files.len() > MAX_TREE_FILES {
return Err("managed runtime pack has too many files".into());
}
let mut paths = BTreeSet::new();
let mut total = 0usize;
for (path, bytes) in files {
valid_path(path)?;
if !paths.insert(path.as_str()) {
return Err("managed runtime pack repeats a file".into());
}
total = total
.checked_add(bytes.len())
.ok_or("managed runtime pack size overflow")?;
if total > MAX_TREE_BYTES {
return Err("managed runtime pack exceeds 256 MiB".into());
}
}
for path in &paths {
for parent in Path::new(path).ancestors().skip(1) {
if parent.to_str().is_some_and(|name| paths.contains(name)) {
return Err("managed runtime pack has a file/directory conflict".into());
}
}
}
let metadata = files
.iter()
.find(|(path, _)| path == "dever-runtime.json")
.ok_or("managed runtime pack has no dever-runtime.json")?;
let manifest: Manifest = serde_json::from_slice(&metadata.1)
.map_err(|error| format!("invalid managed runtime manifest: {error}"))?;
if manifest.format != "dever-worker-runtime-v1"
|| manifest.ecosystem != ecosystem.as_str()
|| manifest.target != target
{
return Err("managed runtime pack identity differs from its lock and target".into());
}
if ecosystem != &Ecosystem::Pip
&& (!manifest.python_wheel_tags.is_empty()
|| !manifest.python_extension_suffixes.is_empty()
|| manifest.python_markers.is_some())
{
return Err("non-Python runtime must not declare Python ABI metadata".into());
}
if ecosystem == &Ecosystem::Go {
if manifest.executable.is_some() || !manifest.arguments.is_empty() {
return Err("managed Go build pack must not declare a runtime interpreter".into());
}
manifest
.build
.as_ref()
.ok_or("managed Go Worker requires a locked compile/link build pack")?
.validate_files(files, target)?;
} else {
if manifest.build.is_some() {
return Err("managed interpreter pack must not declare Go build tools".into());
}
let executable = manifest
.executable
.as_deref()
.ok_or("managed runtime executable is missing")?;
valid_path(executable)?;
if !executable.starts_with("bin/")
|| manifest.arguments.len() > 64
|| manifest
.arguments
.iter()
.any(|argument| !argument.starts_with('-') || argument.contains('\0'))
{
return Err("managed runtime executable or fixed flags are invalid".into());
}
if ecosystem == &Ecosystem::Pip && manifest.arguments != ["-I", "-S", "-B"] {
return Err(
"managed Python runtime requires the fixed -I -S -B isolation flags".into(),
);
}
if ecosystem == &Ecosystem::Pip {
if manifest.python_markers.is_none() {
return Err("managed Python runtime lacks its marker environment".into());
}
super::python_wheel::validate_target(
target,
&manifest.python_wheel_tags,
manifest
.python_markers
.as_ref()
.expect("validated Python markers"),
)?;
let mut suffixes = BTreeSet::new();
if manifest.python_extension_suffixes.is_empty()
|| manifest.python_extension_suffixes.len() > 16
|| manifest.python_extension_suffixes.iter().any(|suffix| {
!suffix.starts_with('.')
|| !suffix.ends_with(".so")
|| !suffix
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
|| !suffixes.insert(suffix)
})
{
return Err(
"managed Python extension suffixes are missing, invalid or repeated".into(),
);
}
}
if !files
.iter()
.any(|(path, bytes)| path == executable && !bytes.is_empty())
{
return Err("managed runtime executable is missing from its pack".into());
}
for (path, bytes) in files {
if path == executable || bytes.starts_with(b"\x7fELF") {
validate_binary(path, bytes, target)?;
}
}
}
Ok(manifest)
}
pub(crate) fn validate_binary(path: &str, bytes: &[u8], target: &str) -> Result<(), String> {
use goblin::elf::{Elf, header};
let (machine, interpreter) = match target.parse::<crate::toolchain::BuildTarget>()? {
crate::toolchain::BuildTarget::LinuxX86_64 => {
(header::EM_X86_64, "/lib64/ld-linux-x86-64.so.2")
}
crate::toolchain::BuildTarget::LinuxAarch64 => {
(header::EM_AARCH64, "/lib/ld-linux-aarch64.so.1")
}
};
let elf =
Elf::parse(bytes).map_err(|error| format!("invalid runtime ELF '{path}': {error}"))?;
if !elf.is_64
|| !elf.little_endian
|| elf.header.e_machine != machine
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
|| elf.interpreter.is_some_and(|actual| actual != interpreter)
{
return Err(format!("runtime ELF '{path}' differs from target {target}"));
}
Ok(())
}