feat: publish Dever core with project scaffolding and versioned AI skills
This commit is contained in:
1 parent
70fa73cdc2
commit
ac458ffa7f
960 files changed
+221319
-1163
No files matched your search
@@ -0,0 +1,48 @@
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::Path;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use dever_core::hir::Program;
|
||||
|
||||
pub(super) fn check_baseline(program: &Program, root: &Path) -> Result<(), String> {
|
||||
let path = root.join("dever.api");
|
||||
let baseline = match fs::read_to_string(&path) {
|
||||
Ok(baseline) => baseline,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => {
|
||||
return Err(format!(
|
||||
"cannot read API baseline '{}': {error}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
};
|
||||
program
|
||||
.check_api(&baseline)
|
||||
.map_err(|error| format!("{}: {error}", path.display()))
|
||||
}
|
||||
|
||||
pub(super) fn write(program: &Program, output: Option<&Path>) -> Result<ExitCode, String> {
|
||||
let snapshot = program.api_snapshot();
|
||||
match output {
|
||||
None => io::stdout()
|
||||
.lock()
|
||||
.write_all(snapshot.as_bytes())
|
||||
.map_err(|error| error.to_string())?,
|
||||
Some(path) => {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path)
|
||||
.map_err(|error| {
|
||||
format!("cannot create API snapshot '{}': {error}", path.display())
|
||||
})?;
|
||||
file.write_all(snapshot.as_bytes())
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| {
|
||||
format!("cannot write API snapshot '{}': {error}", path.display())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(ExitCode::SUCCESS)
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
use std::ffi::OsString;
|
||||
use std::io::{self, Write};
|
||||
|
||||
fn main() {
|
||||
std::process::exit(run());
|
||||
}
|
||||
|
||||
fn run() -> i32 {
|
||||
let arguments: Vec<OsString> = std::env::args_os().skip(1).collect();
|
||||
if arguments.as_slice() == ["--dever-bootstrap-health"] {
|
||||
return 0;
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
if let [flag, root] = arguments.as_slice()
|
||||
&& flag == "--dever-install"
|
||||
{
|
||||
return match dever_cli::toolchain::bootstrap::install(std::path::Path::new(root)) {
|
||||
Ok(version) => {
|
||||
println!("installed Dever {version}");
|
||||
0
|
||||
}
|
||||
Err(error) => fail(error),
|
||||
};
|
||||
}
|
||||
let executable = match std::env::current_exe() {
|
||||
Ok(path) => path,
|
||||
Err(error) => return fail(format!("cannot locate dever launcher: {error}")),
|
||||
};
|
||||
match dever_cli::toolchain::execute(&executable, &arguments) {
|
||||
Ok(dever_cli::toolchain::CommandResult::Exited(code)) => code,
|
||||
Ok(dever_cli::toolchain::CommandResult::Message(message)) => {
|
||||
match writeln!(io::stdout().lock(), "{message}") {
|
||||
Ok(()) => 0,
|
||||
Err(error) => fail(format!("cannot write command output: {error}")),
|
||||
}
|
||||
}
|
||||
Err(error) => fail(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn fail(error: String) -> i32 {
|
||||
let _ = writeln!(io::stderr().lock(), "{error}");
|
||||
1
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
use std::env;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
if env::args_os().skip(1).collect::<Vec<_>>().as_slice() == ["--dever-bootstrap-health"] {
|
||||
return;
|
||||
}
|
||||
let mut arguments = env::args_os().skip(1);
|
||||
let Some(flag) = arguments.next() else {
|
||||
eprintln!("Usage: deverd --root <machine-toolchain-root>");
|
||||
std::process::exit(2);
|
||||
};
|
||||
if flag != "--root" {
|
||||
eprintln!("Usage: deverd --root <machine-toolchain-root>");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let Some(root) = arguments.next() else {
|
||||
eprintln!("deverd: --root requires a machine toolchain root");
|
||||
std::process::exit(2);
|
||||
};
|
||||
if arguments.next().is_some() {
|
||||
eprintln!("deverd: unexpected arguments");
|
||||
std::process::exit(2);
|
||||
}
|
||||
let layout = dever_cli::toolchain::Layout::new(PathBuf::from(root));
|
||||
let result = layout
|
||||
.initialize()
|
||||
.and_then(|()| layout.validate_machine_permissions())
|
||||
.and_then(|()| dever_cli::toolchain::serve(&layout));
|
||||
if let Err(error) = result {
|
||||
eprintln!("deverd: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
//! The default compiler pipeline: checked HIR -> LLVM object -> verified LLD
|
||||
//! inputs. Artifact lifetime, cache integrity and output publication stay in core.
|
||||
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use dever_backend_bridge::{BinaryResource, LinkKind, Target, emit_object_with_resources, link};
|
||||
use dever_core::hir::Program;
|
||||
use dever_core::llvm::ResourceModule;
|
||||
use dever_core::native::{EmbeddedResource, NativeProgram};
|
||||
use dever_core::source::SourceMap;
|
||||
use dever_runtime::config::{CompilationBindings, RuntimeProfile};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use dever_cli::toolchain::compilation::{CompileKind, CompileRequest};
|
||||
use dever_cli::toolchain::runtime_pack::RuntimePack;
|
||||
use dever_cli::toolchain::{BuildTarget, Layout, Version};
|
||||
|
||||
pub fn application(
|
||||
program: &Program,
|
||||
sources: &SourceMap,
|
||||
bindings: &CompilationBindings,
|
||||
resources: &[EmbeddedResource],
|
||||
target: BuildTarget,
|
||||
) -> Result<NativeProgram, String> {
|
||||
if let Some((layout, version)) = managed_compiler()? {
|
||||
let request = CompileRequest::new(
|
||||
version,
|
||||
CompileKind::Application,
|
||||
sources,
|
||||
Some(bindings.clone()),
|
||||
resources,
|
||||
target,
|
||||
)?;
|
||||
return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile(
|
||||
&layout, &request,
|
||||
)?);
|
||||
}
|
||||
let profile = bindings.profile();
|
||||
compile(profile, target, || {
|
||||
dever_core::llvm::emit_executable_with_resources(program, sources, resources)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn test_suite(program: &Program, sources: &SourceMap) -> Result<NativeProgram, String> {
|
||||
let target = BuildTarget::host()?;
|
||||
if let Some((layout, version)) = managed_compiler()? {
|
||||
let request =
|
||||
CompileRequest::new(version, CompileKind::TestSuite, sources, None, &[], target)?;
|
||||
return NativeProgram::from_verified_bytes(&dever_cli::toolchain::compile(
|
||||
&layout, &request,
|
||||
)?);
|
||||
}
|
||||
let profile = test_profile(program);
|
||||
compile(profile, target, || {
|
||||
dever_core::llvm::emit_test_executable(program, sources).map(|ir| ResourceModule {
|
||||
ir,
|
||||
resources: Vec::new(),
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn compiler_directory() -> Result<PathBuf, String> {
|
||||
let executable =
|
||||
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
|
||||
executable
|
||||
.parent()
|
||||
.map(Path::to_owned)
|
||||
.ok_or_else(|| "compiler has no parent directory".into())
|
||||
}
|
||||
|
||||
pub(super) fn managed_compiler() -> Result<Option<(Layout, String)>, String> {
|
||||
let executable =
|
||||
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
|
||||
let directory = executable
|
||||
.parent()
|
||||
.ok_or("compiler has no parent directory")?;
|
||||
let managed = executable.file_stem().and_then(|name| name.to_str()) == Some("dever-core")
|
||||
|| directory
|
||||
.parent()
|
||||
.and_then(Path::file_name)
|
||||
.is_some_and(|name| name == "versions");
|
||||
if !managed {
|
||||
return Ok(None);
|
||||
}
|
||||
let versions = directory
|
||||
.parent()
|
||||
.filter(|path| path.file_name().is_some_and(|name| name == "versions"))
|
||||
.ok_or(
|
||||
"managed compiler must be installed under <toolchain>/versions/<version>/dever-core",
|
||||
)?;
|
||||
if executable.file_stem().and_then(|name| name.to_str()) != Some("dever-core") {
|
||||
return Err("managed compiler must use the installed dever-core executable".into());
|
||||
}
|
||||
let version = directory
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or("invalid managed compiler version directory")?;
|
||||
Version::parse(version)?;
|
||||
if version != env!("CARGO_PKG_VERSION") {
|
||||
return Err("managed compiler version does not match its installation directory".into());
|
||||
}
|
||||
let root = versions
|
||||
.parent()
|
||||
.ok_or("managed compiler has no toolchain root")?;
|
||||
Ok(Some((Layout::new(root), version.to_owned())))
|
||||
}
|
||||
|
||||
fn test_profile(program: &Program) -> RuntimeProfile {
|
||||
RuntimeProfile {
|
||||
sqlite: program.tests().iter().any(|test| test.uses_database()),
|
||||
postgres: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn worker_compile(
|
||||
program: &Program,
|
||||
sources: &SourceMap,
|
||||
request: &CompileRequest,
|
||||
build: &Path,
|
||||
output: &Path,
|
||||
) -> Result<(), String> {
|
||||
let resources = request.embedded_resources();
|
||||
let (profile, ir) = match request.kind {
|
||||
CompileKind::Application => {
|
||||
let profile = program.validate_database_bindings(
|
||||
request
|
||||
.bindings
|
||||
.as_ref()
|
||||
.ok_or("application bindings are missing")?,
|
||||
)?;
|
||||
let ir =
|
||||
dever_core::llvm::emit_executable_with_resources(program, sources, &resources)?;
|
||||
(profile, ir)
|
||||
}
|
||||
CompileKind::TestSuite => (
|
||||
test_profile(program),
|
||||
ResourceModule {
|
||||
ir: dever_core::llvm::emit_test_executable(program, sources)?,
|
||||
resources: Vec::new(),
|
||||
},
|
||||
),
|
||||
};
|
||||
let pack = RuntimePack::load(&compiler_directory()?, profile, request.target)?;
|
||||
link_ir(&ir, &pack, build, output)
|
||||
}
|
||||
|
||||
fn compile<'a>(
|
||||
profile: RuntimeProfile,
|
||||
target: BuildTarget,
|
||||
emit: impl FnOnce() -> Result<ResourceModule<'a>, String>,
|
||||
) -> Result<NativeProgram, String> {
|
||||
let executable =
|
||||
std::env::current_exe().map_err(|error| format!("cannot locate compiler: {error}"))?;
|
||||
let directory = compiler_directory()?;
|
||||
let pack = RuntimePack::load(&directory, profile, target)?;
|
||||
let ir = emit()?;
|
||||
let compiler = dever_cli::toolchain::sha256_file(&executable)?;
|
||||
let mut identity = Sha256::new();
|
||||
for part in [
|
||||
"dever-llvm-executable-v1",
|
||||
&compiler,
|
||||
&pack.identity,
|
||||
target.platform(),
|
||||
&ir.ir,
|
||||
] {
|
||||
identity.update((part.len() as u64).to_le_bytes());
|
||||
identity.update(part.as_bytes());
|
||||
}
|
||||
dever_core::native::compile_artifact(
|
||||
&directory.join("cache/native"),
|
||||
&format!("{:x}", identity.finalize()),
|
||||
|build, output| link_ir(&ir, &pack, build, output),
|
||||
)
|
||||
}
|
||||
|
||||
fn link_ir(
|
||||
module: &ResourceModule<'_>,
|
||||
pack: &RuntimePack,
|
||||
build: &Path,
|
||||
output: &Path,
|
||||
) -> Result<(), String> {
|
||||
let target = Target::ALL
|
||||
.into_iter()
|
||||
.find(|target| target.triple() == pack.target.triple())
|
||||
.ok_or("native runtime target is not supported by the compiler bridge")?;
|
||||
let resources = module
|
||||
.resources
|
||||
.iter()
|
||||
.map(|resource| BinaryResource {
|
||||
symbol: &resource.symbol,
|
||||
bytes: resource.bytes,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let object = emit_object_with_resources(&module.ir, target, &resources)?;
|
||||
let path = build.join("application.o");
|
||||
fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.and_then(|mut file| file.write_all(&object))
|
||||
.map_err(|error| format!("cannot write application object: {error}"))?;
|
||||
let snapshot = pack.snapshot(build)?;
|
||||
let inputs = snapshot
|
||||
.start
|
||||
.into_iter()
|
||||
.chain(std::iter::once(path))
|
||||
.chain(snapshot.libraries)
|
||||
.chain(snapshot.end)
|
||||
.collect::<Vec<_>>();
|
||||
link(target, LinkKind::StaticExecutable, &inputs, output, None)
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ExitCode;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use dever_core::source::SourceMap;
|
||||
|
||||
pub(super) fn execute(root: &Path, sources: &SourceMap, check: bool) -> Result<ExitCode, String> {
|
||||
let mut changes = Vec::new();
|
||||
let mut diagnostics = Vec::new();
|
||||
for source in sources.files() {
|
||||
match dever_core::format::format(source) {
|
||||
Ok(formatted) if formatted != source.text() => {
|
||||
changes.push((source.path(), destination(root, source), formatted));
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(errors) => diagnostics.extend(errors),
|
||||
}
|
||||
}
|
||||
if !diagnostics.is_empty() {
|
||||
return Err(diagnostics
|
||||
.iter()
|
||||
.map(|error| error.render(sources))
|
||||
.collect());
|
||||
}
|
||||
if check {
|
||||
for (path, _, _) in &changes {
|
||||
writeln!(std::io::stdout().lock(), "{}", path.display())
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
return Ok(if changes.is_empty() {
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
ExitCode::FAILURE
|
||||
});
|
||||
}
|
||||
|
||||
// Prepare all replacements before changing any source. A syntax, permission or staging
|
||||
// failure leaves the original files intact; each final replacement is an atomic rename.
|
||||
validate_snapshot(root, sources)?;
|
||||
let mut staged = Vec::new();
|
||||
for (_, destination, formatted) in changes {
|
||||
staged.push(Replacement::prepare(destination, &formatted)?);
|
||||
}
|
||||
validate_snapshot(root, sources)?;
|
||||
for replacement in &mut staged {
|
||||
replacement.commit()?;
|
||||
}
|
||||
Ok(ExitCode::SUCCESS)
|
||||
}
|
||||
|
||||
fn validate_snapshot(root: &Path, sources: &SourceMap) -> Result<(), String> {
|
||||
// Reuse loading rules so added/deleted paths and new symlinks also invalidate the snapshot.
|
||||
let current =
|
||||
SourceMap::load_project(&root.join("module"), &root.join("test")).map_err(|errors| {
|
||||
errors
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
})?;
|
||||
if !current
|
||||
.files()
|
||||
.iter()
|
||||
.map(|source| source.path())
|
||||
.eq(sources.files().iter().map(|source| source.path()))
|
||||
{
|
||||
return Err(format!(
|
||||
"{}: source file set changed during formatting",
|
||||
root.display()
|
||||
));
|
||||
}
|
||||
for (current, source) in current.files().iter().zip(sources.files()) {
|
||||
if current.text() != source.text() {
|
||||
return Err(format!(
|
||||
"{}: source changed during formatting",
|
||||
destination(root, source).display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn destination(root: &Path, source: &dever_core::source::SourceFile) -> PathBuf {
|
||||
if source.is_test() {
|
||||
root.join(source.path())
|
||||
} else {
|
||||
root.join("module").join(source.path())
|
||||
}
|
||||
}
|
||||
|
||||
struct Replacement {
|
||||
destination: PathBuf,
|
||||
temporary: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Replacement {
|
||||
fn prepare(destination: PathBuf, formatted: &str) -> Result<Self, String> {
|
||||
static NEXT: AtomicUsize = AtomicUsize::new(0);
|
||||
let metadata =
|
||||
fs::metadata(&destination).map_err(|error| file_error(&destination, error))?;
|
||||
if metadata.permissions().readonly() {
|
||||
return Err(format!("{}: source is read-only", destination.display()));
|
||||
}
|
||||
let parent = destination.parent().expect("source files have a parent");
|
||||
let (temporary, mut file) = loop {
|
||||
let temporary = parent.join(format!(
|
||||
".dever-fmt-{}-{}.tmp",
|
||||
std::process::id(),
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
match OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&temporary)
|
||||
{
|
||||
Ok(file) => break (temporary, file),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(file_error(&destination, error)),
|
||||
}
|
||||
};
|
||||
let replacement = Self {
|
||||
destination,
|
||||
temporary: Some(temporary),
|
||||
};
|
||||
file.write_all(formatted.as_bytes())
|
||||
.and_then(|()| file.set_permissions(metadata.permissions()))
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| file_error(&replacement.destination, error))?;
|
||||
Ok(replacement)
|
||||
}
|
||||
|
||||
fn commit(&mut self) -> Result<(), String> {
|
||||
let temporary = self.temporary.as_ref().expect("replacement is pending");
|
||||
fs::rename(temporary, &self.destination)
|
||||
.map_err(|error| file_error(&self.destination, error))?;
|
||||
self.temporary = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Replacement {
|
||||
fn drop(&mut self) {
|
||||
if let Some(temporary) = &self.temporary {
|
||||
let _ = fs::remove_file(temporary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn file_error(path: &Path, error: std::io::Error) -> String {
|
||||
format!("{}: cannot format source: {error}", path.display())
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod libs;
|
||||
pub mod packages;
|
||||
pub mod project;
|
||||
pub mod toolchain;
|
||||
pub mod workers;
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,526 @@
|
||||
//! Explicit Lib preparation: shared owned staging and bounded tool execution.
|
||||
|
||||
mod npm;
|
||||
pub mod pack;
|
||||
pub(crate) mod process;
|
||||
mod python;
|
||||
|
||||
use super::{Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryRuntime, RuntimePack};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn require_execution_target(target: &str, operation: &str) -> Result<(), String> {
|
||||
if target != crate::toolchain::BuildTarget::host()?.platform() {
|
||||
return Err(format!(
|
||||
"{operation} for {target} requires prepared target output and a complete build receipt; prepare the signed target inputs in a matching isolated execution environment before the offline build"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Production supplies only authenticated release resources. Explicit author
|
||||
/// tests inject the same immutable inputs without creating a host-tool fallback.
|
||||
pub trait Inputs {
|
||||
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String>;
|
||||
fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String>;
|
||||
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String>;
|
||||
/// Author tools may display bounded hook output without retaining temporary
|
||||
/// staging trees or putting diagnostics in reproducible build identities.
|
||||
fn diagnostic(&self, _output: &[u8]) {}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PythonSystem {
|
||||
pub requires: Vec<String>,
|
||||
pub backend: String,
|
||||
pub backend_path: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NativeRejection {
|
||||
Format,
|
||||
Target,
|
||||
Closure,
|
||||
Unloadable,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Receipt {
|
||||
pub ecosystem: Ecosystem,
|
||||
pub roots: Vec<LibSpec>,
|
||||
pub sources: Vec<LockedArtifact>,
|
||||
pub graph: Option<super::npm::Environment>,
|
||||
pub runtime: RuntimePack,
|
||||
pub auxiliary_runtime: Option<RuntimePack>,
|
||||
pub native_entries: BTreeSet<String>,
|
||||
pub native_rejections: BTreeMap<String, NativeRejection>,
|
||||
pub python_markers: Option<pep508_rs::MarkerEnvironment>,
|
||||
pub tools: pack::Descriptor,
|
||||
pub frontend: String,
|
||||
pub python: Option<PythonSystem>,
|
||||
pub dynamic_requires: Vec<String>,
|
||||
pub dependencies: Box<LockFile>,
|
||||
pub config_settings: BTreeMap<String, serde_json::Value>,
|
||||
pub output: LockedArtifact,
|
||||
}
|
||||
|
||||
impl Receipt {
|
||||
pub fn identity(&self) -> Result<String, String> {
|
||||
serde_json::to_vec(self)
|
||||
.map(|bytes| super::sha256(&bytes))
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Product {
|
||||
pub bytes: Vec<u8>,
|
||||
pub filename: String,
|
||||
pub receipt: Receipt,
|
||||
}
|
||||
|
||||
pub struct Session<'a> {
|
||||
inputs: &'a dyn Inputs,
|
||||
active: RefCell<BTreeSet<String>>,
|
||||
products: RefCell<BTreeMap<String, Product>>,
|
||||
}
|
||||
|
||||
impl<'a> Session<'a> {
|
||||
pub fn new(inputs: &'a dyn Inputs) -> Self {
|
||||
Self {
|
||||
inputs,
|
||||
active: RefCell::new(BTreeSet::new()),
|
||||
products: RefCell::new(BTreeMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn receipts(&self, identities: &BTreeSet<String>) -> Result<Vec<Receipt>, String> {
|
||||
let receipts = self
|
||||
.products
|
||||
.borrow()
|
||||
.values()
|
||||
.map(|product| Ok((product.receipt.identity()?, product.receipt.clone())))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
identities
|
||||
.iter()
|
||||
.map(|identity| {
|
||||
receipts
|
||||
.get(identity)
|
||||
.cloned()
|
||||
.ok_or("prepared build receipt is missing".into())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn remember(&self, key: String, product: Product) -> Result<(), String> {
|
||||
let total = self
|
||||
.products
|
||||
.borrow()
|
||||
.values()
|
||||
.try_fold(product.bytes.len(), |total, cached| {
|
||||
total.checked_add(cached.bytes.len())
|
||||
})
|
||||
.ok_or("build candidate cache byte overflow")?;
|
||||
if total > crate::workers::MAX_TREE_BYTES {
|
||||
return Err("build candidate cache exceeds byte budget".into());
|
||||
}
|
||||
self.products.borrow_mut().insert(key, product);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize(receipts: &mut Vec<Receipt>) -> Result<(), String> {
|
||||
let mut indexed = BTreeMap::new();
|
||||
for receipt in receipts.drain(..) {
|
||||
indexed.insert(receipt.identity()?, receipt);
|
||||
}
|
||||
*receipts = indexed.into_values().collect();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn npm_outputs(lock: &LockFile) -> impl Iterator<Item = &Receipt> {
|
||||
lock.builds
|
||||
.iter()
|
||||
.filter(|receipt| receipt.ecosystem == Ecosystem::Npm)
|
||||
}
|
||||
|
||||
pub(crate) fn npm_receipt<'a>(
|
||||
lock: &'a LockFile,
|
||||
environment: &super::npm::Environment,
|
||||
) -> Result<Option<&'a Receipt>, String> {
|
||||
let Some(identity) = &environment.build else {
|
||||
return Ok(None);
|
||||
};
|
||||
lock.builds
|
||||
.iter()
|
||||
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
|
||||
.map(Some)
|
||||
.ok_or("npm installation receipt is missing".into())
|
||||
}
|
||||
|
||||
/// Offline consumption never invokes a frontend. Removing roots projects the
|
||||
/// already built graph and output; it does not rerun lifecycle side effects.
|
||||
pub fn npm_installation(
|
||||
lock: &LockFile,
|
||||
environment: &super::npm::Environment,
|
||||
bytes: &[u8],
|
||||
) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
|
||||
let receipt = npm_receipt(lock, environment)?.ok_or("npm installation has no build receipt")?;
|
||||
if bytes.len() as u64 != receipt.output.bytes || super::sha256(bytes) != receipt.output.sha256 {
|
||||
return Err("npm built installation differs from its locked artifact".into());
|
||||
}
|
||||
let graph = receipt
|
||||
.graph
|
||||
.as_ref()
|
||||
.ok_or("npm receipt lacks its exact graph")?;
|
||||
let files = super::archive_files("tgz", bytes)?;
|
||||
super::npm::validate_built_files(graph, &files)?;
|
||||
let candidates = files
|
||||
.iter()
|
||||
.filter(|(path, _)| path.ends_with(".node"))
|
||||
.map(|(path, _)| path.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let accounted = receipt
|
||||
.native_entries
|
||||
.iter()
|
||||
.chain(receipt.native_rejections.keys())
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
if candidates != accounted
|
||||
|| receipt
|
||||
.native_rejections
|
||||
.keys()
|
||||
.any(|path| receipt.native_entries.contains(path))
|
||||
{
|
||||
return Err("npm native admission does not cover its exact addon files".into());
|
||||
}
|
||||
for path in &receipt.native_entries {
|
||||
if !files.iter().any(|(name, bytes)| {
|
||||
name == path && path.ends_with(".node") && bytes.starts_with(b"\x7fELF")
|
||||
}) {
|
||||
return Err("npm locked native entry point is absent from output".into());
|
||||
}
|
||||
}
|
||||
let executable = super::registry::archive_executables("tgz", bytes)?;
|
||||
let retained = environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| node.path.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut output = Vec::new();
|
||||
for (path, bytes) in files {
|
||||
if path.ends_with(".node") && !receipt.native_entries.contains(&path) {
|
||||
continue;
|
||||
}
|
||||
if retained.contains(super::npm::installed_owner(&path)?.as_str()) {
|
||||
let mode = executable.contains(&path);
|
||||
output.push((path, bytes, mode));
|
||||
}
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
doctor_at(lock, 0)
|
||||
}
|
||||
|
||||
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if depth > 16 {
|
||||
return Err("build dependency depth exceeds limit".into());
|
||||
}
|
||||
let records = lock
|
||||
.builds
|
||||
.iter()
|
||||
.map(|receipt| Ok((receipt.identity()?, receipt)))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
if records.len() != lock.builds.len() {
|
||||
return Err("duplicate build receipt".into());
|
||||
}
|
||||
let mut used = BTreeSet::new();
|
||||
for lib in &lock.libs {
|
||||
if let Some(identity) = &lib.build {
|
||||
let receipt = records
|
||||
.get(identity)
|
||||
.ok_or("Python Lib has no exact build receipt")?;
|
||||
if lib.spec.ecosystem != Ecosystem::Pip
|
||||
|| receipt.ecosystem != Ecosystem::Pip
|
||||
|| receipt.runtime != lib.runtime
|
||||
|| !lib.artifacts.contains(&receipt.output)
|
||||
|| receipt.roots.len() != 1
|
||||
|| receipt.roots[0].distribution_name() != lib.spec.distribution_name()
|
||||
|| receipt.roots[0].version != lib.spec.version
|
||||
{
|
||||
return Err(
|
||||
"Python Lib build receipt differs from locked artifact identity".into(),
|
||||
);
|
||||
}
|
||||
used.insert(identity.clone());
|
||||
}
|
||||
}
|
||||
for environment in &lock.npm {
|
||||
if let Some(identity) = &environment.build {
|
||||
let receipt = records
|
||||
.get(identity)
|
||||
.ok_or("npm environment has no exact build receipt")?;
|
||||
let graph = receipt
|
||||
.graph
|
||||
.as_ref()
|
||||
.ok_or("npm build receipt lacks its graph")?;
|
||||
if receipt.ecosystem != Ecosystem::Npm
|
||||
|| receipt.roots != graph.roots
|
||||
|| graph.build.is_some()
|
||||
|| !environment
|
||||
.roots
|
||||
.iter()
|
||||
.all(|root| graph.roots.contains(root))
|
||||
|| !environment
|
||||
.instances
|
||||
.iter()
|
||||
.all(|node| graph.instances.contains(node))
|
||||
|| !environment.instances.iter().all(|node| {
|
||||
lock.libs.iter().any(|lib| {
|
||||
lib.spec == node.source.archive
|
||||
&& lib.runtime == receipt.runtime
|
||||
&& lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.all(|artifact| receipt.sources.contains(artifact))
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm installation build receipt differs from locked graph".into());
|
||||
}
|
||||
used.insert(identity.clone());
|
||||
}
|
||||
}
|
||||
if used.len() != records.len() {
|
||||
return Err("lock contains an unreferenced build receipt".into());
|
||||
}
|
||||
for receipt in records.values() {
|
||||
super::validate_runtime(&receipt.runtime)?;
|
||||
if receipt.tools.runtime_sha256 != receipt.runtime.sha256
|
||||
|| receipt.tools.ecosystem != receipt.ecosystem
|
||||
|| receipt.sources.is_empty()
|
||||
|| receipt.output.target != receipt.tools.target
|
||||
|| receipt.roots.is_empty()
|
||||
|| receipt.frontend != frontend(&receipt.ecosystem)?
|
||||
{
|
||||
return Err("build receipt has inconsistent tools, frontend, or target".into());
|
||||
}
|
||||
for artifact in receipt.sources.iter().chain([&receipt.output]) {
|
||||
super::validate_artifact(artifact)?;
|
||||
if artifact.target != receipt.tools.target {
|
||||
return Err("build artifact target differs from its tools".into());
|
||||
}
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Pip
|
||||
&& (receipt.python.is_none()
|
||||
|| receipt.graph.is_some()
|
||||
|| receipt.auxiliary_runtime.is_some())
|
||||
{
|
||||
return Err("Python build receipt lacks its backend configuration".into());
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Pip
|
||||
&& (!receipt.native_entries.is_empty() || !receipt.native_rejections.is_empty())
|
||||
{
|
||||
return Err("Python receipt cannot declare Node native entry points".into());
|
||||
}
|
||||
for path in receipt
|
||||
.native_entries
|
||||
.iter()
|
||||
.chain(receipt.native_rejections.keys())
|
||||
{
|
||||
crate::workers::valid_path(path)?;
|
||||
}
|
||||
let policy = RegistryRuntime {
|
||||
pack: receipt.runtime.clone(),
|
||||
target: receipt.tools.target.clone(),
|
||||
python_markers: receipt.python_markers.clone(),
|
||||
python_wheel_tags: vec![],
|
||||
npm_libc: None,
|
||||
};
|
||||
let auxiliary = receipt
|
||||
.auxiliary_runtime
|
||||
.as_ref()
|
||||
.map(|runtime| RegistryRuntime {
|
||||
pack: runtime.clone(),
|
||||
target: receipt.tools.target.clone(),
|
||||
python_markers: None,
|
||||
python_wheel_tags: vec![],
|
||||
npm_libc: None,
|
||||
});
|
||||
if let Some(runtime) = &receipt.auxiliary_runtime {
|
||||
super::validate_runtime(runtime)?;
|
||||
}
|
||||
receipt.tools.validate(&policy, auxiliary.as_ref())?;
|
||||
if receipt.ecosystem == Ecosystem::Npm
|
||||
&& (receipt.python.is_some()
|
||||
|| receipt.python_markers.is_some()
|
||||
|| !receipt.dynamic_requires.is_empty()
|
||||
|| !receipt.dependencies.libs.is_empty()
|
||||
|| receipt.graph.is_none())
|
||||
{
|
||||
return Err("npm build receipt contains Python backend state".into());
|
||||
}
|
||||
if !receipt.config_settings.is_empty() {
|
||||
return Err("unsupported source-build configuration settings".into());
|
||||
}
|
||||
if let Some(system) = &receipt.python {
|
||||
validate_requirements(receipt, system)?;
|
||||
}
|
||||
if !receipt.dependencies.workers.is_empty()
|
||||
|| !receipt.dependencies.packages.is_empty()
|
||||
|| receipt
|
||||
.dependencies
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
|
||||
{
|
||||
return Err("build dependency closure is not a separate Python environment".into());
|
||||
}
|
||||
doctor_at(&receipt.dependencies, depth + 1)?;
|
||||
// Full graph checks belong to the ordinary lock owner; recursion is
|
||||
// bounded above before invoking that owner on a dependency lock.
|
||||
let mut dependencies = (*receipt.dependencies).clone();
|
||||
dependencies.builds.clear();
|
||||
for lib in &mut dependencies.libs {
|
||||
lib.build = None;
|
||||
}
|
||||
super::doctor(&dependencies)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(), String> {
|
||||
let identifier = |path: &str| {
|
||||
!path.is_empty()
|
||||
&& path.split('.').all(|part| {
|
||||
let mut bytes = part.bytes();
|
||||
bytes
|
||||
.next()
|
||||
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
|
||||
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
|
||||
})
|
||||
};
|
||||
let (module, function) = system
|
||||
.backend
|
||||
.split_once(':')
|
||||
.map(|(module, function)| (module, Some(function)))
|
||||
.unwrap_or((&system.backend, None));
|
||||
if !identifier(module)
|
||||
|| function.is_some_and(|function| !identifier(function))
|
||||
|| system.requires.len() > 1024
|
||||
|| receipt.dynamic_requires.len() > 1024
|
||||
|| system.backend_path.len() > 64
|
||||
{
|
||||
return Err("build receipt has an invalid backend or requirement set".into());
|
||||
}
|
||||
for path in &system.backend_path {
|
||||
if path.is_empty() || path.starts_with('/') || path.contains(['\\', '\0']) {
|
||||
return Err("invalid backend-path in receipt".into());
|
||||
}
|
||||
let mut depth = 0usize;
|
||||
for part in path.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
depth = depth.checked_sub(1).ok_or("backend-path escapes source")?;
|
||||
}
|
||||
_ => depth += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
let markers = receipt
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build receipt lacks marker policy")?;
|
||||
let requirements = system
|
||||
.requires
|
||||
.iter()
|
||||
.chain(&receipt.dynamic_requires)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let document = serde_json::json!({"info":{"requires_dist":requirements}});
|
||||
let expected = super::python_dependencies(&document, markers, &[])?;
|
||||
let libraries = receipt
|
||||
.dependencies
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| (lib.spec.name.as_str(), lib))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut remaining = Vec::new();
|
||||
for (name, rule) in expected {
|
||||
let lib = libraries
|
||||
.get(name.as_str())
|
||||
.ok_or("Python build receipt omits a required dependency")?;
|
||||
let version = lib
|
||||
.spec
|
||||
.version
|
||||
.parse::<pep440_rs::Version>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !rule.contains(&version) {
|
||||
return Err("Python build dependency does not satisfy recorded requirements".into());
|
||||
}
|
||||
remaining.push(lib.spec.name.as_str());
|
||||
}
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(name) = remaining.pop() {
|
||||
if !reached.insert(name) {
|
||||
continue;
|
||||
}
|
||||
let lib = libraries
|
||||
.get(name)
|
||||
.ok_or("build dependency closure is incomplete")?;
|
||||
if lib.runtime != receipt.runtime
|
||||
|| lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.target != receipt.tools.target)
|
||||
{
|
||||
return Err("build dependency runtime or target differs from its receipt".into());
|
||||
}
|
||||
remaining.extend(lib.dependencies.iter().map(|edge| edge.spec.name.as_str()));
|
||||
}
|
||||
if reached.len() != libraries.len() {
|
||||
return Err("build dependency lock contains unrelated libraries".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Offline consumers bind recorded marker evaluation to the immutable runtime
|
||||
/// manifest they have just verified, never to a lock's self-declared policy.
|
||||
pub fn validate_python_policy(
|
||||
lock: &LockFile,
|
||||
lib: &super::LockedLib,
|
||||
markers: &pep508_rs::MarkerEnvironment,
|
||||
) -> Result<(), String> {
|
||||
if let Some(identity) = &lib.build {
|
||||
let receipt = lock
|
||||
.builds
|
||||
.iter()
|
||||
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
|
||||
.ok_or("Python build receipt missing")?;
|
||||
if receipt.python_markers.as_ref() != Some(markers) {
|
||||
return Err("Python build marker policy differs from signed runtime".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn frontend(ecosystem: &Ecosystem) -> Result<String, String> {
|
||||
match ecosystem {
|
||||
Ecosystem::Pip => Ok(super::sha256(include_bytes!("build/python.py"))),
|
||||
Ecosystem::Npm => Ok(super::sha256(
|
||||
&[
|
||||
include_bytes!("build/npm.js").as_slice(),
|
||||
include_bytes!("build/npm/probe.cjs").as_slice(),
|
||||
]
|
||||
.concat(),
|
||||
)),
|
||||
_ => Err("source-build frontend is not available for this ecosystem".into()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Fixed registry-package lifecycle frontend. Runs only in the build sandbox.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script');
|
||||
const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links');
|
||||
|
||||
async function main() {
|
||||
const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8'));
|
||||
const root = '/data/build/output/install';
|
||||
fs.mkdirSync('/data/build/output');
|
||||
fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true });
|
||||
fs.mkdirSync('/data/build/tmp');
|
||||
fs.mkdirSync('/data/build/home');
|
||||
Object.assign(process.env, {
|
||||
PATH: '/worker/runtime/bin:/usr/bin',
|
||||
CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar',
|
||||
PYTHON: request.python,
|
||||
npm_config_python: request.python,
|
||||
npm_config_nodedir: '/worker/npm',
|
||||
npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js',
|
||||
npm_config_build_from_source: 'true',
|
||||
npm_config_cache: '/data/build/npm-cache',
|
||||
TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0',
|
||||
HOME: '/data/build/home',
|
||||
});
|
||||
const nodes = new Map(request.instances.map(node => [node.path, node]));
|
||||
for (const node of request.instances) {
|
||||
const directory = path.join(root, node.path);
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8'));
|
||||
await binLinks({ path: directory, pkg, global: false, top: false, force: false });
|
||||
}
|
||||
const visited = new Set();
|
||||
const required = new Set(request.required);
|
||||
const failed = new Set();
|
||||
const nativeEntries = new Set();
|
||||
async function install(name) {
|
||||
if (visited.has(name)) return;
|
||||
visited.add(name);
|
||||
const node = nodes.get(name);
|
||||
try {
|
||||
for (const edge of node.edges) {
|
||||
if (edge.target && nodes.has(edge.target)) {
|
||||
await install(edge.target);
|
||||
if (failed.has(edge.target) && ['dependency', 'peer'].includes(edge.kind)) {
|
||||
throw new Error('required lifecycle dependency failed: ' + edge.target);
|
||||
}
|
||||
}
|
||||
}
|
||||
const directory = path.join(root, name);
|
||||
for (const event of ['preinstall', 'install', 'postinstall']) {
|
||||
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' });
|
||||
}
|
||||
const native = path.join(directory, 'build', 'Release');
|
||||
if (fs.existsSync(native)) {
|
||||
for (const file of fs.readdirSync(native)) {
|
||||
if (file.endsWith('.node')) {
|
||||
const addon = path.join(native, file);
|
||||
require(addon);
|
||||
nativeEntries.add(path.relative(root, addon));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (required.has(name)) throw error;
|
||||
process.stderr.write('optional npm lifecycle failed: ' + name + ': ' + String(error.message || error) + '\n');
|
||||
failed.add(name);
|
||||
}
|
||||
}
|
||||
for (const node of request.instances) await install(node.path);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of request.instances) {
|
||||
if (!failed.has(node.path) && node.edges.some(edge => ['dependency', 'peer'].includes(edge.kind) && failed.has(edge.target))) {
|
||||
if (required.has(node.path)) throw new Error('required lifecycle dependency failed: ' + node.path);
|
||||
failed.add(node.path);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
const retained = new Set();
|
||||
function retain(name) {
|
||||
if (retained.has(name) || failed.has(name)) return;
|
||||
retained.add(name);
|
||||
for (const edge of nodes.get(name).edges) {
|
||||
if (edge.target) retain(edge.target);
|
||||
}
|
||||
}
|
||||
for (const node of request.roots) retain(node);
|
||||
for (const node of request.instances) {
|
||||
if (!retained.has(node.path)) fs.rmSync(path.join(root, node.path), { recursive: true, force: true });
|
||||
}
|
||||
// Bin links are a build-time PATH input. The immutable Worker tree stores
|
||||
// package files only; it never persists symlinks created by installer hooks.
|
||||
for (const node of request.instances) {
|
||||
fs.rmSync(path.join(root, node.path, 'node_modules', '.bin'), { recursive: true, force: true });
|
||||
fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true });
|
||||
}
|
||||
fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true });
|
||||
fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
|
||||
}
|
||||
|
||||
main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; });
|
||||
@@ -0,0 +1,230 @@
|
||||
//! Registry lifecycle scripts use the signed npm frontend and exact instance graph.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
use std::io::Cursor;
|
||||
|
||||
use super::{Product, Receipt, Session};
|
||||
use crate::libs::{
|
||||
Ecosystem, LockFile, LockedArtifact, LockedLib, RegistryResolver, RegistryRuntime, npm, sha256,
|
||||
};
|
||||
|
||||
mod environment;
|
||||
use environment::{Environment, Output};
|
||||
|
||||
impl Session<'_> {
|
||||
pub(crate) fn npm(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
libraries: &mut Vec<LockedLib>,
|
||||
graph: &mut npm::Environment,
|
||||
runtime: &RegistryRuntime,
|
||||
) -> Result<(), String> {
|
||||
let mut lock = LockFile::new(libraries.clone())?;
|
||||
lock.npm.push(graph.clone());
|
||||
let archives = libraries
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
resolver.store.verify_exact(
|
||||
&artifact.sha256,
|
||||
artifact.bytes,
|
||||
&artifact.target,
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, &lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
if !npm::requires_build(&files)? {
|
||||
return Ok(());
|
||||
}
|
||||
let mut sources = libraries
|
||||
.iter()
|
||||
.flat_map(|lib| lib.artifacts.clone())
|
||||
.collect::<Vec<_>>();
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
let key = format!(
|
||||
"npm:{}:{}:{}",
|
||||
sha256(&serde_json::to_vec(graph).map_err(|error| error.to_string())?),
|
||||
runtime.pack.sha256,
|
||||
sha256(&serde_json::to_vec(&sources).map_err(|error| error.to_string())?)
|
||||
);
|
||||
if let Some(product) = self.products.borrow().get(&key) {
|
||||
*graph = product
|
||||
.receipt
|
||||
.graph
|
||||
.clone()
|
||||
.ok_or("cached npm graph missing")?;
|
||||
graph.build = Some(product.receipt.identity()?);
|
||||
retain_archives(libraries, graph);
|
||||
return Ok(());
|
||||
}
|
||||
let executables = npm::build_executables(graph, &archives)?;
|
||||
let environment = Environment::new(self.inputs, runtime)?;
|
||||
let output = environment.run(self.inputs, graph, &files, &executables)?;
|
||||
prune_failed(graph, &output.failed)?;
|
||||
retain_archives(libraries, graph);
|
||||
npm::validate_built_files(graph, &output.files)?;
|
||||
let bytes = archive(&output)?;
|
||||
let digest = resolver.store.publish(&bytes, &runtime.target)?;
|
||||
if digest != sha256(&bytes) {
|
||||
return Err("built npm cache identity mismatch".into());
|
||||
}
|
||||
let output_artifact = LockedArtifact {
|
||||
target: runtime.target.clone(),
|
||||
path: format!("build/npm/{digest}/installation.tgz"),
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: digest,
|
||||
};
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
roots: graph.roots.clone(),
|
||||
sources,
|
||||
graph: Some(graph.clone()),
|
||||
runtime: runtime.pack.clone(),
|
||||
auxiliary_runtime: Some(environment.python_runtime),
|
||||
native_entries: output.native,
|
||||
native_rejections: output.rejections,
|
||||
python_markers: None,
|
||||
tools: environment.descriptor,
|
||||
frontend: super::frontend(&Ecosystem::Npm)?,
|
||||
python: None,
|
||||
dynamic_requires: vec![],
|
||||
dependencies: Box::new(LockFile::new(vec![])?),
|
||||
config_settings: BTreeMap::new(),
|
||||
output: output_artifact,
|
||||
};
|
||||
graph.build = Some(receipt.identity()?);
|
||||
self.remember(
|
||||
key,
|
||||
Product {
|
||||
bytes,
|
||||
filename: "installation.tgz".into(),
|
||||
receipt,
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn archive(output: &Output) -> Result<Vec<u8>, String> {
|
||||
let encoder = flate2::GzBuilder::new()
|
||||
.mtime(0)
|
||||
.operating_system(255)
|
||||
.write(Vec::new(), flate2::Compression::default());
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for (path, bytes) in &output.files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(if output.executables.contains(path) {
|
||||
0o755
|
||||
} else {
|
||||
0o644
|
||||
});
|
||||
header.set_uid(0);
|
||||
header.set_gid(0);
|
||||
header.set_mtime(0);
|
||||
header.set_cksum();
|
||||
archive
|
||||
.append_data(&mut header, path, Cursor::new(bytes))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
archive
|
||||
.into_inner()
|
||||
.and_then(|encoder| encoder.finish())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn retain_archives(libraries: &mut Vec<LockedLib>, graph: &npm::Environment) {
|
||||
let used = graph
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| &node.source.archive)
|
||||
.collect::<BTreeSet<_>>();
|
||||
libraries.retain(|lib| used.contains(&lib.spec));
|
||||
}
|
||||
|
||||
fn required_nodes(graph: &npm::Environment) -> Result<BTreeSet<String>, String> {
|
||||
reachable(graph, true)
|
||||
}
|
||||
|
||||
fn reachable(graph: &npm::Environment, mandatory_only: bool) -> Result<BTreeSet<String>, String> {
|
||||
let mut remaining = graph
|
||||
.roots
|
||||
.iter()
|
||||
.map(|root| format!("node_modules/{}", root.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = remaining.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = graph
|
||||
.instances
|
||||
.iter()
|
||||
.find(|node| node.path == path)
|
||||
.ok_or("npm reachable instance missing")?;
|
||||
remaining.extend(
|
||||
node.edges
|
||||
.iter()
|
||||
.filter(|edge| {
|
||||
!mandatory_only || matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
|
||||
})
|
||||
.filter_map(|edge| edge.target.clone()),
|
||||
);
|
||||
}
|
||||
Ok(reached)
|
||||
}
|
||||
|
||||
fn prune_failed(graph: &mut npm::Environment, failed: &BTreeSet<String>) -> Result<(), String> {
|
||||
if failed
|
||||
.iter()
|
||||
.any(|path| !graph.instances.iter().any(|node| &node.path == path))
|
||||
{
|
||||
return Err("npm build reported an unknown failed instance".into());
|
||||
}
|
||||
let mut failed = failed.clone();
|
||||
loop {
|
||||
let before = failed.len();
|
||||
for node in &graph.instances {
|
||||
if node.edges.iter().any(|edge| {
|
||||
matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
|
||||
&& edge
|
||||
.target
|
||||
.as_ref()
|
||||
.is_some_and(|path| failed.contains(path))
|
||||
}) {
|
||||
failed.insert(node.path.clone());
|
||||
}
|
||||
}
|
||||
if before == failed.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !required_nodes(graph)?.is_disjoint(&failed) {
|
||||
return Err("required npm lifecycle failed".into());
|
||||
}
|
||||
graph.instances.retain(|node| !failed.contains(&node.path));
|
||||
for node in &mut graph.instances {
|
||||
for edge in &mut node.edges {
|
||||
if edge
|
||||
.target
|
||||
.as_ref()
|
||||
.is_some_and(|path| failed.contains(path))
|
||||
{
|
||||
edge.target = None;
|
||||
edge.omission = Some(npm::Omission::BuildFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
let reached = reachable(graph, false)?;
|
||||
graph.instances.retain(|node| reached.contains(&node.path));
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
//! One immutable Node/Python/tool environment for registry lifecycle execution.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
|
||||
use crate::libs::build::NativeRejection;
|
||||
use crate::libs::build::{Inputs, pack, process::Stage};
|
||||
use crate::libs::{Ecosystem, RegistryRuntime, RuntimePack, archive_files, npm, sha256};
|
||||
use crate::workers::runtime;
|
||||
|
||||
pub(super) struct Environment {
|
||||
files: BTreeMap<String, (Vec<u8>, bool)>,
|
||||
assets: Vec<(String, Vec<u8>)>,
|
||||
node: String,
|
||||
node_paths: BTreeSet<String>,
|
||||
python: String,
|
||||
pub python_runtime: RuntimePack,
|
||||
pub descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
pub(super) struct Output {
|
||||
pub files: Vec<(String, Vec<u8>)>,
|
||||
pub executables: BTreeSet<String>,
|
||||
pub failed: BTreeSet<String>,
|
||||
pub native: BTreeSet<String>,
|
||||
pub rejections: BTreeMap<String, NativeRejection>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ResultMetadata {
|
||||
failed: BTreeSet<String>,
|
||||
native: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
pub fn new(inputs: &dyn Inputs, runtime: &RegistryRuntime) -> Result<Self, String> {
|
||||
crate::libs::build::require_execution_target(&runtime.target, "npm lifecycle build")?;
|
||||
let (node, bytes) = inputs.runtime(Ecosystem::Npm)?;
|
||||
if node.pack != runtime.pack || sha256(&bytes) != runtime.pack.sha256 {
|
||||
return Err("npm build runtime differs from resolver".into());
|
||||
}
|
||||
let node_files = archive_files("tgz", &bytes)?;
|
||||
let node_manifest = runtime::validate(&node_files, &Ecosystem::Npm, &runtime.target)?;
|
||||
let node_paths = node_files
|
||||
.iter()
|
||||
.map(|(path, _)| format!("runtime/{path}"))
|
||||
.collect();
|
||||
let node = format!(
|
||||
"runtime/{}",
|
||||
node_manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.ok_or("Node executable missing")?
|
||||
);
|
||||
let (python, bytes) = inputs.runtime(Ecosystem::Pip)?;
|
||||
if sha256(&bytes) != python.pack.sha256 {
|
||||
return Err("auxiliary Python runtime hash mismatch".into());
|
||||
}
|
||||
let python_files = archive_files("tgz", &bytes)?;
|
||||
let python_manifest = runtime::validate(&python_files, &Ecosystem::Pip, &runtime.target)?;
|
||||
let python_executable = format!(
|
||||
"runtime/{}",
|
||||
python_manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.ok_or("build Python executable missing")?
|
||||
);
|
||||
let (descriptor, bytes) = inputs.tools(Ecosystem::Npm)?;
|
||||
descriptor.validate(runtime, Some(&python))?;
|
||||
let (manifest, tools) = pack::unpack(&bytes, &descriptor)?;
|
||||
let assets = inputs.assets()?;
|
||||
dever_sandbox::validate_assets(&assets)?;
|
||||
pack::validate_tools(&manifest, &tools, &assets)?;
|
||||
let mut files = BTreeMap::new();
|
||||
for (path, bytes) in node_files {
|
||||
let executable = node_manifest.executable(&path);
|
||||
files.insert(format!("runtime/{path}"), (bytes, executable));
|
||||
}
|
||||
for (path, bytes) in python_files {
|
||||
if path == "dever-runtime.json" {
|
||||
continue;
|
||||
}
|
||||
let executable = python_manifest.executable(&path);
|
||||
let path = format!("runtime/{path}");
|
||||
if let Some((existing, _)) = files.get(&path) {
|
||||
if existing != &bytes {
|
||||
return Err("Node and auxiliary Python runtime paths conflict".into());
|
||||
}
|
||||
} else {
|
||||
files.insert(path, (bytes, executable));
|
||||
}
|
||||
}
|
||||
for (path, bytes) in tools {
|
||||
let executable = manifest.executables.contains(&path);
|
||||
if files.insert(path, (bytes, executable)).is_some() {
|
||||
return Err("npm tools overlap runtimes".into());
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
files,
|
||||
assets,
|
||||
node,
|
||||
node_paths,
|
||||
python: python_executable,
|
||||
python_runtime: python.pack,
|
||||
descriptor,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
&self,
|
||||
inputs: &dyn Inputs,
|
||||
graph: &npm::Environment,
|
||||
files: &[(String, Vec<u8>)],
|
||||
executables: &BTreeSet<String>,
|
||||
) -> Result<Output, String> {
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
stage.write(&format!("input/{path}"), bytes, *executable)?;
|
||||
}
|
||||
for (path, bytes) in &self.assets {
|
||||
stage.write(
|
||||
&format!("sandbox/{path}"),
|
||||
bytes,
|
||||
dever_sandbox::asset_executable(path),
|
||||
)?;
|
||||
}
|
||||
for (path, bytes) in files {
|
||||
stage.write(
|
||||
&format!("input/install/{path}"),
|
||||
bytes,
|
||||
executables.contains(path),
|
||||
)?;
|
||||
}
|
||||
let request = serde_json::json!({
|
||||
"instances":graph.instances, "required":super::required_nodes(graph)?,
|
||||
"roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(),
|
||||
"python":format!("/worker/{}", self.python),
|
||||
});
|
||||
stage.write(
|
||||
"input/request.json",
|
||||
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
)?;
|
||||
stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?;
|
||||
let work = stage.0.join("work");
|
||||
fs::create_dir(&work).map_err(|error| error.to_string())?;
|
||||
let grants = [dever_sandbox::Grant {
|
||||
source: work.clone(),
|
||||
destination: "/data/build".into(),
|
||||
writable: true,
|
||||
}];
|
||||
let arguments = [input.join("frontend.js").into_os_string()];
|
||||
let executable = input.join(&self.node);
|
||||
let assets = stage.0.join("sandbox");
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities {
|
||||
network: false,
|
||||
process: true,
|
||||
},
|
||||
grants: &grants,
|
||||
};
|
||||
stage.run(
|
||||
launch.command_for_build(&input.join("rootfs"))?,
|
||||
"isolated npm lifecycle",
|
||||
&work,
|
||||
)?;
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
let mut outputs = super::super::python::output_files(&work.join("output"))?;
|
||||
let response: ResultMetadata = serde_json::from_slice(
|
||||
&outputs
|
||||
.remove("result.json")
|
||||
.ok_or("npm build result missing")?,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut files = Vec::new();
|
||||
let mut executables = BTreeSet::new();
|
||||
for (path, bytes) in outputs {
|
||||
let path = path
|
||||
.strip_prefix("install/")
|
||||
.ok_or("npm lifecycle output is outside its installation")?
|
||||
.to_owned();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if fs::symlink_metadata(work.join("output/install").join(&path))
|
||||
.map_err(|error| error.to_string())?
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o111
|
||||
!= 0
|
||||
{
|
||||
executables.insert(path.clone());
|
||||
}
|
||||
}
|
||||
files.push((path, bytes));
|
||||
}
|
||||
let mut output = Output {
|
||||
files,
|
||||
executables,
|
||||
failed: response.failed,
|
||||
native: response.native,
|
||||
rejections: BTreeMap::new(),
|
||||
};
|
||||
// Release the large compiler tree before staging the much smaller
|
||||
// runtime-only native admission sandbox.
|
||||
drop(stage);
|
||||
self.admit_native(inputs, &mut output)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn admit_native(&self, inputs: &dyn Inputs, output: &mut Output) -> Result<(), String> {
|
||||
let required = std::mem::take(&mut output.native);
|
||||
let candidates = output
|
||||
.files
|
||||
.iter()
|
||||
.filter(|(path, _)| path.ends_with(".node"))
|
||||
.collect::<Vec<_>>();
|
||||
if candidates.is_empty() {
|
||||
return if required.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("npm generated native entry point is missing".into())
|
||||
};
|
||||
}
|
||||
let machine = match self.descriptor.target.as_str() {
|
||||
"linux-x86_64" => goblin::elf::header::EM_X86_64,
|
||||
"linux-aarch64" => goblin::elf::header::EM_AARCH64,
|
||||
_ => return Err("Node native admission target is unsupported".into()),
|
||||
};
|
||||
let mut files = self
|
||||
.files
|
||||
.iter()
|
||||
.filter(|(path, _)| self.node_paths.contains(*path))
|
||||
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
files.extend(
|
||||
self.assets
|
||||
.iter()
|
||||
.map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())),
|
||||
);
|
||||
files.extend(
|
||||
output
|
||||
.files
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.clone(), bytes.as_slice())),
|
||||
);
|
||||
let stage = Stage::new()?;
|
||||
for (path, bytes) in &files {
|
||||
let relative = if path.starts_with("sandbox/") {
|
||||
path.clone()
|
||||
} else {
|
||||
format!("input/{path}")
|
||||
};
|
||||
let executable = path == &self.node
|
||||
|| path
|
||||
.strip_prefix("sandbox/")
|
||||
.is_some_and(dever_sandbox::asset_executable);
|
||||
stage.write(&relative, bytes, executable)?;
|
||||
}
|
||||
stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?;
|
||||
let input = stage.0.join("input");
|
||||
let executable = input.join(&self.node);
|
||||
let assets = stage.0.join("sandbox");
|
||||
for (path, bytes) in candidates {
|
||||
let rejection = match goblin::elf::Elf::parse(bytes) {
|
||||
Err(_) => Some(NativeRejection::Format),
|
||||
Ok(elf) if !elf.is_64 || !elf.little_endian || elf.header.e_machine != machine => {
|
||||
Some(NativeRejection::Target)
|
||||
}
|
||||
Ok(_) => {
|
||||
if let Err(error) = crate::workers::python_wheel::validate_native(
|
||||
&self.descriptor.target,
|
||||
&[],
|
||||
&self.node,
|
||||
&files,
|
||||
&BTreeSet::from([path.clone()]),
|
||||
) {
|
||||
inputs.diagnostic(
|
||||
format!("npm addon '{path}' has no packaged loader closure: {error}\n")
|
||||
.as_bytes(),
|
||||
);
|
||||
Some(NativeRejection::Closure)
|
||||
} else {
|
||||
let arguments = [
|
||||
input.join("probe.cjs").into_os_string(),
|
||||
input.join(path).into_os_string(),
|
||||
];
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities::default(),
|
||||
grants: &[],
|
||||
};
|
||||
let admitted = stage.probe(launch.command()?)?;
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
if admitted {
|
||||
None
|
||||
} else {
|
||||
Some(NativeRejection::Unloadable)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(reason) = rejection {
|
||||
output.rejections.insert(path.clone(), reason);
|
||||
} else {
|
||||
output.native.insert(path.clone());
|
||||
}
|
||||
}
|
||||
if !required.is_subset(&output.native) {
|
||||
return Err("source-built npm addon did not pass runtime-only native admission".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// No package entry modules are executed. Probe exactly one immutable addon in
|
||||
// an isolated runtime-only process, so addon globals cannot affect its peers.
|
||||
try {
|
||||
require(process.argv[2]);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
process.stderr.write(String(error.stack || error) + '\n');
|
||||
process.exit(10);
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
//! Signed, target-specific source-build tools. Never sourced from host PATH.
|
||||
|
||||
use crate::libs::{Ecosystem, RegistryRuntime};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub const MAX_PACK_BYTES: usize = 256 * 1024 * 1024;
|
||||
pub const MAX_EXPANDED_BYTES: usize = 512 * 1024 * 1024;
|
||||
pub const MAX_FILES: usize = 65_536;
|
||||
|
||||
pub type PackFiles = Vec<(String, Vec<u8>)>;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Manifest {
|
||||
pub format: String,
|
||||
pub ecosystem: Ecosystem,
|
||||
pub target: String,
|
||||
pub runtime_sha256: String,
|
||||
pub python_runtime_sha256: Option<String>,
|
||||
pub executables: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Descriptor {
|
||||
pub format: String,
|
||||
pub ecosystem: Ecosystem,
|
||||
pub target: String,
|
||||
pub runtime_sha256: String,
|
||||
pub python_runtime_sha256: Option<String>,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
fn hash(value: &str) -> Result<(), String> {
|
||||
if value.len() != 64
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err("build pack identity must be canonical SHA-256".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Descriptor {
|
||||
pub fn validate(
|
||||
&self,
|
||||
runtime: &RegistryRuntime,
|
||||
python: Option<&RegistryRuntime>,
|
||||
) -> Result<(), String> {
|
||||
if self.format != "dever-registry-build-v1"
|
||||
|| self.ecosystem == Ecosystem::Go
|
||||
|| self.target != runtime.target
|
||||
|| self.runtime_sha256 != runtime.pack.sha256
|
||||
{
|
||||
return Err("signed build pack does not match its runtime and target".into());
|
||||
}
|
||||
hash(&self.sha256)?;
|
||||
hash(&self.runtime_sha256)?;
|
||||
match (&self.ecosystem, &self.python_runtime_sha256, python) {
|
||||
(Ecosystem::Pip, None, _) => Ok(()),
|
||||
(Ecosystem::Npm, Some(identity), Some(python))
|
||||
if identity == &python.pack.sha256 && python.target == self.target =>
|
||||
{
|
||||
hash(identity)
|
||||
}
|
||||
_ => Err("build pack has no exact auxiliary Python runtime binding".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub fn validate(
|
||||
&self,
|
||||
files: &[(String, Vec<u8>)],
|
||||
descriptor: &Descriptor,
|
||||
) -> Result<(), String> {
|
||||
if self.format != "dever-lib-build-v1"
|
||||
|| self.ecosystem != descriptor.ecosystem
|
||||
|| self.target != descriptor.target
|
||||
|| self.runtime_sha256 != descriptor.runtime_sha256
|
||||
|| self.python_runtime_sha256 != descriptor.python_runtime_sha256
|
||||
{
|
||||
return Err("build pack manifest differs from its signed descriptor".into());
|
||||
}
|
||||
let total = files
|
||||
.iter()
|
||||
.try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len()))
|
||||
.ok_or("build pack expanded size overflow")?;
|
||||
if files.len() > MAX_FILES || total > MAX_EXPANDED_BYTES {
|
||||
return Err("build pack exceeds file or byte budget".into());
|
||||
}
|
||||
let mut names = BTreeSet::new();
|
||||
for (path, _) in files {
|
||||
crate::workers::valid_path(path)?;
|
||||
if path != "dever-build.json"
|
||||
&& !path.starts_with("rootfs/usr/")
|
||||
&& !(self.ecosystem == Ecosystem::Pip && path.starts_with("runtime/include/"))
|
||||
&& !(self.ecosystem == Ecosystem::Npm
|
||||
&& (path.starts_with("npm/include/node/")
|
||||
|| path.starts_with("npm/frontend/node_modules/")))
|
||||
{
|
||||
return Err(format!(
|
||||
"build pack path '{path}' is outside its fixed tool layout"
|
||||
));
|
||||
}
|
||||
if !names.insert(path.as_str()) {
|
||||
return Err("build pack repeats a path".into());
|
||||
}
|
||||
}
|
||||
let mut executable_names = BTreeSet::new();
|
||||
for path in &self.executables {
|
||||
if !names.contains(path.as_str())
|
||||
|| !executable_names.insert(path.as_str())
|
||||
|| !(path.starts_with("rootfs/usr/")
|
||||
|| self.ecosystem == Ecosystem::Npm
|
||||
&& path.starts_with("npm/frontend/node_modules/"))
|
||||
{
|
||||
return Err(
|
||||
"build executable is absent, repeated, or outside its tool root".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
for name in ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] {
|
||||
if !executable_names.contains(format!("rootfs/usr/bin/{name}").as_str()) {
|
||||
return Err(format!("build pack is missing required tool '{name}'"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unpack(bytes: &[u8], descriptor: &Descriptor) -> Result<(Manifest, PackFiles), String> {
|
||||
if bytes.len() > MAX_PACK_BYTES || crate::libs::sha256(bytes) != descriptor.sha256 {
|
||||
return Err("build pack exceeds its budget or SHA-256 differs".into());
|
||||
}
|
||||
let files = super::super::registry::tar_files(bytes, "tgz", MAX_EXPANDED_BYTES as u64)?;
|
||||
let metadata = files
|
||||
.iter()
|
||||
.find(|(path, _)| path == "dever-build.json")
|
||||
.ok_or("build pack is missing dever-build.json")?;
|
||||
let manifest: Manifest = serde_json::from_slice(&metadata.1)
|
||||
.map_err(|error| format!("invalid build pack manifest: {error}"))?;
|
||||
manifest.validate(&files, descriptor)?;
|
||||
Ok((manifest, files))
|
||||
}
|
||||
|
||||
pub fn validate_tools(
|
||||
manifest: &Manifest,
|
||||
files: &[(String, Vec<u8>)],
|
||||
assets: &[(String, Vec<u8>)],
|
||||
) -> Result<(), String> {
|
||||
let mut supplied = files
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (path, bytes) in assets {
|
||||
supplied.insert(format!("sandbox/{path}"), bytes);
|
||||
}
|
||||
crate::workers::python_wheel::native::validate_build_tools(
|
||||
&manifest.target,
|
||||
&manifest.executables,
|
||||
&supplied,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//! One lifecycle for every managed compiler and package build hook.
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(300);
|
||||
const MAX_OUTPUT: u64 = 16 * 1024 * 1024;
|
||||
static NEXT_STAGE: AtomicU64 = AtomicU64::new(0);
|
||||
mod budget;
|
||||
|
||||
pub(crate) struct Stage(pub(crate) PathBuf);
|
||||
|
||||
impl Stage {
|
||||
pub(crate) fn new() -> Result<Self, String> {
|
||||
for _ in 0..32 {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"dever-lib-build-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_STAGE.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
#[cfg(unix)]
|
||||
let created = {
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
builder.mode(0o700);
|
||||
builder.create(&path)
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let created = fs::create_dir(&path);
|
||||
match created {
|
||||
Ok(()) => return Ok(Self(path)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(format!("cannot create private build staging: {error}")),
|
||||
}
|
||||
}
|
||||
Err("cannot allocate private build staging".into())
|
||||
}
|
||||
|
||||
pub(crate) fn write(
|
||||
&self,
|
||||
relative: &str,
|
||||
bytes: &[u8],
|
||||
executable: bool,
|
||||
) -> Result<PathBuf, String> {
|
||||
crate::workers::valid_path(relative)?;
|
||||
let path = self.0.join(relative);
|
||||
fs::create_dir_all(path.parent().expect("validated relative path"))
|
||||
.map_err(|error| format!("cannot create build input directory: {error}"))?;
|
||||
// This tree is private and never a writable child mount. A second write
|
||||
// must not silently replace a previously verified input.
|
||||
use std::io::Write;
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot stage build input: {error}"))?;
|
||||
file.write_all(bytes)
|
||||
.map_err(|error| format!("cannot write build input: {error}"))?;
|
||||
#[cfg(unix)]
|
||||
if executable {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
file.set_permissions(fs::Permissions::from_mode(0o700))
|
||||
.map_err(|error| format!("cannot mark build tool executable: {error}"))?;
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub(crate) fn run(
|
||||
&self,
|
||||
command: Command,
|
||||
operation: &str,
|
||||
writable: &Path,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let status = self.execute(command, operation, Some(writable))?;
|
||||
if !status.success() {
|
||||
return Err(format!(
|
||||
"{operation} failed: {}",
|
||||
String::from_utf8_lossy(&self.diagnostics()?)
|
||||
));
|
||||
}
|
||||
fs::read(self.0.join("tool-stdout"))
|
||||
.map_err(|error| format!("cannot read {operation} output: {error}"))
|
||||
}
|
||||
|
||||
pub(crate) fn probe(&self, command: Command) -> Result<bool, String> {
|
||||
match self
|
||||
.execute(command, "isolated Node addon probe", None)?
|
||||
.code()
|
||||
{
|
||||
Some(0) => Ok(true),
|
||||
Some(10) => Ok(false),
|
||||
_ => Err(format!(
|
||||
"Node addon probe failed outside its loader check: {}",
|
||||
String::from_utf8_lossy(&self.diagnostics()?)
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&self,
|
||||
mut command: Command,
|
||||
operation: &str,
|
||||
writable: Option<&Path>,
|
||||
) -> Result<std::process::ExitStatus, String> {
|
||||
// Captures are outside every child-writable grant. Namespace teardown
|
||||
// reaps descendants when the owned bootstrap is killed on any exit path.
|
||||
let stdout = self.0.join("tool-stdout");
|
||||
let stderr = self.0.join("tool-stderr");
|
||||
let child = command
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(
|
||||
File::create(&stdout).map_err(|error| error.to_string())?,
|
||||
))
|
||||
.stderr(Stdio::from(
|
||||
File::create(&stderr).map_err(|error| error.to_string())?,
|
||||
))
|
||||
.spawn()
|
||||
.map_err(|error| format!("cannot execute {operation}: {error}"))?;
|
||||
let mut child = OwnedChild(child);
|
||||
let deadline = Instant::now() + DEADLINE;
|
||||
let mut next_scratch_check = Instant::now();
|
||||
let within_budget = || -> Result<(), String> {
|
||||
for path in [&stdout, &stderr] {
|
||||
if fs::metadata(path).map_err(|error| error.to_string())?.len() > MAX_OUTPUT {
|
||||
return Err(format!("{operation} output exceeds byte limit"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let status = loop {
|
||||
within_budget()?;
|
||||
if Instant::now() >= next_scratch_check {
|
||||
if let Some(path) = writable {
|
||||
budget::check(path)?;
|
||||
}
|
||||
next_scratch_check = Instant::now() + Duration::from_millis(250);
|
||||
}
|
||||
if let Some(status) = child
|
||||
.0
|
||||
.try_wait()
|
||||
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
|
||||
{
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("{operation} exceeded its five-minute deadline"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(25));
|
||||
};
|
||||
within_budget()?;
|
||||
if let Some(path) = writable {
|
||||
budget::check(path)?;
|
||||
}
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub(crate) fn diagnostics(&self) -> Result<Vec<u8>, String> {
|
||||
let mut bytes = Vec::new();
|
||||
for name in ["tool-stdout", "tool-stderr"] {
|
||||
File::open(self.0.join(name))
|
||||
.and_then(|file| file.take(MAX_OUTPUT).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read build diagnostic: {error}"))?;
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Stage {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct OwnedChild(Child);
|
||||
|
||||
impl Drop for OwnedChild {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
//! Periodic scratch accounting is separate from immutable runtime/tool inputs.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
const MAX_BYTES: u64 = 512 * 1024 * 1024;
|
||||
const MAX_ENTRIES: usize = 65_536;
|
||||
const MAX_DEPTH: usize = 128;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Budget {
|
||||
bytes: u64,
|
||||
entries: usize,
|
||||
}
|
||||
|
||||
pub(super) fn check(root: &Path) -> Result<(), String> {
|
||||
visit(root, 0, &mut Budget::default()).map_err(|error| format!("build scratch budget: {error}"))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> {
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
if depth > MAX_DEPTH {
|
||||
return Err("directory depth exceeds limit".into());
|
||||
}
|
||||
// An open directory pins the inode throughout enumeration. Child-created
|
||||
// symlinks and rename races cannot redirect accounting outside this tree.
|
||||
let directory = match fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::DIRECTORY).bits() as i32)
|
||||
.open(path)
|
||||
{
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(format!("cannot pin scratch directory: {error}")),
|
||||
};
|
||||
let pinned = format!("/proc/self/fd/{}", directory.as_raw_fd());
|
||||
for entry in fs::read_dir(pinned).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let metadata = match entry.metadata() {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error.to_string()),
|
||||
};
|
||||
budget.entries += 1;
|
||||
if budget.entries > MAX_ENTRIES {
|
||||
return Err("entry count exceeds limit".into());
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
visit(&entry.path(), depth + 1, budget)?;
|
||||
} else {
|
||||
budget.bytes = budget
|
||||
.bytes
|
||||
.checked_add(metadata.len())
|
||||
.ok_or("byte count overflow")?;
|
||||
if budget.bytes > MAX_BYTES {
|
||||
return Err("bytes exceed 512 MiB limit".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> {
|
||||
Err("managed build scratch accounting requires its Linux sandbox".into())
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Fixed PEP 517 frontend, executed only inside the managed build sandbox."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
|
||||
def build_system(source):
|
||||
project = source / "pyproject.toml"
|
||||
config = tomllib.loads(project.read_text("utf-8")) if project.exists() else {}
|
||||
system = config.get("build-system")
|
||||
if system is None:
|
||||
system = {"requires": ["setuptools>=40.8.0"], "build-backend": "setuptools.build_meta:__legacy__"}
|
||||
if not isinstance(system, dict) or not isinstance(system.get("requires"), list):
|
||||
raise ValueError("build-system.requires must be an array")
|
||||
requires = system["requires"]
|
||||
if len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
|
||||
raise ValueError("invalid static build requirements")
|
||||
backend = system.get("build-backend", "setuptools.build_meta:__legacy__")
|
||||
identifier = r"[A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*"
|
||||
if not isinstance(backend, str) or not re.fullmatch(identifier + "(?::" + identifier + ")?", backend):
|
||||
raise ValueError("invalid build-backend entry point")
|
||||
paths = system.get("backend-path", [])
|
||||
if not isinstance(paths, list) or len(paths) > 64:
|
||||
raise ValueError("invalid backend-path")
|
||||
for name in paths:
|
||||
if not isinstance(name, str) or not name or Path(name).is_absolute() or "\\" in name:
|
||||
raise ValueError("backend-path must be source-relative")
|
||||
resolved = (source / name).resolve()
|
||||
if not resolved.is_relative_to(source.resolve()) or not resolved.is_dir():
|
||||
raise ValueError("backend-path escapes the source tree")
|
||||
return {"requires": requires, "backend": backend, "backend_path": paths}
|
||||
|
||||
|
||||
def backend_object(source, system):
|
||||
paths = [(source / path).resolve() for path in system["backend_path"]]
|
||||
sys.path[:0] = [str(path) for path in paths]
|
||||
module_name, _, object_name = system["backend"].partition(":")
|
||||
module = importlib.import_module(module_name)
|
||||
if paths:
|
||||
origin = Path(module.__file__).resolve()
|
||||
if not any(origin.is_relative_to(path) for path in paths):
|
||||
raise ValueError("in-tree backend was loaded outside backend-path")
|
||||
backend = module
|
||||
for part in object_name.split(".") if object_name else []:
|
||||
backend = getattr(backend, part)
|
||||
return backend
|
||||
|
||||
|
||||
def leaf(value, suffix):
|
||||
if not isinstance(value, str) or Path(value).name != value or not value.endswith(suffix):
|
||||
raise ValueError("build hook returned an invalid output name")
|
||||
return value
|
||||
|
||||
|
||||
def execute(request, source, output):
|
||||
system = build_system(source)
|
||||
if request["phase"] == "inspect":
|
||||
return system
|
||||
if system != request["system"]:
|
||||
raise ValueError("source build configuration changed between hooks")
|
||||
backend = backend_object(source, system)
|
||||
settings = request["config_settings"]
|
||||
if request["phase"] == "requires":
|
||||
hook = getattr(backend, "get_requires_for_build_wheel", None)
|
||||
requires = hook(settings) if hook else []
|
||||
if not isinstance(requires, list) or len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
|
||||
raise ValueError("invalid dynamic build requirements")
|
||||
return {"requires": requires}
|
||||
if request["phase"] == "metadata":
|
||||
metadata = output / "metadata"
|
||||
metadata.mkdir()
|
||||
hook = getattr(backend, "prepare_metadata_for_build_wheel", None)
|
||||
prepared = metadata / leaf(hook(str(metadata), settings), ".dist-info") if hook else None
|
||||
if prepared is not None and (not prepared.is_dir() or prepared.is_symlink()):
|
||||
raise ValueError("metadata hook did not create its declared directory")
|
||||
return {"metadata": str(prepared.relative_to(output)) if prepared else None}
|
||||
prepared = None
|
||||
if request.get("metadata") is not None:
|
||||
shutil.copytree("/worker/prepared", output / "prepared")
|
||||
prepared = output / "prepared" / request["metadata"]
|
||||
if not prepared.resolve().is_relative_to((output / "prepared").resolve()):
|
||||
raise ValueError("prepared metadata path escapes its snapshot")
|
||||
wheels = output / "wheels"
|
||||
wheels.mkdir()
|
||||
name = leaf(backend.build_wheel(str(wheels), settings, str(prepared) if prepared else None), ".whl")
|
||||
return {"filename": name}
|
||||
|
||||
|
||||
def main():
|
||||
request = json.loads(Path("/worker/request.json").read_text("utf-8"))
|
||||
work = Path("/data/build")
|
||||
source = work / "source"
|
||||
shutil.copytree("/worker/source", source)
|
||||
output = work / "output"
|
||||
output.mkdir()
|
||||
(work / "tmp").mkdir()
|
||||
# These values are part of the fixed frontend, never project/host settings.
|
||||
os.environ.update({
|
||||
"PATH": "/worker/runtime/bin:/usr/bin",
|
||||
"CC": "/usr/bin/cc", "CXX": "/usr/bin/c++",
|
||||
"AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib",
|
||||
"LDSHARED": "/usr/bin/cc -shared",
|
||||
"CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.",
|
||||
"CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"],
|
||||
"TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0",
|
||||
})
|
||||
os.chdir(source)
|
||||
result = execute(request, source, output)
|
||||
(output / "result.json").write_text(json.dumps(result, sort_keys=True), "utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,564 @@
|
||||
//! PEP 517 source candidates enter the normal dependency solver as real wheels.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
|
||||
use crate::libs::{
|
||||
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
|
||||
sha256,
|
||||
};
|
||||
use crate::workers::{python_wheel::Wheel, runtime};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Environment {
|
||||
files: BTreeMap<String, (Vec<u8>, bool)>,
|
||||
assets: Vec<(String, Vec<u8>)>,
|
||||
executable: String,
|
||||
version: String,
|
||||
extension_suffixes: Vec<String>,
|
||||
descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
struct Active<'a> {
|
||||
session: &'a Session<'a>,
|
||||
key: String,
|
||||
}
|
||||
impl Drop for Active<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.session.active.borrow_mut().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(crate) fn python(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
spec: &LibSpec,
|
||||
runtime: &RegistryRuntime,
|
||||
filename: &str,
|
||||
source: &[u8],
|
||||
) -> Result<Product, String> {
|
||||
let key = format!(
|
||||
"{}@{}:{}:{}",
|
||||
spec.distribution_name(),
|
||||
spec.version,
|
||||
sha256(source),
|
||||
runtime.pack.sha256
|
||||
);
|
||||
if let Some(product) = self.products.borrow().get(&key) {
|
||||
return Ok(product.clone());
|
||||
}
|
||||
if self.active.borrow().len() >= 16 || !self.active.borrow_mut().insert(key.clone()) {
|
||||
return Err(format!(
|
||||
"cyclic or overly deep Python build requirement for {}",
|
||||
spec.key()
|
||||
));
|
||||
}
|
||||
let _active = Active {
|
||||
session: self,
|
||||
key: key.clone(),
|
||||
};
|
||||
let sources = source_files(filename, source)?;
|
||||
let environment = Environment::new(self, runtime)?;
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected =
|
||||
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
|
||||
let system: PythonSystem = serde_json::from_value(inspected.response)
|
||||
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
|
||||
let mut requirements = system.requires.clone();
|
||||
let mut dependencies = resolver.build_requirements(&requirements, runtime)?;
|
||||
let mut stable = None;
|
||||
let mut discovered = BTreeSet::new();
|
||||
for _ in 0..8 {
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"requires","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let dynamic: Vec<String> = serde_json::from_value(
|
||||
result
|
||||
.response
|
||||
.get("requires")
|
||||
.cloned()
|
||||
.ok_or("build backend omitted requirements")?,
|
||||
)
|
||||
.map_err(|error| format!("invalid dynamic Python build requirements: {error}"))?;
|
||||
discovered.extend(dynamic);
|
||||
requirements = system.requires.iter().chain(&discovered).cloned().collect();
|
||||
let next = resolver.build_requirements(&requirements, runtime)?;
|
||||
if next == dependencies {
|
||||
stable = Some(discovered.iter().cloned().collect());
|
||||
break;
|
||||
}
|
||||
dependencies = next;
|
||||
}
|
||||
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
|
||||
let metadata = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
let output = artifact(
|
||||
runtime,
|
||||
format!(
|
||||
"lib/pip/{}/{}/archive.whl",
|
||||
sha256(spec.distribution_name().as_bytes()),
|
||||
spec.version
|
||||
),
|
||||
&bytes,
|
||||
);
|
||||
let source_artifact = artifact(
|
||||
runtime,
|
||||
format!("build/source/{}/source", sha256(source)),
|
||||
source,
|
||||
);
|
||||
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
|
||||
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
|
||||
return Err("build artifact cache identity mismatch".into());
|
||||
}
|
||||
}
|
||||
let base: LibSpec = format!("pip:{}@{}", spec.distribution_name(), spec.version).parse()?;
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
roots: vec![base],
|
||||
sources: vec![source_artifact],
|
||||
graph: None,
|
||||
runtime: runtime.pack.clone(),
|
||||
auxiliary_runtime: None,
|
||||
native_entries: BTreeSet::new(),
|
||||
native_rejections: BTreeMap::new(),
|
||||
python_markers: runtime.python_markers.clone(),
|
||||
tools: environment.descriptor,
|
||||
frontend: super::frontend(&Ecosystem::Pip)?,
|
||||
python: Some(system),
|
||||
dynamic_requires: dynamic,
|
||||
dependencies: Box::new(dependencies),
|
||||
config_settings: BTreeMap::new(),
|
||||
output,
|
||||
};
|
||||
let product = Product {
|
||||
bytes,
|
||||
filename: filename.into(),
|
||||
receipt,
|
||||
};
|
||||
self.remember(key, product.clone())?;
|
||||
Ok(product)
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArtifact {
|
||||
LockedArtifact {
|
||||
target: runtime.target.clone(),
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
struct Source {
|
||||
files: Vec<(String, Vec<u8>)>,
|
||||
executables: BTreeSet<String>,
|
||||
}
|
||||
|
||||
fn source_files(filename: &str, bytes: &[u8]) -> Result<Source, String> {
|
||||
let extension = if filename.ends_with(".tar.gz") {
|
||||
"tgz"
|
||||
} else if filename.ends_with(".zip") {
|
||||
"zip"
|
||||
} else {
|
||||
return Err("unsupported Python source archive format".into());
|
||||
};
|
||||
let files = archive_files(extension, bytes)?;
|
||||
let executables = crate::libs::registry::archive_executables(extension, bytes)?;
|
||||
let root = files
|
||||
.first()
|
||||
.and_then(|(path, _)| path.split_once('/'))
|
||||
.map(|(root, _)| format!("{root}/"))
|
||||
.ok_or("Python source archive needs one root directory")?;
|
||||
let files = files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| {
|
||||
let path = path
|
||||
.strip_prefix(&root)
|
||||
.ok_or("Python source archive contains multiple roots")?
|
||||
.to_owned();
|
||||
crate::workers::valid_path(&path)?;
|
||||
Ok((path, bytes))
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
let executables = executables
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
path.strip_prefix(&root)
|
||||
.map(str::to_owned)
|
||||
.ok_or("source executable escapes archive root".into())
|
||||
})
|
||||
.collect::<Result<_, String>>()?;
|
||||
Ok(Source { files, executables })
|
||||
}
|
||||
|
||||
struct ResultFiles {
|
||||
response: Value,
|
||||
outputs: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
|
||||
super::require_execution_target(&requested.target, "Python source build")?;
|
||||
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
|
||||
if runtime.pack != requested.pack || sha256(&bytes) != requested.pack.sha256 {
|
||||
return Err("build Python runtime differs from resolver identity".into());
|
||||
}
|
||||
let runtime_files = archive_files("tgz", &bytes)?;
|
||||
let manifest = runtime::validate(&runtime_files, &Ecosystem::Pip, &requested.target)?;
|
||||
if manifest.python_wheel_tags != requested.python_wheel_tags
|
||||
|| manifest.python_markers != requested.python_markers
|
||||
{
|
||||
return Err("build runtime policy differs from signed descriptor".into());
|
||||
}
|
||||
let executable = manifest
|
||||
.executable
|
||||
.clone()
|
||||
.ok_or("build Python executable missing")?;
|
||||
let executable = format!("runtime/{executable}");
|
||||
let mut files = runtime_files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| {
|
||||
let executable = manifest.executable(&path);
|
||||
(format!("runtime/{path}"), (bytes, executable))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let (descriptor, bytes) = session.inputs.tools(Ecosystem::Pip)?;
|
||||
descriptor.validate(requested, None)?;
|
||||
let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?;
|
||||
let assets = session.inputs.assets()?;
|
||||
dever_sandbox::validate_assets(&assets)?;
|
||||
pack::validate_tools(&tools, &tool_files, &assets)?;
|
||||
for (path, bytes) in tool_files {
|
||||
let executable = tools.executables.contains(&path);
|
||||
if files.insert(path, (bytes, executable)).is_some() {
|
||||
return Err("build tools overlap runtime inputs".into());
|
||||
}
|
||||
}
|
||||
let version = requested
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("build Python markers absent")?
|
||||
.python_version()
|
||||
.to_string();
|
||||
Ok(Self {
|
||||
files,
|
||||
assets,
|
||||
executable,
|
||||
version,
|
||||
extension_suffixes: manifest.python_extension_suffixes,
|
||||
descriptor,
|
||||
})
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
mut request: Value,
|
||||
prepared: Option<&BTreeMap<String, Vec<u8>>>,
|
||||
) -> Result<ResultFiles, String> {
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
stage.write(&format!("input/{path}"), bytes, *executable)?;
|
||||
}
|
||||
for (path, bytes) in &self.assets {
|
||||
stage.write(
|
||||
&format!("sandbox/{path}"),
|
||||
bytes,
|
||||
dever_sandbox::asset_executable(path),
|
||||
)?;
|
||||
}
|
||||
for (path, bytes) in &source.files {
|
||||
stage.write(
|
||||
&format!("input/source/{path}"),
|
||||
bytes,
|
||||
source.executables.contains(path),
|
||||
)?;
|
||||
}
|
||||
if let Some(prepared) = prepared {
|
||||
for (path, bytes) in prepared {
|
||||
stage.write(&format!("input/prepared/{path}"), bytes, false)?;
|
||||
}
|
||||
}
|
||||
let mut installed = BTreeMap::<String, Vec<u8>>::new();
|
||||
let runtime = resolver
|
||||
.runtimes
|
||||
.get(&Ecosystem::Pip)
|
||||
.ok_or("build dependency runtime is missing")?;
|
||||
let mut distributions = BTreeSet::new();
|
||||
for lib in &dependencies.libs {
|
||||
super::validate_python_policy(
|
||||
dependencies,
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build markers absent")?,
|
||||
)?;
|
||||
if lib.runtime != runtime.pack || lib.artifacts.len() != 1 {
|
||||
return Err(
|
||||
"build dependency does not use the exact Python runtime and wheel".into(),
|
||||
);
|
||||
}
|
||||
let artifact = &lib.artifacts[0];
|
||||
let bytes =
|
||||
resolver
|
||||
.store
|
||||
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
|
||||
wheel.validate_dependencies(
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build markers absent")?,
|
||||
)?;
|
||||
if !distributions.insert((
|
||||
lib.spec.distribution_name().to_owned(),
|
||||
lib.spec.version.clone(),
|
||||
)) {
|
||||
continue;
|
||||
}
|
||||
for file in wheel.install(&self.executable, &self.version)? {
|
||||
if self.files.contains_key(&file.path)
|
||||
|| installed
|
||||
.insert(file.path.clone(), file.bytes.clone())
|
||||
.is_some()
|
||||
{
|
||||
return Err("build dependency installation has conflicting file owners".into());
|
||||
}
|
||||
stage.write(
|
||||
&format!("input/{}", file.path),
|
||||
&file.bytes,
|
||||
file.executable,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
let mut supplied = self
|
||||
.files
|
||||
.iter()
|
||||
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (path, bytes) in &self.assets {
|
||||
supplied.insert(format!("sandbox/{path}"), bytes);
|
||||
}
|
||||
for (path, bytes) in &installed {
|
||||
supplied.insert(path.clone(), bytes);
|
||||
}
|
||||
crate::workers::python_wheel::validate_native(
|
||||
&runtime.target,
|
||||
&self.extension_suffixes,
|
||||
&self.executable,
|
||||
&supplied,
|
||||
&installed.keys().cloned().collect(),
|
||||
)?;
|
||||
request["python_version"] = json!(self.version);
|
||||
request["config_settings"] = json!({});
|
||||
stage.write(
|
||||
"input/request.json",
|
||||
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
)?;
|
||||
stage.write("input/frontend.py", include_bytes!("python.py"), false)?;
|
||||
let work = stage.0.join("work");
|
||||
fs::create_dir(&work).map_err(|error| error.to_string())?;
|
||||
let grants = [dever_sandbox::Grant {
|
||||
source: work.clone(),
|
||||
destination: "/data/build".into(),
|
||||
writable: true,
|
||||
}];
|
||||
let arguments = [
|
||||
OsString::from("-I"),
|
||||
OsString::from("-B"),
|
||||
input.join("frontend.py").into_os_string(),
|
||||
];
|
||||
let executable = input.join(&self.executable);
|
||||
let assets = stage.0.join("sandbox");
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities {
|
||||
network: false,
|
||||
process: true,
|
||||
},
|
||||
grants: &grants,
|
||||
};
|
||||
stage.run(
|
||||
launch.command_for_build(&input.join("rootfs"))?,
|
||||
"isolated Python build hook",
|
||||
&work,
|
||||
)?;
|
||||
if let Some(session) = resolver.build {
|
||||
session.inputs.diagnostic(&stage.diagnostics()?);
|
||||
}
|
||||
let outputs = output_files(&work.join("output"))?;
|
||||
let response = serde_json::from_slice(
|
||||
outputs
|
||||
.get("result.json")
|
||||
.ok_or("build hook produced no result")?,
|
||||
)
|
||||
.map_err(|error| format!("invalid build hook response: {error}"))?;
|
||||
Ok(ResultFiles { response, outputs })
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn output_files(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
|
||||
let mut files = BTreeMap::new();
|
||||
let mut pending = vec![root.to_owned()];
|
||||
let mut remaining = crate::workers::MAX_TREE_BYTES;
|
||||
let mut entries = 0usize;
|
||||
#[cfg(unix)]
|
||||
let mut links = BTreeMap::<(u64, u64), (u64, u64)>::new();
|
||||
while let Some(directory) = pending.pop() {
|
||||
if fs::symlink_metadata(&directory)
|
||||
.map_err(|error| error.to_string())?
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
{
|
||||
return Err("build output directory is a symbolic link".into());
|
||||
}
|
||||
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
entries += 1;
|
||||
if entries > crate::workers::MAX_TREE_FILES {
|
||||
return Err("build output exceeds file budget".into());
|
||||
}
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let kind = entry.file_type().map_err(|error| error.to_string())?;
|
||||
if kind.is_dir() {
|
||||
pending.push(entry.path());
|
||||
continue;
|
||||
}
|
||||
if !kind.is_file() {
|
||||
return Err("build output contains a link or special file".into());
|
||||
}
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
|
||||
}
|
||||
let file = options
|
||||
.open(entry.path())
|
||||
.map_err(|error| error.to_string())?;
|
||||
let metadata = file.metadata().map_err(|error| error.to_string())?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let aliases = links
|
||||
.entry((metadata.dev(), metadata.ino()))
|
||||
.or_insert((metadata.nlink(), 0));
|
||||
if aliases.0 != metadata.nlink() {
|
||||
return Err("build output link count changed during collection".into());
|
||||
}
|
||||
aliases.1 += 1;
|
||||
}
|
||||
if !metadata.is_file() || metadata.len() > remaining as u64 {
|
||||
return Err("build output exceeds byte budget".into());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
file.take(remaining as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
remaining = remaining
|
||||
.checked_sub(bytes.len())
|
||||
.ok_or("build output exceeds byte budget")?;
|
||||
let path = entry.path();
|
||||
let name = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.to_str()
|
||||
.ok_or("build output path is not UTF-8")?
|
||||
.to_owned();
|
||||
crate::workers::valid_path(&name)?;
|
||||
files.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
// node-gyp's COPY rule may hard-link two output names. Namespace teardown
|
||||
// precedes this read: accept aliases only when every link is accounted for
|
||||
// inside the output root, then publish independent immutable byte entries.
|
||||
#[cfg(unix)]
|
||||
if links
|
||||
.values()
|
||||
.any(|(expected, observed)| expected != observed)
|
||||
{
|
||||
return Err("build output has a hard-link alias outside its output root".into());
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn compare_metadata(
|
||||
prefix: &str,
|
||||
outputs: &BTreeMap<String, Vec<u8>>,
|
||||
wheel: &[u8],
|
||||
) -> Result<(), String> {
|
||||
crate::workers::valid_path(prefix)?;
|
||||
let distribution = PathBuf::from(prefix)
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or("invalid prepared metadata directory")?
|
||||
.to_owned();
|
||||
let wheel = archive_files("whl", wheel)?
|
||||
.into_iter()
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let prefix = format!("{prefix}/");
|
||||
let mut count = 0;
|
||||
for (path, bytes) in outputs {
|
||||
if let Some(relative) = path.strip_prefix(&prefix) {
|
||||
count += 1;
|
||||
if wheel.get(&format!("{distribution}/{relative}")) != Some(bytes) {
|
||||
return Err(
|
||||
"built wheel metadata differs from prepare_metadata_for_build_wheel".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return Err("metadata hook produced an empty directory".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Serialize dependency mutations from the first project read to publication.
|
||||
//! The guard file is stable: replacing dever.lock must not replace its mutex.
|
||||
|
||||
use std::fs::{self, File, Metadata, OpenOptions};
|
||||
use std::path::Path;
|
||||
|
||||
pub(crate) struct ProjectMutation {
|
||||
_file: File,
|
||||
}
|
||||
|
||||
impl ProjectMutation {
|
||||
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
|
||||
if !matches!(command, "add" | "update" | "remove") {
|
||||
return Ok(None);
|
||||
}
|
||||
let directory = fs::symlink_metadata(root)
|
||||
.map_err(|error| format!("cannot inspect project mutation directory: {error}"))?;
|
||||
validate(&directory, true)?;
|
||||
let path = root.join(".dever-project.lock");
|
||||
if let Ok(metadata) = fs::symlink_metadata(&path) {
|
||||
validate(&metadata, false)?;
|
||||
}
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true).write(true).create(true).truncate(false);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options
|
||||
.mode(0o600)
|
||||
.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
|
||||
}
|
||||
let file = options
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot open project mutation lock: {error}"))?;
|
||||
let opened = file.metadata().map_err(|error| error.to_string())?;
|
||||
validate(&opened, false)?;
|
||||
let current = fs::symlink_metadata(&path).map_err(|error| error.to_string())?;
|
||||
validate(¤t, false)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
if opened.dev() != current.dev() || opened.ino() != current.ino() {
|
||||
return Err("project mutation lock changed while opening".into());
|
||||
}
|
||||
}
|
||||
fs2::FileExt::try_lock_exclusive(&file).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock {
|
||||
"project dependency mutation is busy; retry after the current command finishes"
|
||||
.into()
|
||||
} else {
|
||||
format!("cannot acquire project mutation lock: {error}")
|
||||
}
|
||||
})?;
|
||||
Ok(Some(Self { _file: file }))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(metadata: &Metadata, directory: bool) -> Result<(), String> {
|
||||
if metadata.file_type().is_symlink()
|
||||
|| (directory && !metadata.is_dir())
|
||||
|| (!directory && !metadata.is_file())
|
||||
{
|
||||
return Err(
|
||||
"project mutation path must be a real directory or regular lock file, without symlinks"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let forbidden = if directory { 0o022 } else { 0o077 };
|
||||
if metadata.uid() != rustix::process::geteuid().as_raw()
|
||||
|| metadata.mode() & forbidden != 0
|
||||
|| (!directory && metadata.nlink() != 1)
|
||||
{
|
||||
return Err("project mutation directory must be owned and not group/other writable; lock must be owned, private and unlinked elsewhere".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,995 @@
|
||||
//! npm package instances follow Node's ancestor lookup, not one global version
|
||||
//! per name. Archives remain content-addressed and shared between instances.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
|
||||
use node_semver::{Range, Version};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{Ecosystem, LibSpec, LockFile, LockedLib, RegistryRuntime, archive_files};
|
||||
|
||||
mod resolve;
|
||||
pub(super) use resolve::resolve;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Kind {
|
||||
Dependency,
|
||||
Optional,
|
||||
Peer,
|
||||
OptionalPeer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Omission {
|
||||
Target,
|
||||
Unavailable,
|
||||
Conflict,
|
||||
OptionalPeerAbsent,
|
||||
BuildFailure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Edge {
|
||||
pub name: String,
|
||||
pub requirement: String,
|
||||
pub kind: Kind,
|
||||
pub target: Option<String>,
|
||||
pub omission: Option<Omission>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Source {
|
||||
pub archive: LibSpec,
|
||||
pub prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Instance {
|
||||
pub path: String,
|
||||
pub spec: LibSpec,
|
||||
pub source: Source,
|
||||
/// Exact bundled layout of a root archive, including hoisted transitives.
|
||||
pub bundled: BTreeMap<String, LibSpec>,
|
||||
pub edges: Vec<Edge>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Environment {
|
||||
pub roots: Vec<LibSpec>,
|
||||
pub instances: Vec<Instance>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build: Option<String>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
/// Explicit single-package graph for author fixtures with no dependencies.
|
||||
/// Materialization still checks package.json against this graph.
|
||||
pub fn single(spec: LibSpec) -> Self {
|
||||
Self {
|
||||
build: None,
|
||||
roots: vec![spec.clone()],
|
||||
instances: vec![Instance {
|
||||
path: child("", &spec.name),
|
||||
spec: spec.clone(),
|
||||
source: Source {
|
||||
archive: spec,
|
||||
prefix: "package/".into(),
|
||||
},
|
||||
bundled: BTreeMap::new(),
|
||||
edges: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn roots(specs: &[LibSpec]) -> Vec<LibSpec> {
|
||||
let mut roots = specs
|
||||
.iter()
|
||||
.filter(|spec| spec.ecosystem == Ecosystem::Npm)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
roots.sort();
|
||||
roots.dedup();
|
||||
roots
|
||||
}
|
||||
|
||||
pub fn environment<'a>(
|
||||
lock: &'a LockFile,
|
||||
requested: &[LibSpec],
|
||||
) -> Result<Option<&'a Environment>, String> {
|
||||
let requested = roots(requested);
|
||||
if requested.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
lock.npm
|
||||
.iter()
|
||||
.find(|env| env.roots == requested)
|
||||
.map(Some)
|
||||
.ok_or("npm Worker roots have no locked installation environment".into())
|
||||
}
|
||||
|
||||
pub fn archive_roots(lock: &LockFile, requested: &[LibSpec]) -> Result<BTreeSet<LibSpec>, String> {
|
||||
let requested = roots(requested);
|
||||
if requested.is_empty() {
|
||||
return Ok(BTreeSet::new());
|
||||
}
|
||||
if let Some(environment) = lock
|
||||
.npm
|
||||
.iter()
|
||||
.find(|environment| environment.roots == requested)
|
||||
{
|
||||
return Ok(environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| node.source.archive.clone())
|
||||
.collect());
|
||||
}
|
||||
// Built-in metadata fixtures are not registry packages and have no tree.
|
||||
let fixtures = super::super::FixtureRegistry::builtin();
|
||||
if requested
|
||||
.iter()
|
||||
.all(|spec| fixtures.package(spec).is_some())
|
||||
{
|
||||
return Ok(requested.into_iter().collect());
|
||||
}
|
||||
Err("npm roots have no exact locked installation environment".into())
|
||||
}
|
||||
|
||||
pub fn retain(lock: &LockFile, groups: &[Vec<LibSpec>]) -> Result<Vec<Environment>, String> {
|
||||
let mut environments = Vec::new();
|
||||
for group in groups {
|
||||
let requested = roots(group);
|
||||
if requested.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let original = lock
|
||||
.npm
|
||||
.iter()
|
||||
.find(|env| env.roots == requested)
|
||||
.or_else(|| {
|
||||
lock.npm
|
||||
.iter()
|
||||
.find(|env| requested.iter().all(|root| env.roots.contains(root)))
|
||||
});
|
||||
let Some(original) = original else {
|
||||
archive_roots(lock, group)?;
|
||||
continue;
|
||||
};
|
||||
let mut queue = requested
|
||||
.iter()
|
||||
.map(|spec| child("", &spec.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut paths = BTreeSet::new();
|
||||
while let Some(path) = queue.pop_front() {
|
||||
if !paths.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = original
|
||||
.instances
|
||||
.iter()
|
||||
.find(|node| node.path == path)
|
||||
.ok_or("npm retained graph lacks a dependency instance")?;
|
||||
queue.extend(node.edges.iter().filter_map(|edge| edge.target.clone()));
|
||||
}
|
||||
environments.push(Environment {
|
||||
build: original.build.clone(),
|
||||
roots: requested,
|
||||
instances: original
|
||||
.instances
|
||||
.iter()
|
||||
.filter(|node| paths.contains(&node.path))
|
||||
.cloned()
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
normalize(&mut environments)?;
|
||||
Ok(environments)
|
||||
}
|
||||
|
||||
pub(super) fn child(parent: &str, name: &str) -> String {
|
||||
if parent.is_empty() {
|
||||
format!("node_modules/{name}")
|
||||
} else {
|
||||
format!("{parent}/node_modules/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parent(path: &str) -> &str {
|
||||
path.rsplit_once("/node_modules/")
|
||||
.map_or("", |(parent, _)| parent)
|
||||
}
|
||||
|
||||
fn validate_path(path: &str) -> Result<(), String> {
|
||||
if path.len() > 4096 {
|
||||
return Err("npm instance path exceeds its bound".into());
|
||||
}
|
||||
let path = path
|
||||
.strip_prefix("node_modules/")
|
||||
.ok_or("invalid npm instance path")?;
|
||||
for name in path.split("/node_modules/") {
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn visible<'a>(
|
||||
instances: &'a BTreeMap<String, Instance>,
|
||||
consumer: &str,
|
||||
name: &str,
|
||||
peer: bool,
|
||||
) -> Option<&'a Instance> {
|
||||
let mut scope = if peer { parent(consumer) } else { consumer };
|
||||
loop {
|
||||
if let Some(found) = instances.get(&child(scope, name)) {
|
||||
return Some(found);
|
||||
}
|
||||
if scope.is_empty() {
|
||||
return None;
|
||||
}
|
||||
scope = parent(scope);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct Metadata {
|
||||
pub spec: LibSpec,
|
||||
pub edges: Vec<Edge>,
|
||||
pub bundles: BTreeSet<String>,
|
||||
pub document: Value,
|
||||
}
|
||||
|
||||
fn requirements(document: &Value, field: &str) -> Result<BTreeMap<String, String>, String> {
|
||||
let Some(value) = document.get(field) else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
value
|
||||
.as_object()
|
||||
.ok_or_else(|| format!("npm {field} must be an object"))?
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
let rule = value.as_str().ok_or("npm dependency range must be text")?;
|
||||
rule.parse::<Range>()
|
||||
.map_err(|error| format!("invalid npm range for {name}: {error}"))?;
|
||||
Ok((name.clone(), rule.into()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub(super) fn parse(document: Value) -> Result<Self, String> {
|
||||
let spec: LibSpec = format!(
|
||||
"npm:{}@{}",
|
||||
super::field(&document, "name")?,
|
||||
super::field(&document, "version")?
|
||||
)
|
||||
.parse()?;
|
||||
let mut regular = requirements(&document, "dependencies")?;
|
||||
let optional = requirements(&document, "optionalDependencies")?;
|
||||
regular.retain(|name, _| !optional.contains_key(name));
|
||||
let mut peers = requirements(&document, "peerDependencies")?;
|
||||
let peer_meta = document
|
||||
.get("peerDependenciesMeta")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_object()
|
||||
.ok_or("npm peerDependenciesMeta must be an object")
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(meta) = peer_meta {
|
||||
for (name, value) in meta {
|
||||
if !peers.contains_key(name)
|
||||
|| !value.is_object()
|
||||
|| value.get("optional").is_some_and(|flag| !flag.is_boolean())
|
||||
{
|
||||
return Err("invalid npm optional peer metadata".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
// npm has one outgoing edge per name: optional > production > peer.
|
||||
peers.retain(|name, _| !regular.contains_key(name) && !optional.contains_key(name));
|
||||
let mut edges = Vec::new();
|
||||
for (entries, kind) in [
|
||||
(regular.clone(), Kind::Dependency),
|
||||
(optional.clone(), Kind::Optional),
|
||||
(peers, Kind::Peer),
|
||||
] {
|
||||
for (name, requirement) in entries {
|
||||
let kind = if kind == Kind::Peer
|
||||
&& peer_meta
|
||||
.and_then(|meta| meta.get(&name))
|
||||
.and_then(|value| value.get("optional"))
|
||||
.and_then(Value::as_bool)
|
||||
== Some(true)
|
||||
{
|
||||
Kind::OptionalPeer
|
||||
} else {
|
||||
kind.clone()
|
||||
};
|
||||
edges.push(Edge {
|
||||
name,
|
||||
requirement,
|
||||
kind,
|
||||
target: None,
|
||||
omission: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
edges.sort();
|
||||
let bundles = |value: &Value| -> Result<BTreeSet<String>, String> {
|
||||
match value {
|
||||
Value::Bool(true) => Ok(regular.keys().chain(optional.keys()).cloned().collect()),
|
||||
Value::Bool(false) => Ok(BTreeSet::new()),
|
||||
Value::Array(values) => values
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let name = value
|
||||
.as_str()
|
||||
.ok_or("npm bundled dependency name must be text")?;
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
if !regular.contains_key(name) && !optional.contains_key(name) {
|
||||
return Err(
|
||||
"npm bundled dependency is not a declared dependency".into()
|
||||
);
|
||||
}
|
||||
Ok(name.to_owned())
|
||||
})
|
||||
.collect(),
|
||||
_ => Err("npm bundleDependencies must be an array or boolean".into()),
|
||||
}
|
||||
};
|
||||
let first = document
|
||||
.get("bundleDependencies")
|
||||
.map(bundles)
|
||||
.transpose()?;
|
||||
let second = document
|
||||
.get("bundledDependencies")
|
||||
.map(bundles)
|
||||
.transpose()?;
|
||||
if first.is_some() && second.is_some() && first != second {
|
||||
return Err("npm bundled dependency aliases disagree".into());
|
||||
}
|
||||
Ok(Self {
|
||||
spec,
|
||||
edges,
|
||||
bundles: first.or(second).unwrap_or_default(),
|
||||
document,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn eligible(&self, runtime: &RegistryRuntime) -> Result<bool, String> {
|
||||
let (os, cpu) = runtime
|
||||
.target
|
||||
.split_once('-')
|
||||
.ok_or("npm runtime target lacks platform identity")?;
|
||||
let os = match os {
|
||||
"windows" => "win32",
|
||||
"macos" => "darwin",
|
||||
other => other,
|
||||
};
|
||||
let cpu = match cpu {
|
||||
"x86_64" => "x64",
|
||||
"aarch64" => "arm64",
|
||||
other => other,
|
||||
};
|
||||
for (field, selected) in [
|
||||
("os", Some(os)),
|
||||
("cpu", Some(cpu)),
|
||||
("libc", runtime.npm_libc.as_deref()),
|
||||
] {
|
||||
if field == "libc" && os != "linux" {
|
||||
continue;
|
||||
}
|
||||
let Some(value) = self.document.get(field) else {
|
||||
continue;
|
||||
};
|
||||
let values = value
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("npm {field} must be an array"))?;
|
||||
let values = values
|
||||
.iter()
|
||||
.map(|value| value.as_str().ok_or("npm platform selector must be text"))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let Some(selected) = selected else {
|
||||
return Ok(false);
|
||||
};
|
||||
if values
|
||||
.iter()
|
||||
.any(|value| value.strip_prefix('!') == Some(selected))
|
||||
|| (values.iter().any(|value| !value.starts_with('!'))
|
||||
&& !values.contains(&selected)
|
||||
&& !values.contains(&"any"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Archive {
|
||||
pub files: Vec<(String, Vec<u8>)>,
|
||||
pub packages: BTreeMap<String, Metadata>,
|
||||
}
|
||||
|
||||
impl Archive {
|
||||
pub(super) fn bundled(&self) -> BTreeMap<String, LibSpec> {
|
||||
self.packages
|
||||
.iter()
|
||||
.filter(|(prefix, _)| prefix.as_str() != "package/")
|
||||
.map(|(prefix, metadata)| {
|
||||
(
|
||||
prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/')
|
||||
.into(),
|
||||
metadata.spec.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn parse(spec: &LibSpec, bytes: &[u8]) -> Result<Self, String> {
|
||||
let files = archive_files("tgz", bytes)?;
|
||||
let mut packages = BTreeMap::<String, Metadata>::new();
|
||||
for (path, bytes) in &files {
|
||||
let relative = path
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm archive must have a package/ root")?;
|
||||
if relative == "package.json" || relative.ends_with("/package.json") {
|
||||
let prefix = path.strip_suffix("package.json").unwrap();
|
||||
if prefix != "package/" {
|
||||
let nested = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
if validate_path(nested).is_err() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let document = super::parse_json(bytes)?;
|
||||
let metadata = Metadata::parse(document)?;
|
||||
if prefix == "package/" {
|
||||
if metadata.spec != *spec {
|
||||
return Err(
|
||||
"npm package.json identity differs from its registry archive".into(),
|
||||
);
|
||||
}
|
||||
} else if !prefix.ends_with(&format!("node_modules/{}/", metadata.spec.name)) {
|
||||
return Err("npm bundled package identity differs from its path".into());
|
||||
}
|
||||
packages.insert(prefix.into(), metadata);
|
||||
}
|
||||
}
|
||||
let root = packages
|
||||
.get("package/")
|
||||
.ok_or("npm archive lacks package.json")?;
|
||||
for name in &root.bundles {
|
||||
if !packages.contains_key(&format!("package/node_modules/{name}/")) {
|
||||
return Err("npm declared bundled dependency is absent from its archive".into());
|
||||
}
|
||||
}
|
||||
for (path, _) in &files {
|
||||
if !packages.contains_key(&source_owner(path)?) {
|
||||
return Err("npm archive has unowned node_modules content".into());
|
||||
}
|
||||
}
|
||||
let nodes = packages
|
||||
.iter()
|
||||
.filter(|(prefix, _)| prefix.as_str() != "package/")
|
||||
.map(|(prefix, metadata)| {
|
||||
let path = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/')
|
||||
.to_owned();
|
||||
(
|
||||
path.clone(),
|
||||
Instance {
|
||||
path,
|
||||
spec: metadata.spec.clone(),
|
||||
source: Source {
|
||||
archive: spec.clone(),
|
||||
prefix: prefix.clone(),
|
||||
},
|
||||
bundled: BTreeMap::new(),
|
||||
edges: Vec::new(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut pending = root
|
||||
.bundles
|
||||
.iter()
|
||||
.map(|name| child("", name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = nodes
|
||||
.get(&path)
|
||||
.ok_or("npm bundle references a missing package")?;
|
||||
for edge in &packages[&node.source.prefix].edges {
|
||||
let peer = matches!(edge.kind, Kind::Peer | Kind::OptionalPeer);
|
||||
if let Some(found) = visible(&nodes, &path, &edge.name, peer) {
|
||||
if !Version::from_str(&found.spec.version)
|
||||
.map_err(|error| error.to_string())?
|
||||
.satisfies(
|
||||
&edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| error.to_string())?,
|
||||
)
|
||||
{
|
||||
return Err("npm bundled dependency has an incompatible version".into());
|
||||
}
|
||||
pending.push_back(found.path.clone());
|
||||
} else if edge.kind == Kind::Dependency
|
||||
&& edge.name != spec.name
|
||||
&& !root.edges.iter().any(|root| root.name == edge.name)
|
||||
{
|
||||
return Err("npm bundled dependency closure is incomplete".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
if reached.len() != nodes.len() {
|
||||
return Err("npm archive contains undeclared or unreachable bundled packages".into());
|
||||
}
|
||||
Ok(Self { files, packages })
|
||||
}
|
||||
}
|
||||
|
||||
fn source_owner(path: &str) -> Result<String, String> {
|
||||
let Some((parent, tail)) = path.rsplit_once("/node_modules/") else {
|
||||
return Ok("package/".into());
|
||||
};
|
||||
let mut parts = tail.split('/');
|
||||
let first = parts.next().ok_or("npm archive has invalid package path")?;
|
||||
let name = if first.starts_with('@') {
|
||||
format!(
|
||||
"{first}/{}",
|
||||
parts
|
||||
.next()
|
||||
.ok_or("npm scoped package path is incomplete")?
|
||||
)
|
||||
} else {
|
||||
first.into()
|
||||
};
|
||||
let prefix = format!("{parent}/node_modules/{name}");
|
||||
validate_path(
|
||||
prefix
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm bundled archive has invalid root")?,
|
||||
)?;
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_source_owner(environment: &Environment, node: &Instance) -> Result<(), String> {
|
||||
for owner in &environment.instances {
|
||||
for (relative, spec) in &owner.bundled {
|
||||
if node.path == format!("{}/{relative}", owner.path)
|
||||
&& (node.spec != *spec
|
||||
|| node.source.archive != owner.source.archive
|
||||
|| node.source.prefix != format!("package/{relative}/"))
|
||||
{
|
||||
return Err(
|
||||
"npm bundled instance is not owned by its parent archive installation".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if node.source.prefix == "package/" {
|
||||
for (relative, spec) in &node.bundled {
|
||||
validate_path(relative)?;
|
||||
if spec.ecosystem != Ecosystem::Npm
|
||||
|| !relative.ends_with(&format!("node_modules/{}", spec.name))
|
||||
{
|
||||
return Err("invalid npm bundled layout identity".into());
|
||||
}
|
||||
super::super::validate_name(&Ecosystem::Npm, &spec.name)?;
|
||||
super::super::validate_version(&Ecosystem::Npm, &spec.version)?;
|
||||
}
|
||||
return if node.source.archive == node.spec {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("npm root archive identity differs from its instance".into())
|
||||
};
|
||||
}
|
||||
let relative = node
|
||||
.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.and_then(|prefix| prefix.strip_suffix('/'))
|
||||
.ok_or("invalid npm bundled source prefix")?;
|
||||
validate_path(relative)?;
|
||||
if !node.bundled.is_empty()
|
||||
|| !environment.instances.iter().any(|owner| {
|
||||
owner.source.prefix == "package/"
|
||||
&& owner.source.archive == node.source.archive
|
||||
&& owner.spec == node.source.archive
|
||||
&& node.path == format!("{}/{relative}", owner.path)
|
||||
&& owner.bundled.get(relative) == Some(&node.spec)
|
||||
})
|
||||
{
|
||||
return Err("npm bundled instance is not owned by its parent archive installation".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate immutable manifests and plan borrowed files before cloning any
|
||||
/// installation output. Reused archives must not multiply past the tree budget.
|
||||
pub fn files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
source_files(environment, lock, archive, false)
|
||||
}
|
||||
|
||||
pub fn build_files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
source_files(environment, lock, archive, true)
|
||||
}
|
||||
|
||||
pub(crate) fn build_executables(
|
||||
environment: &Environment,
|
||||
archives: &BTreeMap<LibSpec, Vec<u8>>,
|
||||
) -> Result<BTreeSet<String>, String> {
|
||||
let modes = archives
|
||||
.iter()
|
||||
.map(|(spec, bytes)| Ok((spec, super::archive_executables("tgz", bytes)?)))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let mut installed = BTreeSet::new();
|
||||
for node in &environment.instances {
|
||||
for path in modes
|
||||
.get(&node.source.archive)
|
||||
.ok_or("npm source modes missing")?
|
||||
{
|
||||
if source_owner(path)? == node.source.prefix {
|
||||
let relative = path
|
||||
.strip_prefix(&node.source.prefix)
|
||||
.ok_or("npm source mode outside owner")?;
|
||||
installed.insert(format!("{}/{relative}", node.path));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(installed)
|
||||
}
|
||||
|
||||
pub(crate) fn installed_owner(path: &str) -> Result<String, String> {
|
||||
source_owner(&format!("package/{path}"))?
|
||||
.strip_prefix("package/")
|
||||
.map(|owner| owner.trim_end_matches('/').into())
|
||||
.ok_or("npm output is outside its installation".into())
|
||||
}
|
||||
|
||||
fn source_files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
mut archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
preparing: bool,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
let mut archives = BTreeMap::new();
|
||||
let mut archive_bytes = 0usize;
|
||||
for node in &environment.instances {
|
||||
validate_source_owner(environment, node)?;
|
||||
if archives.contains_key(&node.source.archive) {
|
||||
continue;
|
||||
}
|
||||
let lib = lock
|
||||
.libs
|
||||
.iter()
|
||||
.find(|lib| lib.spec == node.source.archive)
|
||||
.ok_or("npm graph source archive is missing")?;
|
||||
let parsed = Archive::parse(&lib.spec, archive(lib)?)?;
|
||||
for (_, bytes) in &parsed.files {
|
||||
archive_bytes = archive_bytes
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm archive size overflow")?;
|
||||
if archive_bytes > crate::workers::MAX_TREE_BYTES {
|
||||
return Err("npm archive cache exceeds its tree byte budget".into());
|
||||
}
|
||||
}
|
||||
archives.insert(lib.spec.clone(), parsed);
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
let mut installed_bytes = 0usize;
|
||||
for node in &environment.instances {
|
||||
let source = &archives[&node.source.archive];
|
||||
if node.source.prefix == "package/" && node.bundled != source.bundled() {
|
||||
return Err("npm bundled layout differs from its parent archive".into());
|
||||
}
|
||||
let metadata = source
|
||||
.packages
|
||||
.get(&node.source.prefix)
|
||||
.ok_or("npm instance source package is absent")?;
|
||||
if metadata.spec != node.spec
|
||||
|| metadata.edges.len() != node.edges.len()
|
||||
|| metadata.edges.iter().any(|expected| {
|
||||
!node.edges.iter().any(|edge| {
|
||||
edge.name == expected.name
|
||||
&& edge.requirement == expected.requirement
|
||||
&& edge.kind == expected.kind
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm locked dependency graph differs from package.json".into());
|
||||
}
|
||||
if !preparing
|
||||
&& (metadata.document.get("gypfile").and_then(Value::as_bool) == Some(true)
|
||||
|| metadata
|
||||
.document
|
||||
.get("scripts")
|
||||
.and_then(Value::as_object)
|
||||
.is_some_and(|scripts| {
|
||||
["preinstall", "install", "postinstall"]
|
||||
.iter()
|
||||
.any(|name| scripts.contains_key(*name))
|
||||
}))
|
||||
{
|
||||
return Err(
|
||||
"managed npm Worker rejects lifecycle scripts and native build hooks".into(),
|
||||
);
|
||||
}
|
||||
for (path, bytes) in &source.files {
|
||||
if source_owner(path)? != node.source.prefix {
|
||||
continue;
|
||||
}
|
||||
let relative = path
|
||||
.strip_prefix(&node.source.prefix)
|
||||
.ok_or("npm file is outside its instance source")?;
|
||||
if !preparing && (relative.ends_with(".node") || relative == "binding.gyp") {
|
||||
return Err("managed npm Worker rejects native package content".into());
|
||||
}
|
||||
let installed = format!("{}/{relative}", node.path);
|
||||
installed_bytes = installed_bytes
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm installed size overflow")?;
|
||||
if installed_bytes > crate::workers::MAX_TREE_BYTES
|
||||
|| files.len() >= crate::workers::MAX_TREE_FILES
|
||||
{
|
||||
return Err("npm installation exceeds its tree file/byte budget".into());
|
||||
}
|
||||
if files.insert(installed, bytes).is_some() {
|
||||
return Err("npm installed files have conflicting owners".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| (path, bytes.clone()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn normalize(environments: &mut Vec<Environment>) -> Result<(), String> {
|
||||
for env in environments.iter_mut() {
|
||||
env.roots.sort();
|
||||
env.roots.dedup();
|
||||
for node in &mut env.instances {
|
||||
node.edges.sort();
|
||||
}
|
||||
env.instances
|
||||
.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
}
|
||||
environments.sort_by(|left, right| left.roots.cmp(&right.roots));
|
||||
for pair in environments.windows(2) {
|
||||
if pair[0].roots == pair[1].roots && pair[0] != pair[1] {
|
||||
return Err("npm roots have inconsistent installation environments".into());
|
||||
}
|
||||
}
|
||||
environments.dedup();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn requires_build(files: &[(String, Vec<u8>)]) -> Result<bool, String> {
|
||||
for (path, bytes) in files {
|
||||
if path.ends_with(".node") || path.ends_with("/binding.gyp") {
|
||||
return Ok(true);
|
||||
}
|
||||
if path.ends_with("/package.json") {
|
||||
let metadata: Value = serde_json::from_slice(bytes)
|
||||
.map_err(|error| format!("invalid npm installation metadata: {error}"))?;
|
||||
if metadata.get("gypfile").and_then(Value::as_bool) == Some(true)
|
||||
|| metadata
|
||||
.get("scripts")
|
||||
.and_then(Value::as_object)
|
||||
.is_some_and(|scripts| {
|
||||
["preinstall", "install", "postinstall"]
|
||||
.iter()
|
||||
.any(|name| scripts.contains_key(*name))
|
||||
})
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Lifecycle output may add generated files, but cannot rewrite dependency
|
||||
/// identities, create undeclared package instances, or escape the locked tree.
|
||||
pub fn validate_built_files(
|
||||
environment: &Environment,
|
||||
files: &[(String, Vec<u8>)],
|
||||
) -> Result<(), String> {
|
||||
let nodes = environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| (node.path.as_str(), node))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut manifests = BTreeSet::new();
|
||||
let mut names = BTreeSet::new();
|
||||
let mut total = 0usize;
|
||||
for (path, bytes) in files {
|
||||
crate::workers::valid_path(path)?;
|
||||
let owner = source_owner(&format!("package/{path}"))?;
|
||||
let owner = owner
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm output is outside its installation")?
|
||||
.trim_end_matches('/');
|
||||
let node = nodes
|
||||
.get(owner)
|
||||
.ok_or("npm build output introduces an unlocked instance")?;
|
||||
if !names.insert(path) {
|
||||
return Err("npm build output repeats a file".into());
|
||||
}
|
||||
total = total
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm output byte overflow")?;
|
||||
if total > crate::workers::MAX_TREE_BYTES || names.len() > crate::workers::MAX_TREE_FILES {
|
||||
return Err("npm output exceeds installation budget".into());
|
||||
}
|
||||
if path == &format!("{owner}/package.json") {
|
||||
let metadata =
|
||||
Metadata::parse(serde_json::from_slice(bytes).map_err(|error| error.to_string())?)?;
|
||||
if metadata.spec != node.spec
|
||||
|| metadata.edges.len() != node.edges.len()
|
||||
|| metadata.edges.iter().any(|expected| {
|
||||
!node.edges.iter().any(|edge| {
|
||||
edge.name == expected.name
|
||||
&& edge.requirement == expected.requirement
|
||||
&& edge.kind == expected.kind
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm lifecycle changed locked package identity or dependencies".into());
|
||||
}
|
||||
manifests.insert(owner.to_owned());
|
||||
}
|
||||
}
|
||||
if manifests.len() != nodes.len() {
|
||||
return Err("npm lifecycle removed a locked package manifest".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
for env in &lock.npm {
|
||||
if env.roots.is_empty()
|
||||
|| env.roots != roots(&env.roots)
|
||||
|| env.instances.len() > super::MAX_PACKAGES
|
||||
{
|
||||
return Err("invalid npm installation environment roots or size".into());
|
||||
}
|
||||
let nodes = env
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| (node.path.clone(), node.clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
if nodes.len() != env.instances.len() {
|
||||
return Err("duplicate npm instance path".into());
|
||||
}
|
||||
let mut pending = VecDeque::new();
|
||||
for root in &env.roots {
|
||||
let path = child("", &root.name);
|
||||
if nodes.get(&path).map(|node| &node.spec) != Some(root) {
|
||||
return Err("npm root is not installed at its exact path".into());
|
||||
}
|
||||
pending.push_back(path);
|
||||
}
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = nodes
|
||||
.get(&path)
|
||||
.ok_or("npm edge references a missing instance")?;
|
||||
validate_source_owner(env, node)?;
|
||||
validate_path(&node.path)?;
|
||||
super::super::validate_name(&Ecosystem::Npm, &node.spec.name)?;
|
||||
super::super::validate_version(&Ecosystem::Npm, &node.spec.version)?;
|
||||
if node.spec.ecosystem != Ecosystem::Npm
|
||||
|| !node
|
||||
.path
|
||||
.ends_with(&format!("node_modules/{}", node.spec.name))
|
||||
{
|
||||
return Err("npm instance identity differs from its path".into());
|
||||
}
|
||||
if !lock
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec == node.source.archive && lib.spec.ecosystem == Ecosystem::Npm)
|
||||
{
|
||||
return Err("npm instance source archive is not locked".into());
|
||||
}
|
||||
if node.source.prefix != "package/" {
|
||||
validate_path(
|
||||
node.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.and_then(|path| path.strip_suffix('/'))
|
||||
.ok_or("invalid npm bundled source prefix")?,
|
||||
)?;
|
||||
} else if node.source.archive != node.spec {
|
||||
return Err("npm root archive identity differs from its instance".into());
|
||||
}
|
||||
let mut unique = BTreeSet::new();
|
||||
for edge in &node.edges {
|
||||
if !unique.insert((&edge.name, &edge.kind)) {
|
||||
return Err("duplicate npm dependency edge".into());
|
||||
}
|
||||
let range = edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let actual = visible(
|
||||
&nodes,
|
||||
&path,
|
||||
&edge.name,
|
||||
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
);
|
||||
match (&edge.target, &edge.omission) {
|
||||
(Some(target), None)
|
||||
if actual.is_some_and(|found| {
|
||||
found.path == *target
|
||||
&& Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&range))
|
||||
}) =>
|
||||
{
|
||||
pending.push_back(target.clone());
|
||||
}
|
||||
(None, Some(reason))
|
||||
if actual.is_none()
|
||||
&& (edge.kind == Kind::Optional
|
||||
&& *reason != Omission::OptionalPeerAbsent
|
||||
|| edge.kind == Kind::OptionalPeer
|
||||
&& matches!(
|
||||
reason,
|
||||
Omission::OptionalPeerAbsent | Omission::BuildFailure
|
||||
))
|
||||
&& (*reason != Omission::BuildFailure || env.build.is_some()) => {}
|
||||
_ => {
|
||||
return Err(
|
||||
"npm edge differs from Node lookup or its required range".into()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if reached.len() != nodes.len() {
|
||||
return Err("npm environment contains unreachable package instances".into());
|
||||
}
|
||||
}
|
||||
for worker in &lock.workers {
|
||||
if worker.ecosystem == "npm" {
|
||||
environment(lock, &worker.libs)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
use std::str::FromStr;
|
||||
@@ -0,0 +1,436 @@
|
||||
use super::*;
|
||||
use crate::libs::registry::{MAX_PACKAGES, MAX_RESOLUTION_ATTEMPTS, RegistryResolver, SearchError};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Request {
|
||||
consumer: Option<String>,
|
||||
edge: Edge,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct State {
|
||||
nodes: BTreeMap<String, Instance>,
|
||||
pending: VecDeque<Request>,
|
||||
}
|
||||
|
||||
struct Catalog<'a, 'r> {
|
||||
resolver: &'a RegistryResolver<'r>,
|
||||
runtime: &'a RegistryRuntime,
|
||||
indexes: BTreeMap<String, Vec<String>>,
|
||||
metadata: BTreeMap<(String, String), Value>,
|
||||
archives: BTreeMap<LibSpec, (LockedLib, Archive)>,
|
||||
attempts: usize,
|
||||
archive_bytes: usize,
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(
|
||||
resolver: &RegistryResolver<'_>,
|
||||
requested: &[LibSpec],
|
||||
runtime: &RegistryRuntime,
|
||||
) -> Result<(Vec<LockedLib>, Environment), String> {
|
||||
let roots = roots(requested);
|
||||
let mut state = State::default();
|
||||
for spec in &roots {
|
||||
state.pending.push_back(Request {
|
||||
consumer: None,
|
||||
edge: Edge {
|
||||
name: spec.name.clone(),
|
||||
requirement: format!("={}", spec.version),
|
||||
kind: Kind::Dependency,
|
||||
target: None,
|
||||
omission: None,
|
||||
},
|
||||
});
|
||||
}
|
||||
let mut catalog = Catalog {
|
||||
resolver,
|
||||
runtime,
|
||||
indexes: BTreeMap::new(),
|
||||
metadata: BTreeMap::new(),
|
||||
archives: BTreeMap::new(),
|
||||
attempts: 0,
|
||||
archive_bytes: 0,
|
||||
};
|
||||
let mut state = catalog.search(state).map_err(SearchError::message)?;
|
||||
let mut pending = roots
|
||||
.iter()
|
||||
.map(|root| child("", &root.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
pending.extend(
|
||||
state.nodes[&path]
|
||||
.edges
|
||||
.iter()
|
||||
.filter_map(|edge| edge.target.clone()),
|
||||
);
|
||||
}
|
||||
state.nodes.retain(|path, _| reached.contains(path));
|
||||
let used = state
|
||||
.nodes
|
||||
.values()
|
||||
.map(|node| node.source.archive.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let libraries = used
|
||||
.into_iter()
|
||||
.map(|spec| {
|
||||
catalog
|
||||
.archives
|
||||
.remove(&spec)
|
||||
.expect("resolved npm archive")
|
||||
.0
|
||||
})
|
||||
.collect();
|
||||
Ok((
|
||||
libraries,
|
||||
Environment {
|
||||
build: None,
|
||||
roots,
|
||||
instances: state.nodes.into_values().collect(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
impl Catalog<'_, '_> {
|
||||
fn archive(&mut self, spec: &LibSpec) -> Result<(), SearchError> {
|
||||
if self.archives.contains_key(spec) {
|
||||
return Ok(());
|
||||
}
|
||||
let document = self
|
||||
.resolver
|
||||
.registry_metadata(
|
||||
Ecosystem::Npm,
|
||||
&spec.name,
|
||||
&spec.version,
|
||||
&mut self.metadata,
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
let bytes = self
|
||||
.resolver
|
||||
.npm_archive(spec, &document)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
let archive = Archive::parse(spec, &bytes).map_err(SearchError::Fatal)?;
|
||||
self.archive_bytes = self
|
||||
.archive_bytes
|
||||
.checked_add(
|
||||
archive
|
||||
.files
|
||||
.iter()
|
||||
.map(|(_, bytes)| bytes.len())
|
||||
.sum::<usize>(),
|
||||
)
|
||||
.ok_or_else(|| SearchError::Fatal("npm archive cache size overflow".into()))?;
|
||||
if self.archive_bytes > 256 * 1024 * 1024 || self.archives.len() >= MAX_PACKAGES {
|
||||
return Err(SearchError::Fatal(
|
||||
"npm archive cache exceeds its 256 MiB/package budget".into(),
|
||||
));
|
||||
}
|
||||
let lib = self
|
||||
.resolver
|
||||
.lock_archive(
|
||||
spec,
|
||||
Vec::new(),
|
||||
self.runtime,
|
||||
"tgz",
|
||||
&bytes,
|
||||
"npm-instances-v1",
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.archives.insert(spec.clone(), (lib, archive));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn metadata(&self, node: &Instance) -> &Metadata {
|
||||
&self.archives[&node.source.archive].1.packages[&node.source.prefix]
|
||||
}
|
||||
|
||||
fn bind(
|
||||
state: &mut State,
|
||||
request: &Request,
|
||||
target: Option<String>,
|
||||
omission: Option<Omission>,
|
||||
) {
|
||||
if let Some(consumer) = &request.consumer {
|
||||
let edge = state
|
||||
.nodes
|
||||
.get_mut(consumer)
|
||||
.expect("pending npm consumer")
|
||||
.edges
|
||||
.iter_mut()
|
||||
.find(|edge| edge.name == request.edge.name && edge.kind == request.edge.kind)
|
||||
.expect("pending npm edge");
|
||||
edge.target = target;
|
||||
edge.omission = omission;
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&self, state: &mut State, path: &str, spec: &LibSpec) -> Result<(), SearchError> {
|
||||
for (prefix, metadata) in &self.archives[spec].1.packages {
|
||||
let relative = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
let location = if relative.is_empty() {
|
||||
path.into()
|
||||
} else {
|
||||
format!("{path}/{relative}")
|
||||
};
|
||||
let instance = Instance {
|
||||
path: location.clone(),
|
||||
spec: metadata.spec.clone(),
|
||||
source: Source {
|
||||
archive: spec.clone(),
|
||||
prefix: prefix.clone(),
|
||||
},
|
||||
bundled: if relative.is_empty() {
|
||||
self.archives[spec].1.bundled()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
},
|
||||
edges: metadata.edges.clone(),
|
||||
};
|
||||
if state.nodes.insert(location.clone(), instance).is_some() {
|
||||
return Err(SearchError::Conflict(
|
||||
"npm package installation paths collide".into(),
|
||||
));
|
||||
}
|
||||
for edge in &metadata.edges {
|
||||
if edge.kind != Kind::OptionalPeer {
|
||||
state.pending.push_back(Request {
|
||||
consumer: Some(location.clone()),
|
||||
edge: edge.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prune_bundled_orphans(&self, state: &mut State, root: &str) {
|
||||
let mut pending = VecDeque::from([root.to_owned()]);
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let Some(node) = state.nodes.get(&path) else {
|
||||
continue;
|
||||
};
|
||||
for edge in &self.metadata(node).edges {
|
||||
if let Some(found) = visible(
|
||||
&state.nodes,
|
||||
&path,
|
||||
&edge.name,
|
||||
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
) {
|
||||
pending.push_back(found.path.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let prefix = format!("{root}/");
|
||||
state.nodes.retain(|path, node| {
|
||||
node.source.prefix == "package/" || !path.starts_with(&prefix) || reached.contains(path)
|
||||
});
|
||||
}
|
||||
|
||||
fn search(&mut self, mut state: State) -> Result<State, SearchError> {
|
||||
self.attempts += 1;
|
||||
if self.attempts > MAX_RESOLUTION_ATTEMPTS
|
||||
|| state.nodes.len() > MAX_PACKAGES
|
||||
|| state.nodes.keys().any(|path| path.len() > 4096)
|
||||
{
|
||||
return Err(SearchError::Fatal(
|
||||
"npm instance graph exceeds its resolution budget".into(),
|
||||
));
|
||||
}
|
||||
while let Some(request) = state.pending.pop_front() {
|
||||
if request
|
||||
.consumer
|
||||
.as_ref()
|
||||
.is_some_and(|consumer| !state.nodes.contains_key(consumer))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let consumer = request.consumer.as_deref().unwrap_or("");
|
||||
let peer = request.edge.kind == Kind::Peer;
|
||||
let scope = if peer { parent(consumer) } else { consumer };
|
||||
let location = child(scope, &request.edge.name);
|
||||
let rule = request
|
||||
.edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| SearchError::Fatal(error.to_string()))?;
|
||||
let existing = visible(&state.nodes, consumer, &request.edge.name, peer).cloned();
|
||||
if let Some(found) = &existing {
|
||||
let eligible = self
|
||||
.metadata(found)
|
||||
.eligible(self.runtime)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
if !eligible
|
||||
&& request.edge.kind == Kind::Optional
|
||||
&& found.source.prefix != "package/"
|
||||
{
|
||||
let prefix = format!("{}/", found.path);
|
||||
state
|
||||
.nodes
|
||||
.retain(|path, _| path != &found.path && !path.starts_with(&prefix));
|
||||
let relative = found
|
||||
.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
let owner = found
|
||||
.path
|
||||
.strip_suffix(relative)
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
self.prune_bundled_orphans(&mut state, owner);
|
||||
Self::bind(&mut state, &request, None, Some(Omission::Target));
|
||||
continue;
|
||||
}
|
||||
// Ordinary dependencies get a private nested slot. Reuse an
|
||||
// ancestor only to close a real package cycle, never a sibling's
|
||||
// independently resolved peer context.
|
||||
let bundled = state.nodes.get(consumer).is_some_and(|node| {
|
||||
node.source.prefix != "package/" && node.source.archive == found.source.archive
|
||||
});
|
||||
let reusable = peer
|
||||
|| bundled
|
||||
|| found.path == location
|
||||
|| consumer == found.path
|
||||
|| consumer.starts_with(&format!("{}/", found.path));
|
||||
if reusable
|
||||
&& eligible
|
||||
&& Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&rule))
|
||||
{
|
||||
Self::bind(&mut state, &request, Some(found.path.clone()), None);
|
||||
continue;
|
||||
}
|
||||
if found.path == location {
|
||||
return Err(SearchError::Conflict(format!(
|
||||
"npm dependency/peer conflict for {} at {location}",
|
||||
request.edge.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(node) = request
|
||||
.consumer
|
||||
.as_ref()
|
||||
.and_then(|consumer| state.nodes.get(consumer))
|
||||
&& node.source.prefix != "package/"
|
||||
&& !peer
|
||||
{
|
||||
if request.edge.kind == Kind::Optional && existing.is_none() {
|
||||
Self::bind(&mut state, &request, None, Some(Omission::Unavailable));
|
||||
continue;
|
||||
}
|
||||
return Err(SearchError::Fatal(
|
||||
"npm bundled dependency closure is incomplete".into(),
|
||||
));
|
||||
}
|
||||
let versions = if let Some(versions) = self.indexes.get(&request.edge.name) {
|
||||
versions.clone()
|
||||
} else {
|
||||
let versions = self
|
||||
.resolver
|
||||
.registry_versions(Ecosystem::Npm, &request.edge.name)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.indexes
|
||||
.insert(request.edge.name.clone(), versions.clone());
|
||||
versions
|
||||
};
|
||||
let mut reason = Omission::Unavailable;
|
||||
for version in versions.into_iter().filter(|version| {
|
||||
Version::from_str(version).is_ok_and(|version| version.satisfies(&rule))
|
||||
}) {
|
||||
self.attempts += 1;
|
||||
if self.attempts > MAX_RESOLUTION_ATTEMPTS {
|
||||
return Err(SearchError::Fatal(
|
||||
"npm resolution exceeds its candidate budget".into(),
|
||||
));
|
||||
}
|
||||
let spec = LibSpec {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
name: request.edge.name.clone(),
|
||||
version,
|
||||
};
|
||||
self.archive(&spec)?;
|
||||
if !self.archives[&spec].1.packages["package/"]
|
||||
.eligible(self.runtime)
|
||||
.map_err(SearchError::Fatal)?
|
||||
{
|
||||
reason = Omission::Target;
|
||||
continue;
|
||||
}
|
||||
reason = Omission::Conflict;
|
||||
let mut candidate = state.clone();
|
||||
self.add(&mut candidate, &location, &spec)?;
|
||||
Self::bind(&mut candidate, &request, Some(location.clone()), None);
|
||||
match self.search(candidate) {
|
||||
Ok(resolved) => return Ok(resolved),
|
||||
Err(SearchError::Conflict(_)) => {}
|
||||
Err(fatal) => return Err(fatal),
|
||||
}
|
||||
}
|
||||
if request.edge.kind == Kind::Optional && existing.is_none() {
|
||||
Self::bind(&mut state, &request, None, Some(reason));
|
||||
continue;
|
||||
}
|
||||
return Err(SearchError::Conflict(format!(
|
||||
"no npm version satisfies {} at {location}",
|
||||
request.edge.name
|
||||
)));
|
||||
}
|
||||
let edges = state
|
||||
.nodes
|
||||
.values()
|
||||
.flat_map(|node| {
|
||||
node.edges.iter().map(|edge| Request {
|
||||
consumer: Some(node.path.clone()),
|
||||
edge: edge.clone(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for request in edges {
|
||||
let found = visible(
|
||||
&state.nodes,
|
||||
request.consumer.as_deref().unwrap(),
|
||||
&request.edge.name,
|
||||
matches!(request.edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
);
|
||||
if let Some(found) = found {
|
||||
let range = request
|
||||
.edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| SearchError::Fatal(error.to_string()))?;
|
||||
if !Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&range))
|
||||
{
|
||||
return Err(SearchError::Conflict(
|
||||
"npm dependency or peer conflicts with final Node lookup".into(),
|
||||
));
|
||||
}
|
||||
let target = found.path.clone();
|
||||
Self::bind(&mut state, &request, Some(target), None);
|
||||
} else if request.edge.kind == Kind::OptionalPeer {
|
||||
Self::bind(
|
||||
&mut state,
|
||||
&request,
|
||||
None,
|
||||
Some(Omission::OptionalPeerAbsent),
|
||||
);
|
||||
} else if request.edge.kind != Kind::Optional || request.edge.omission.is_none() {
|
||||
return Err(SearchError::Conflict(
|
||||
"npm required dependency is absent from final Node lookup".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
//! Authenticated Go checksum database receipts. Trust comes from the verifier
|
||||
//! supplied by the caller, never from a key stored in a project's lock file.
|
||||
//! Formats follow golang.org/x/mod/sumdb/{note,tlog,dirhash} and RFC 6962.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use ring::signature::{ED25519, UnparsedPublicKey};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{RegistryTransport, go_path, zip_entries};
|
||||
|
||||
mod merkle;
|
||||
|
||||
pub const OFFICIAL_VERIFIER: &str =
|
||||
"sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8";
|
||||
const MAX_NOTE_BYTES: usize = 16 * 1024;
|
||||
const MAX_RECORDS: usize = 1024;
|
||||
const MAX_EVIDENCE_BYTES: usize = 16 * 1024 * 1024;
|
||||
type Hash = [u8; 32];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct Tree {
|
||||
size: u64,
|
||||
hash: Hash,
|
||||
}
|
||||
|
||||
fn encode_hash(hash: Hash) -> String {
|
||||
STANDARD.encode(hash)
|
||||
}
|
||||
|
||||
fn decode_hash(value: &str) -> Result<Hash, String> {
|
||||
STANDARD
|
||||
.decode(value)
|
||||
.map_err(|_| "sumdb hash is not base64")?
|
||||
.try_into()
|
||||
.map_err(|_| "sumdb hash must contain 32 bytes".into())
|
||||
}
|
||||
|
||||
/// Explicit verifier injection supports registry protocol tests and author
|
||||
/// tooling. Released project commands always select `official()`.
|
||||
#[derive(Clone)]
|
||||
pub struct Verifier {
|
||||
name: String,
|
||||
key_hash: [u8; 4],
|
||||
key: [u8; 32],
|
||||
}
|
||||
|
||||
impl Verifier {
|
||||
pub fn official() -> Self {
|
||||
Self::new(OFFICIAL_VERIFIER).expect("fixed official sumdb verifier")
|
||||
}
|
||||
|
||||
pub fn new(encoded: &str) -> Result<Self, String> {
|
||||
// '+' also occurs inside base64; only the first two separators delimit.
|
||||
let (name, rest) = encoded.split_once('+').ok_or("invalid sumdb verifier")?;
|
||||
let (key_hash, encoded_key) = rest.split_once('+').ok_or("invalid sumdb verifier")?;
|
||||
if name.is_empty() || name.chars().any(char::is_whitespace) || key_hash.len() != 8 {
|
||||
return Err("invalid sumdb verifier identity".into());
|
||||
}
|
||||
let expected = u32::from_str_radix(key_hash, 16)
|
||||
.map_err(|_| "invalid sumdb key hash")?
|
||||
.to_be_bytes();
|
||||
let key = STANDARD
|
||||
.decode(encoded_key)
|
||||
.map_err(|_| "invalid sumdb verifier encoding")?;
|
||||
if key.len() != 33 || key[0] != 1 {
|
||||
return Err("sumdb requires an Ed25519 verifier".into());
|
||||
}
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(name.as_bytes());
|
||||
hash.update(b"\n");
|
||||
hash.update(&key);
|
||||
if hash.finalize()[..4] != expected {
|
||||
return Err("sumdb verifier key hash mismatch".into());
|
||||
}
|
||||
Ok(Self {
|
||||
name: name.into(),
|
||||
key_hash: expected,
|
||||
key: key[1..].try_into().expect("validated Ed25519 key length"),
|
||||
})
|
||||
}
|
||||
|
||||
fn checkpoint(&self, note: &str) -> Result<Tree, String> {
|
||||
if note.len() > MAX_NOTE_BYTES || note.bytes().any(|byte| byte < 32 && byte != b'\n') {
|
||||
return Err("invalid sumdb checkpoint note".into());
|
||||
}
|
||||
let (text, signatures) = note
|
||||
.rsplit_once("\n\n")
|
||||
.ok_or("sumdb checkpoint lacks signatures")?;
|
||||
let message = format!("{text}\n");
|
||||
if !signatures.ends_with('\n') {
|
||||
return Err("unterminated sumdb signature".into());
|
||||
}
|
||||
let mut verified = false;
|
||||
for (index, line) in signatures.lines().enumerate() {
|
||||
if index >= 100 {
|
||||
return Err("sumdb checkpoint has too many signatures".into());
|
||||
}
|
||||
let (name, encoded) = line
|
||||
.strip_prefix("— ")
|
||||
.and_then(|line| line.split_once(' '))
|
||||
.ok_or("malformed sumdb signature")?;
|
||||
let signature = STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|_| "invalid sumdb signature encoding")?;
|
||||
if name.is_empty() || name.chars().any(char::is_whitespace) || signature.len() < 5 {
|
||||
return Err("malformed sumdb signature identity".into());
|
||||
}
|
||||
if name == self.name && signature[..4] == self.key_hash {
|
||||
UnparsedPublicKey::new(&ED25519, self.key)
|
||||
.verify(message.as_bytes(), &signature[4..])
|
||||
.map_err(|_| "sumdb checkpoint signature verification failed")?;
|
||||
verified = true;
|
||||
}
|
||||
}
|
||||
if !verified {
|
||||
return Err("sumdb checkpoint has no trusted signature".into());
|
||||
}
|
||||
let mut lines = text.lines();
|
||||
if lines.next() != Some("go.sum database tree") {
|
||||
return Err("unsupported sumdb checkpoint format".into());
|
||||
}
|
||||
let size_text = lines.next().ok_or("sumdb checkpoint lacks tree size")?;
|
||||
let size = decimal(size_text)?;
|
||||
let hash = decode_hash(lines.next().ok_or("sumdb checkpoint lacks root hash")?)?;
|
||||
let empty: Hash = Sha256::digest([]).into();
|
||||
if size == 0 && hash != empty {
|
||||
return Err("invalid sumdb empty root".into());
|
||||
}
|
||||
Ok(Tree { size, hash })
|
||||
}
|
||||
}
|
||||
|
||||
fn decimal(value: &str) -> Result<u64, String> {
|
||||
let number = value
|
||||
.parse::<u64>()
|
||||
.map_err(|_| "invalid sumdb record/tree number")?;
|
||||
if number > i64::MAX as u64 || value != number.to_string() {
|
||||
return Err("noncanonical sumdb record/tree number".into());
|
||||
}
|
||||
Ok(number)
|
||||
}
|
||||
|
||||
struct Lookup<'a> {
|
||||
id: u64,
|
||||
text: &'a str,
|
||||
checkpoint: &'a str,
|
||||
}
|
||||
|
||||
fn lookup(value: &str) -> Result<Lookup<'_>, String> {
|
||||
if value.len() > MAX_NOTE_BYTES || value.bytes().any(|byte| byte < 32 && byte != b'\n') {
|
||||
return Err("invalid sumdb lookup text".into());
|
||||
}
|
||||
let (id, rest) = value
|
||||
.split_once('\n')
|
||||
.ok_or("sumdb lookup lacks record number")?;
|
||||
let end = rest
|
||||
.find("\n\n")
|
||||
.ok_or("sumdb lookup lacks record terminator")?;
|
||||
Ok(Lookup {
|
||||
id: decimal(id)?,
|
||||
text: &rest[..end + 1],
|
||||
checkpoint: &rest[end + 2..],
|
||||
})
|
||||
}
|
||||
|
||||
fn hashes(record: &str, module: &str, version: &str) -> Result<(String, String), String> {
|
||||
let version = format!("v{version}");
|
||||
let mod_version = format!("{version}/go.mod");
|
||||
let mut values = BTreeMap::new();
|
||||
for line in record.lines() {
|
||||
let parts = line.split(' ').collect::<Vec<_>>();
|
||||
if parts.len() != 3
|
||||
|| parts[0] != module
|
||||
|| (parts[1] != version && parts[1] != mod_version)
|
||||
{
|
||||
return Err("sumdb lookup record has a different module/version identity".into());
|
||||
}
|
||||
decode_hash(
|
||||
parts[2]
|
||||
.strip_prefix("h1:")
|
||||
.ok_or("sumdb record requires h1 hashes")?,
|
||||
)?;
|
||||
if values.insert(parts[1], parts[2]).is_some() {
|
||||
return Err("sumdb lookup repeats a module checksum".into());
|
||||
}
|
||||
}
|
||||
Ok((
|
||||
values
|
||||
.get(mod_version.as_str())
|
||||
.ok_or("sumdb lookup lacks go.mod hash")?
|
||||
.to_string(),
|
||||
values
|
||||
.get(version.as_str())
|
||||
.ok_or("sumdb lookup lacks module ZIP hash")?
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn dirhash(files: &[(String, Vec<u8>)]) -> Result<String, String> {
|
||||
let mut files = files.iter().collect::<Vec<_>>();
|
||||
files.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut hash = Sha256::new();
|
||||
for (name, bytes) in files {
|
||||
if name.contains('\n') || !seen.insert(name) {
|
||||
return Err("invalid or duplicate Go dirhash filename".into());
|
||||
}
|
||||
hash.update(format!("{} {name}\n", crate::libs::sha256(bytes)));
|
||||
}
|
||||
Ok(format!("h1:{}", STANDARD.encode(hash.finalize())))
|
||||
}
|
||||
|
||||
pub fn mod_hash(bytes: &[u8]) -> String {
|
||||
// The Go protocol hashes a synthetic directory containing only go.mod.
|
||||
dirhash(&[("go.mod".into(), bytes.to_vec())]).expect("fixed dirhash filename")
|
||||
}
|
||||
|
||||
pub fn zip_hash(module: &str, version: &str, bytes: &[u8]) -> Result<String, String> {
|
||||
let files = zip_entries(bytes)?;
|
||||
let prefix = format!("{module}@v{version}/");
|
||||
if files.iter().any(|(path, _)| !path.starts_with(&prefix)) {
|
||||
return Err("Go module archive has a mismatched module root".into());
|
||||
}
|
||||
dirhash(&files)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Record {
|
||||
pub module: String,
|
||||
pub version: String,
|
||||
pub go_mod: String,
|
||||
pub lookup: String,
|
||||
pub inclusion: Vec<String>,
|
||||
pub consistency: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Evidence {
|
||||
pub previous: Option<String>,
|
||||
pub checkpoint: String,
|
||||
pub consistency: Vec<String>,
|
||||
pub records: Vec<Record>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Verified {
|
||||
pub checkpoint: String,
|
||||
checksums: BTreeMap<(String, String), String>,
|
||||
}
|
||||
|
||||
impl Verified {
|
||||
pub fn verify_zip(&self, module: &str, version: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
let expected = self
|
||||
.checksums
|
||||
.get(&(module.into(), version.into()))
|
||||
.ok_or("Go module lacks authenticated sumdb evidence")?;
|
||||
if zip_hash(module, version, bytes)? != *expected {
|
||||
return Err("Go module ZIP does not match authenticated sumdb h1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn contains(&self, module: &str, version: &str) -> bool {
|
||||
self.checksums
|
||||
.contains_key(&(module.into(), version.into()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
pub fn verify(&self, verifier: &Verifier) -> Result<Verified, String> {
|
||||
if self.records.len() > MAX_RECORDS
|
||||
|| serde_json::to_vec(self).map_err(|e| e.to_string())?.len() > MAX_EVIDENCE_BYTES
|
||||
{
|
||||
return Err("sumdb evidence exceeds its size budget".into());
|
||||
}
|
||||
let mut current = verifier.checkpoint(&self.checkpoint)?;
|
||||
if let Some(previous) = &self.previous {
|
||||
let older = verifier.checkpoint(previous)?;
|
||||
if older.size > current.size {
|
||||
return Err("sumdb checkpoint rollback rejected".into());
|
||||
}
|
||||
merkle::check_consistency(&self.consistency, older, current)?;
|
||||
} else if !self.consistency.is_empty() {
|
||||
return Err("sumdb first checkpoint has an unexpected consistency proof".into());
|
||||
}
|
||||
let mut latest = self.checkpoint.clone();
|
||||
let mut checksums = BTreeMap::new();
|
||||
for record in &self.records {
|
||||
let entry = lookup(&record.lookup)?;
|
||||
let tree = verifier.checkpoint(entry.checkpoint)?;
|
||||
merkle::check_record(&record.inclusion, tree, entry.id, entry.text.as_bytes())?;
|
||||
let (older, newer) = if tree.size < current.size {
|
||||
(tree, current)
|
||||
} else {
|
||||
(current, tree)
|
||||
};
|
||||
merkle::check_consistency(&record.consistency, older, newer)?;
|
||||
let (mod_h1, zip_h1) = hashes(entry.text, &record.module, &record.version)?;
|
||||
if mod_hash(record.go_mod.as_bytes()) != mod_h1 {
|
||||
return Err("Go go.mod does not match authenticated sumdb h1".into());
|
||||
}
|
||||
if checksums
|
||||
.insert((record.module.clone(), record.version.clone()), zip_h1)
|
||||
.is_some()
|
||||
{
|
||||
return Err("sumdb evidence repeats a module/version".into());
|
||||
}
|
||||
if tree.size > current.size {
|
||||
current = tree;
|
||||
latest = entry.checkpoint.into();
|
||||
}
|
||||
}
|
||||
Ok(Verified {
|
||||
checkpoint: latest,
|
||||
checksums,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One explicit resolution sequence owns the checkpoint. The project lock is
|
||||
/// its persistent anchor; deleting that lock intentionally starts first trust.
|
||||
pub struct ChecksumDatabase {
|
||||
verifier: Verifier,
|
||||
latest: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
impl ChecksumDatabase {
|
||||
pub fn verifier(&self) -> &Verifier {
|
||||
&self.verifier
|
||||
}
|
||||
|
||||
pub fn new(verifier: Verifier, previous: Option<String>) -> Result<Self, String> {
|
||||
if let Some(note) = &previous {
|
||||
verifier.checkpoint(note)?;
|
||||
}
|
||||
Ok(Self {
|
||||
verifier,
|
||||
latest: Mutex::new(previous),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn official(previous: Option<String>) -> Result<Self, String> {
|
||||
Self::new(Verifier::official(), previous)
|
||||
}
|
||||
|
||||
pub(super) fn begin<'a>(
|
||||
&'a self,
|
||||
transport: &'a dyn RegistryTransport,
|
||||
) -> Result<Session<'a>, String> {
|
||||
let guard = self
|
||||
.latest
|
||||
.lock()
|
||||
.map_err(|_| "sumdb checkpoint state poisoned")?;
|
||||
let checkpoint = String::from_utf8(transport.get_sumdb("/latest", MAX_NOTE_BYTES)?)
|
||||
.map_err(|_| "sumdb checkpoint is not UTF-8")?;
|
||||
let current = self.verifier.checkpoint(&checkpoint)?;
|
||||
let mut tiles = merkle::Tiles::new(transport);
|
||||
let consistency = if let Some(previous) = &*guard {
|
||||
let older = self.verifier.checkpoint(previous)?;
|
||||
if older.size > current.size {
|
||||
return Err("sumdb checkpoint rollback rejected".into());
|
||||
}
|
||||
tiles.consistency(older, current)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let evidence = Evidence {
|
||||
previous: guard.clone(),
|
||||
checkpoint: checkpoint.clone(),
|
||||
consistency,
|
||||
records: Vec::new(),
|
||||
};
|
||||
Ok(Session {
|
||||
verifier: &self.verifier,
|
||||
transport,
|
||||
guard,
|
||||
tiles,
|
||||
current,
|
||||
latest: checkpoint,
|
||||
evidence,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Session<'a> {
|
||||
verifier: &'a Verifier,
|
||||
transport: &'a dyn RegistryTransport,
|
||||
guard: MutexGuard<'a, Option<String>>,
|
||||
tiles: merkle::Tiles<'a>,
|
||||
current: Tree,
|
||||
latest: String,
|
||||
evidence: Evidence,
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn verify_mod(
|
||||
&mut self,
|
||||
module: &str,
|
||||
version: &str,
|
||||
source: &[u8],
|
||||
) -> Result<(), String> {
|
||||
if let Some(record) = self
|
||||
.evidence
|
||||
.records
|
||||
.iter()
|
||||
.find(|record| record.module == module && record.version == version)
|
||||
{
|
||||
if record.go_mod.as_bytes() != source {
|
||||
return Err("Go go.mod changed during resolution".into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if self.evidence.records.len() >= MAX_RECORDS {
|
||||
return Err("sumdb resolution exceeds record limit".into());
|
||||
}
|
||||
let path = format!("/lookup/{}@v{}", go_path(module), go_path(version));
|
||||
let response = String::from_utf8(self.transport.get_sumdb(&path, MAX_NOTE_BYTES)?)
|
||||
.map_err(|_| "sumdb lookup is not UTF-8")?;
|
||||
let entry = lookup(&response)?;
|
||||
let tree = self.verifier.checkpoint(entry.checkpoint)?;
|
||||
let inclusion = self.tiles.inclusion(tree.size, entry.id)?;
|
||||
merkle::check_record(&inclusion, tree, entry.id, entry.text.as_bytes())?;
|
||||
let (older, newer) = if tree.size < self.current.size {
|
||||
(tree, self.current)
|
||||
} else {
|
||||
(self.current, tree)
|
||||
};
|
||||
let consistency = self.tiles.consistency(older, newer)?;
|
||||
let (expected, _) = hashes(entry.text, module, version)?;
|
||||
if mod_hash(source) != expected {
|
||||
return Err("Go go.mod does not match authenticated sumdb h1".into());
|
||||
}
|
||||
if tree.size > self.current.size {
|
||||
self.current = tree;
|
||||
self.latest = entry.checkpoint.into();
|
||||
}
|
||||
let record = Record {
|
||||
module: module.into(),
|
||||
version: version.into(),
|
||||
go_mod: String::from_utf8(source.to_vec()).map_err(|_| "Go go.mod is not UTF-8")?,
|
||||
lookup: response,
|
||||
inclusion,
|
||||
consistency,
|
||||
};
|
||||
self.evidence.records.push(record);
|
||||
if serde_json::to_vec(&self.evidence)
|
||||
.map_err(|e| e.to_string())?
|
||||
.len()
|
||||
> MAX_EVIDENCE_BYTES
|
||||
{
|
||||
return Err("sumdb evidence exceeds its size budget".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn verify_zip(
|
||||
&self,
|
||||
module: &str,
|
||||
version: &str,
|
||||
bytes: &[u8],
|
||||
) -> Result<(), String> {
|
||||
let record = self
|
||||
.evidence
|
||||
.records
|
||||
.iter()
|
||||
.find(|record| record.module == module && record.version == version)
|
||||
.ok_or("Go module ZIP has no authenticated metadata")?;
|
||||
let (_, expected) = hashes(lookup(&record.lookup)?.text, module, version)?;
|
||||
if zip_hash(module, version, bytes)? != expected {
|
||||
return Err("Go module ZIP does not match authenticated sumdb h1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn finish(mut self) -> Evidence {
|
||||
*self.guard = Some(self.latest);
|
||||
self.evidence
|
||||
}
|
||||
}
|
||||
|
||||
pub fn verify_chain(evidence: &[Evidence], verifier: &Verifier) -> Result<Verified, String> {
|
||||
let mut latest = None;
|
||||
let mut checksums = BTreeMap::new();
|
||||
if evidence.len() > MAX_RECORDS {
|
||||
return Err("sumdb resolution history exceeds its limit".into());
|
||||
}
|
||||
let mut bytes = 0usize;
|
||||
for receipt in evidence {
|
||||
bytes = bytes
|
||||
.checked_add(
|
||||
serde_json::to_vec(receipt)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len(),
|
||||
)
|
||||
.ok_or("sumdb resolution history size overflow")?;
|
||||
if bytes > MAX_EVIDENCE_BYTES {
|
||||
return Err("sumdb resolution history exceeds its size budget".into());
|
||||
}
|
||||
if let Some(previous) = &latest
|
||||
&& receipt.previous.as_ref() != Some(previous)
|
||||
{
|
||||
return Err("sumdb resolution checkpoints are not connected".into());
|
||||
}
|
||||
let verified = receipt.verify(verifier)?;
|
||||
for (identity, hash) in verified.checksums {
|
||||
if let Some(old) = checksums.insert(identity, hash.clone())
|
||||
&& old != hash
|
||||
{
|
||||
return Err("sumdb resolutions disagree about module content".into());
|
||||
}
|
||||
}
|
||||
latest = Some(verified.checkpoint);
|
||||
}
|
||||
Ok(Verified {
|
||||
checkpoint: latest.unwrap_or_default(),
|
||||
checksums,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
//! RFC 6962 tree proofs and Go's height-eight tile addressing.
|
||||
//! Protocol reference: golang.org/x/mod/sumdb/tlog (BSD-3-Clause).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{Hash, Tree, decode_hash, encode_hash};
|
||||
use crate::libs::RegistryTransport;
|
||||
|
||||
pub(super) fn leaf(bytes: &[u8]) -> Hash {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update([0]);
|
||||
hash.update(bytes);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
fn node(left: Hash, right: Hash) -> Hash {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update([1]);
|
||||
hash.update(left);
|
||||
hash.update(right);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
fn split(size: u64) -> u64 {
|
||||
1 << (63 - (size - 1).leading_zeros())
|
||||
}
|
||||
|
||||
pub(super) fn check_record(
|
||||
proof: &[String],
|
||||
tree: Tree,
|
||||
id: u64,
|
||||
bytes: &[u8],
|
||||
) -> Result<(), String> {
|
||||
if id >= tree.size || proof.len() > 63 {
|
||||
return Err("sumdb record proof has invalid bounds".into());
|
||||
}
|
||||
fn root(proof: &[String], size: u64, id: u64, value: Hash) -> Result<Hash, String> {
|
||||
if size == 1 {
|
||||
return if proof.is_empty() {
|
||||
Ok(value)
|
||||
} else {
|
||||
Err("sumdb inclusion proof has surplus hashes".into())
|
||||
};
|
||||
}
|
||||
let (last, rest) = proof
|
||||
.split_last()
|
||||
.ok_or("sumdb inclusion proof is incomplete")?;
|
||||
let sibling = decode_hash(last)?;
|
||||
let left = split(size);
|
||||
if id < left {
|
||||
Ok(node(root(rest, left, id, value)?, sibling))
|
||||
} else {
|
||||
Ok(node(sibling, root(rest, size - left, id - left, value)?))
|
||||
}
|
||||
}
|
||||
if root(proof, tree.size, id, leaf(bytes))? != tree.hash {
|
||||
return Err("sumdb inclusion proof does not match signed checkpoint".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn check_consistency(proof: &[String], older: Tree, newer: Tree) -> Result<(), String> {
|
||||
if older.size > newer.size || proof.len() > 63 {
|
||||
return Err("sumdb consistency proof has invalid bounds".into());
|
||||
}
|
||||
if older.size == 0 {
|
||||
let empty: Hash = Sha256::digest([]).into();
|
||||
return if proof.is_empty() && older.hash == empty {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("sumdb empty checkpoint is invalid".into())
|
||||
};
|
||||
}
|
||||
fn roots(
|
||||
proof: &[String],
|
||||
lo: u64,
|
||||
size: u64,
|
||||
old_size: u64,
|
||||
old_hash: Hash,
|
||||
) -> Result<(Hash, Hash), String> {
|
||||
if old_size == size {
|
||||
return if lo == 0 && proof.is_empty() {
|
||||
Ok((old_hash, old_hash))
|
||||
} else if lo != 0 && proof.len() == 1 {
|
||||
let hash = decode_hash(&proof[0])?;
|
||||
Ok((hash, hash))
|
||||
} else {
|
||||
Err("sumdb consistency proof has surplus or missing hashes".into())
|
||||
};
|
||||
}
|
||||
let (last, rest) = proof
|
||||
.split_last()
|
||||
.ok_or("sumdb consistency proof is incomplete")?;
|
||||
let sibling = decode_hash(last)?;
|
||||
let left = split(size);
|
||||
if old_size <= left {
|
||||
let (old, new) = roots(rest, lo, left, old_size, old_hash)?;
|
||||
Ok((old, node(new, sibling)))
|
||||
} else {
|
||||
let (old, new) = roots(rest, lo + left, size - left, old_size - left, old_hash)?;
|
||||
Ok((node(sibling, old), node(sibling, new)))
|
||||
}
|
||||
}
|
||||
let (old, new) = roots(proof, 0, newer.size, older.size, older.hash)?;
|
||||
if old != older.hash || new != newer.hash {
|
||||
return Err("sumdb consistency proof detects a conflicting history".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetched tiles are only inputs to proofs; no byte is trusted merely because
|
||||
/// it is cached. Persisted Evidence contains only completely verified proofs.
|
||||
pub(super) struct Tiles<'a> {
|
||||
transport: &'a dyn RegistryTransport,
|
||||
cache: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl<'a> Tiles<'a> {
|
||||
pub(super) fn new(transport: &'a dyn RegistryTransport) -> Self {
|
||||
Self {
|
||||
transport,
|
||||
cache: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn subtree(&mut self, lo: u64, size: u64, tree_size: u64) -> Result<Hash, String> {
|
||||
if size.is_power_of_two() {
|
||||
let level = size.trailing_zeros();
|
||||
let tile_level = level / 8;
|
||||
let tile_scale = tile_level * 8;
|
||||
let count = 1u64 << (level % 8);
|
||||
let first = lo >> tile_scale;
|
||||
let index = first / 256;
|
||||
let offset = first % 256;
|
||||
let width = (tree_size >> tile_scale)
|
||||
.checked_sub(index * 256)
|
||||
.ok_or("sumdb tile lies outside checkpoint")?
|
||||
.min(256);
|
||||
if offset + count > width {
|
||||
return Err("sumdb tile does not cover requested subtree".into());
|
||||
}
|
||||
let mut chunks = vec![format!("{:03}", index % 1000)];
|
||||
let mut prefix = index / 1000;
|
||||
while prefix > 0 {
|
||||
chunks.push(format!("x{:03}", prefix % 1000));
|
||||
prefix /= 1000;
|
||||
}
|
||||
chunks.reverse();
|
||||
let full = format!("/tile/8/{tile_level}/{}", chunks.join("/"));
|
||||
let path = if width == 256 {
|
||||
full.clone()
|
||||
} else {
|
||||
format!("{full}.p/{width}")
|
||||
};
|
||||
if !self.cache.contains_key(&path) {
|
||||
if self.cache.len() >= 4096 {
|
||||
return Err("sumdb tile cache exceeds its 32 MiB budget".into());
|
||||
}
|
||||
let bytes = match self.transport.get_sumdb(&path, 8192) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) if width < 256 => {
|
||||
let mut bytes = self.transport.get_sumdb(&full, 8192)?;
|
||||
if bytes.len() != 8192 {
|
||||
return Err("sumdb full tile has wrong length".into());
|
||||
}
|
||||
bytes.truncate(width as usize * 32);
|
||||
bytes
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if bytes.len() != width as usize * 32 {
|
||||
return Err("sumdb tile has wrong length".into());
|
||||
}
|
||||
self.cache.insert(path.clone(), bytes);
|
||||
}
|
||||
fn fold(bytes: &[u8]) -> Hash {
|
||||
if bytes.len() == 32 {
|
||||
return bytes.try_into().expect("one tile hash");
|
||||
}
|
||||
let (left, right) = bytes.split_at(bytes.len() / 2);
|
||||
node(fold(left), fold(right))
|
||||
}
|
||||
return Ok(fold(
|
||||
&self.cache[&path][offset as usize * 32..(offset + count) as usize * 32],
|
||||
));
|
||||
}
|
||||
let left = split(size);
|
||||
Ok(node(
|
||||
self.subtree(lo, left, tree_size)?,
|
||||
self.subtree(lo + left, size - left, tree_size)?,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn inclusion(&mut self, size: u64, id: u64) -> Result<Vec<String>, String> {
|
||||
fn walk(
|
||||
tiles: &mut Tiles<'_>,
|
||||
lo: u64,
|
||||
size: u64,
|
||||
id: u64,
|
||||
total: u64,
|
||||
proof: &mut Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
if size == 1 {
|
||||
return Ok(());
|
||||
}
|
||||
let left = split(size);
|
||||
let sibling = if id < lo + left {
|
||||
walk(tiles, lo, left, id, total, proof)?;
|
||||
tiles.subtree(lo + left, size - left, total)?
|
||||
} else {
|
||||
walk(tiles, lo + left, size - left, id, total, proof)?;
|
||||
tiles.subtree(lo, left, total)?
|
||||
};
|
||||
proof.push(encode_hash(sibling));
|
||||
Ok(())
|
||||
}
|
||||
if id >= size {
|
||||
return Err("sumdb record lies outside signed checkpoint".into());
|
||||
}
|
||||
let mut proof = Vec::new();
|
||||
walk(self, 0, size, id, size, &mut proof)?;
|
||||
Ok(proof)
|
||||
}
|
||||
|
||||
pub(super) fn consistency(&mut self, older: Tree, newer: Tree) -> Result<Vec<String>, String> {
|
||||
fn walk(
|
||||
tiles: &mut Tiles<'_>,
|
||||
lo: u64,
|
||||
size: u64,
|
||||
old_size: u64,
|
||||
total: u64,
|
||||
proof: &mut Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
if old_size == size {
|
||||
if lo != 0 {
|
||||
proof.push(encode_hash(tiles.subtree(lo, size, total)?));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let left = split(size);
|
||||
let sibling = if old_size <= left {
|
||||
walk(tiles, lo, left, old_size, total, proof)?;
|
||||
tiles.subtree(lo + left, size - left, total)?
|
||||
} else {
|
||||
walk(tiles, lo + left, size - left, old_size - left, total, proof)?;
|
||||
tiles.subtree(lo, left, total)?
|
||||
};
|
||||
proof.push(encode_hash(sibling));
|
||||
Ok(())
|
||||
}
|
||||
let mut proof = Vec::new();
|
||||
if older.size != 0 {
|
||||
walk(self, 0, newer.size, older.size, newer.size, &mut proof)?;
|
||||
}
|
||||
check_consistency(&proof, older, newer)?;
|
||||
Ok(proof)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
use std::env;
|
||||
use std::ffi::OsString;
|
||||
use std::io::{self, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Command, ExitCode};
|
||||
|
||||
use dever_cli::libs;
|
||||
use dever_cli::toolchain::BuildTarget;
|
||||
use dever_core::source::SourceMap;
|
||||
|
||||
mod api;
|
||||
mod compile;
|
||||
mod format;
|
||||
mod test_runner;
|
||||
mod worker;
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../../../test/dever-cli-tests/build_arguments.rs"]
|
||||
mod build_arguments;
|
||||
|
||||
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
|
||||
|
||||
enum Action {
|
||||
Check,
|
||||
Api {
|
||||
output: Option<PathBuf>,
|
||||
},
|
||||
Format {
|
||||
check: bool,
|
||||
},
|
||||
Test,
|
||||
Run {
|
||||
tenant_id: Option<i64>,
|
||||
arguments: Vec<OsString>,
|
||||
},
|
||||
Build {
|
||||
output: PathBuf,
|
||||
target: BuildTarget,
|
||||
},
|
||||
Clean,
|
||||
TenantMigrate {
|
||||
tenant_id: i64,
|
||||
},
|
||||
TenantOwner {
|
||||
tenant_id: i64,
|
||||
site: String,
|
||||
user_id: i64,
|
||||
},
|
||||
TenantComponent {
|
||||
tenant_id: i64,
|
||||
component: String,
|
||||
enabled: bool,
|
||||
},
|
||||
Lib {
|
||||
command: String,
|
||||
specs: Vec<String>,
|
||||
target: BuildTarget,
|
||||
},
|
||||
Package {
|
||||
command: String,
|
||||
specs: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Action {
|
||||
fn loads_settings(&self) -> bool {
|
||||
matches!(self, Self::Check | Self::Api { .. }) || self.compiles_application()
|
||||
}
|
||||
|
||||
fn compiles_application(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Run { .. }
|
||||
| Self::Build { .. }
|
||||
| Self::TenantMigrate { .. }
|
||||
| Self::TenantOwner { .. }
|
||||
| Self::TenantComponent { .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match execute() {
|
||||
Ok(code) => code,
|
||||
Err(error) => {
|
||||
// Preserve failure status even if stderr is unavailable.
|
||||
let _ = writeln!(io::stderr().lock(), "{error}");
|
||||
ExitCode::FAILURE
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn execute() -> Result<ExitCode, String> {
|
||||
let arguments: Vec<_> = env::args_os().skip(1).collect();
|
||||
if arguments.first().is_some_and(|argument| argument == "new") {
|
||||
let (destination, markdown) = match &arguments[1..] {
|
||||
[destination] if !destination.to_string_lossy().starts_with('-') => {
|
||||
(destination, false)
|
||||
}
|
||||
[destination, option]
|
||||
if !destination.to_string_lossy().starts_with('-') && option == "--markdown" =>
|
||||
{
|
||||
(destination, true)
|
||||
}
|
||||
_ => return Err("Usage: dever new <new-directory> [--markdown]".into()),
|
||||
};
|
||||
dever_cli::project::create(std::path::Path::new(destination), markdown)?;
|
||||
writeln!(
|
||||
io::stdout().lock(),
|
||||
"created Dever project {}",
|
||||
destination.to_string_lossy()
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if arguments.as_slice() == [OsString::from("--dever-compile-worker")] {
|
||||
worker::execute()?;
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if arguments.as_slice() == [OsString::from("--dever-health")] {
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if arguments.as_slice() == [OsString::from("--version")] {
|
||||
writeln!(io::stdout().lock(), "{}", env!("CARGO_PKG_VERSION"))
|
||||
.map_err(|error| error.to_string())?;
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if arguments.as_slice() == [OsString::from("--help")] {
|
||||
writeln!(
|
||||
io::stdout().lock(),
|
||||
"Create a project: dever new <new-directory> [--markdown]"
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
writeln!(io::stdout().lock(), "{USAGE}").map_err(|error| error.to_string())?;
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
let (root, action) = arguments_for(&arguments)?;
|
||||
if let Action::Lib {
|
||||
command,
|
||||
specs,
|
||||
target,
|
||||
} = &action
|
||||
{
|
||||
let message = dever_cli::libs::execute_for_target(command, &root, specs, *target)?;
|
||||
if !message.is_empty() {
|
||||
writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?;
|
||||
}
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if let Action::Package { command, specs } = &action {
|
||||
let message = dever_cli::packages::execute(command, &root, specs)?;
|
||||
if !message.is_empty() {
|
||||
writeln!(io::stdout().lock(), "{message}").map_err(|error| error.to_string())?;
|
||||
}
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if matches!(action, Action::Clean) {
|
||||
let summary = dever_core::native::clean_project_artifacts(&root)
|
||||
.map_err(|error| format!("cannot clean Dever artifacts: {error}"))?;
|
||||
writeln!(
|
||||
io::stdout().lock(),
|
||||
"cleaned {} directories, {} files, {} bytes",
|
||||
summary.directories,
|
||||
summary.files,
|
||||
summary.bytes,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
let source_root = root.join("module");
|
||||
let packages = if matches!(action, Action::Format { .. }) {
|
||||
Vec::new()
|
||||
} else {
|
||||
dever_cli::packages::sources(&root)?
|
||||
};
|
||||
let sources = SourceMap::load_with_packages(
|
||||
&source_root,
|
||||
matches!(action, Action::Test | Action::Format { .. })
|
||||
.then(|| root.join("test"))
|
||||
.as_deref(),
|
||||
&packages,
|
||||
)
|
||||
.map_err(|errors| {
|
||||
errors
|
||||
.iter()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
})?;
|
||||
if !sources.files().iter().any(|source| !source.is_test()) {
|
||||
return Err(format!(
|
||||
"{}: no .dever or .dever.md source files",
|
||||
source_root.display()
|
||||
));
|
||||
}
|
||||
if action.compiles_application()
|
||||
&& sources.files().iter().any(|source| {
|
||||
source
|
||||
.path()
|
||||
.file_name()
|
||||
.is_some_and(|name| name == "main.dever" || name == "main.dever.md")
|
||||
})
|
||||
{
|
||||
return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into());
|
||||
}
|
||||
if let Action::Format { check } = action {
|
||||
return format::execute(&root, &sources, check);
|
||||
}
|
||||
let settings = action
|
||||
.loads_settings()
|
||||
.then(|| dever_runtime::config::Settings::load_project(&root))
|
||||
.transpose()?;
|
||||
let checked = match settings.as_ref() {
|
||||
Some(settings) => dever_core::check_with_settings(&sources, settings),
|
||||
None => dever_core::check(&sources),
|
||||
};
|
||||
let program = checked.map_err(|errors| {
|
||||
errors
|
||||
.iter()
|
||||
.map(|error| error.render(&sources))
|
||||
.collect::<String>()
|
||||
})?;
|
||||
for warning in program.warnings() {
|
||||
if SourceMap::is_standard(warning.primary.source) {
|
||||
continue;
|
||||
}
|
||||
write!(io::stderr().lock(), "{}", warning.render(&sources))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
if let Action::Api { output } = action {
|
||||
return api::write(&program, output.as_deref());
|
||||
}
|
||||
api::check_baseline(&program, &root)?;
|
||||
if matches!(action, Action::Check) {
|
||||
return Ok(ExitCode::SUCCESS);
|
||||
}
|
||||
if matches!(action, Action::Test) {
|
||||
return test_runner::execute(&program, &sources);
|
||||
}
|
||||
let target = match &action {
|
||||
Action::Build { target, .. } => *target,
|
||||
_ => BuildTarget::host()?,
|
||||
};
|
||||
let external_report = if action.compiles_application() {
|
||||
libs::prepare_program(&root, &program, target)?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if action.compiles_application() {
|
||||
let errors = program.application_errors()?;
|
||||
if !errors.is_empty() {
|
||||
return Err(errors.iter().map(|error| error.render(&sources)).collect());
|
||||
}
|
||||
}
|
||||
match &action {
|
||||
Action::Check => unreachable!("check returned before target selection"),
|
||||
Action::Lib { .. } => unreachable!("lib returned before source checking"),
|
||||
Action::Package { .. } => unreachable!("package returned before source checking"),
|
||||
Action::Api { .. } => unreachable!("api returned before baseline checking"),
|
||||
Action::Format { .. } => unreachable!("fmt returned before checking"),
|
||||
Action::Test => unreachable!("test returned before application compilation"),
|
||||
Action::Run { .. }
|
||||
| Action::Build { .. }
|
||||
| Action::TenantMigrate { .. }
|
||||
| Action::TenantOwner { .. }
|
||||
| Action::TenantComponent { .. } => {}
|
||||
Action::Clean => unreachable!("clean returned before source loading"),
|
||||
}
|
||||
let settings = settings
|
||||
.as_ref()
|
||||
.expect("application action loaded settings before checking");
|
||||
program.validate_database_settings(settings)?;
|
||||
let resources = libs::prepare_resources(&root, &program, target)?;
|
||||
let native = compile::application(
|
||||
&program,
|
||||
&sources,
|
||||
&settings.compilation_bindings(),
|
||||
&resources,
|
||||
target,
|
||||
)?;
|
||||
match action {
|
||||
Action::Package { .. } => unreachable!("package returned before source checking"),
|
||||
Action::Build { output, .. } => {
|
||||
native.save(&output).map_err(|error| {
|
||||
format!("cannot save executable '{}': {error}", output.display())
|
||||
})?;
|
||||
if let Some(report) = external_report {
|
||||
eprintln!("{}", report.summary());
|
||||
libs::write_report(&output, &report)?;
|
||||
}
|
||||
Ok(ExitCode::SUCCESS)
|
||||
}
|
||||
Action::Run {
|
||||
tenant_id,
|
||||
arguments,
|
||||
} => {
|
||||
let arguments = match tenant_id {
|
||||
Some(tenant_id) => [
|
||||
vec![
|
||||
OsString::from("--dever-tenant-cmd"),
|
||||
OsString::from(tenant_id.to_string()),
|
||||
],
|
||||
arguments,
|
||||
]
|
||||
.concat(),
|
||||
None => arguments,
|
||||
};
|
||||
run_native(&native, &root, &arguments)
|
||||
}
|
||||
Action::TenantMigrate { tenant_id } => run_native(
|
||||
&native,
|
||||
&root,
|
||||
&[
|
||||
OsString::from("--dever-tenant-migrate"),
|
||||
OsString::from(tenant_id.to_string()),
|
||||
],
|
||||
),
|
||||
Action::TenantOwner {
|
||||
tenant_id,
|
||||
site,
|
||||
user_id,
|
||||
} => run_native(
|
||||
&native,
|
||||
&root,
|
||||
&[
|
||||
OsString::from("--dever-tenant-owner"),
|
||||
OsString::from(tenant_id.to_string()),
|
||||
OsString::from(site),
|
||||
OsString::from(user_id.to_string()),
|
||||
],
|
||||
),
|
||||
Action::TenantComponent {
|
||||
tenant_id,
|
||||
component,
|
||||
enabled,
|
||||
} => run_native(
|
||||
&native,
|
||||
&root,
|
||||
&[
|
||||
OsString::from("--dever-tenant-component"),
|
||||
OsString::from(tenant_id.to_string()),
|
||||
OsString::from(if enabled { "enable" } else { "disable" }),
|
||||
OsString::from(component),
|
||||
],
|
||||
),
|
||||
Action::Check
|
||||
| Action::Api { .. }
|
||||
| Action::Format { .. }
|
||||
| Action::Test
|
||||
| Action::Lib { .. }
|
||||
| Action::Clean => {
|
||||
unreachable!("check and fmt returned before native compilation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_native(
|
||||
native: &dever_core::native::NativeProgram,
|
||||
project_root: &PathBuf,
|
||||
arguments: &[OsString],
|
||||
) -> Result<ExitCode, String> {
|
||||
stage_runtime_config(native.executable(), project_root)?;
|
||||
let status = Command::new(native.executable())
|
||||
.current_dir(project_root)
|
||||
.args(arguments)
|
||||
.status()
|
||||
.map_err(|error| format!("cannot run native program: {error}"))?;
|
||||
Ok(if status.success() {
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
ExitCode::FAILURE
|
||||
})
|
||||
}
|
||||
|
||||
fn stage_runtime_config(
|
||||
executable: &std::path::Path,
|
||||
project_root: &std::path::Path,
|
||||
) -> Result<(), String> {
|
||||
let source = project_root.join("config");
|
||||
let destination = executable
|
||||
.parent()
|
||||
.ok_or("native executable has no parent directory")?
|
||||
.join("config");
|
||||
copy_config_directory(&source, &destination)
|
||||
}
|
||||
|
||||
fn copy_config_directory(
|
||||
source: &std::path::Path,
|
||||
destination: &std::path::Path,
|
||||
) -> Result<(), String> {
|
||||
let metadata = std::fs::symlink_metadata(source)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?;
|
||||
if metadata.file_type().is_symlink() || !metadata.is_dir() {
|
||||
return Err(format!(
|
||||
"project config '{}' must be a real directory",
|
||||
source.display()
|
||||
));
|
||||
}
|
||||
std::fs::create_dir(destination)
|
||||
.map_err(|error| format!("cannot stage config '{}': {error}", destination.display()))?;
|
||||
for entry in std::fs::read_dir(source)
|
||||
.map_err(|error| format!("cannot read '{}': {error}", source.display()))?
|
||||
{
|
||||
let entry =
|
||||
entry.map_err(|error| format!("cannot read '{}': {error}", source.display()))?;
|
||||
let source_path = entry.path();
|
||||
let destination_path = destination.join(entry.file_name());
|
||||
let metadata = std::fs::symlink_metadata(&source_path)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", source_path.display()))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(format!(
|
||||
"project config '{}' must not contain symbolic links",
|
||||
source_path.display()
|
||||
));
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
copy_config_directory(&source_path, &destination_path)?;
|
||||
} else if metadata.is_file() {
|
||||
std::fs::copy(&source_path, &destination_path).map_err(|error| {
|
||||
format!("cannot stage config '{}': {error}", source_path.display())
|
||||
})?;
|
||||
} else {
|
||||
return Err(format!(
|
||||
"project config '{}' must contain only files and directories",
|
||||
source_path.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn arguments_for(arguments: &[OsString]) -> Result<(PathBuf, Action), String> {
|
||||
match arguments {
|
||||
[command, operation, root, rest @ ..]
|
||||
if command == "lib"
|
||||
&& matches!(
|
||||
operation.to_str(),
|
||||
Some("add" | "list" | "update" | "remove" | "doctor")
|
||||
) =>
|
||||
{
|
||||
let command = operation.to_str().unwrap().to_owned();
|
||||
let (specs, target) = lib_arguments(rest)?;
|
||||
Ok((
|
||||
root.into(),
|
||||
Action::Lib {
|
||||
command,
|
||||
specs,
|
||||
target,
|
||||
},
|
||||
))
|
||||
}
|
||||
[command, operation, root, rest @ ..]
|
||||
if command == "package"
|
||||
&& matches!(
|
||||
operation.to_str(),
|
||||
Some("add" | "list" | "update" | "remove" | "doctor")
|
||||
) =>
|
||||
{
|
||||
let command = operation.to_str().unwrap().to_owned();
|
||||
let specs = rest
|
||||
.iter()
|
||||
.map(|value| {
|
||||
value
|
||||
.to_str()
|
||||
.map(str::to_owned)
|
||||
.ok_or("Package spec must be UTF-8".to_owned())
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
Ok((root.into(), Action::Package { command, specs }))
|
||||
}
|
||||
[command, root] if command == "check" => Ok((root.into(), Action::Check)),
|
||||
[command, root] if command == "api" => Ok((root.into(), Action::Api { output: None })),
|
||||
[command, root, flag, output] if command == "api" && flag == "--output" => Ok((
|
||||
root.into(),
|
||||
Action::Api {
|
||||
output: Some(output.into()),
|
||||
},
|
||||
)),
|
||||
[command, root] if command == "fmt" => Ok((root.into(), Action::Format { check: false })),
|
||||
[command, root, flag] if command == "fmt" && flag == "--check" => {
|
||||
Ok((root.into(), Action::Format { check: true }))
|
||||
}
|
||||
[command, root] if command == "run" => Ok((
|
||||
root.into(),
|
||||
Action::Run {
|
||||
tenant_id: None,
|
||||
arguments: Vec::new(),
|
||||
},
|
||||
)),
|
||||
[command, root, separator, name, input] if command == "run" && separator == "--" => Ok((
|
||||
root.into(),
|
||||
Action::Run {
|
||||
tenant_id: None,
|
||||
arguments: vec![name.clone(), input.clone()],
|
||||
},
|
||||
)),
|
||||
[command, root, tenant, tenant_id, separator, name, input]
|
||||
if command == "run" && tenant == "--tenant" && separator == "--" =>
|
||||
{
|
||||
let tenant_id = positive_tenant_id(tenant_id)?;
|
||||
Ok((
|
||||
root.into(),
|
||||
Action::Run {
|
||||
tenant_id: Some(tenant_id),
|
||||
arguments: vec![name.clone(), input.clone()],
|
||||
},
|
||||
))
|
||||
}
|
||||
[command, root] if command == "test" => Ok((root.into(), Action::Test)),
|
||||
[command, root] if command == "clean" => Ok((root.into(), Action::Clean)),
|
||||
[command, root, rest @ ..] if command == "build" => {
|
||||
let mut output = None;
|
||||
let mut target = None;
|
||||
let mut options = rest.iter();
|
||||
while let Some(option) = options.next() {
|
||||
match option.to_str() {
|
||||
Some("--output") if output.is_none() => {
|
||||
output = Some(PathBuf::from(option_value(&mut options, "--output")?));
|
||||
}
|
||||
Some("--target") if target.is_none() => {
|
||||
target = Some(parse_target(option_value(&mut options, "--target")?)?);
|
||||
}
|
||||
_ => return Err("build accepts --output and --target exactly once; unknown or duplicate option".into()),
|
||||
}
|
||||
}
|
||||
Ok((
|
||||
root.into(),
|
||||
Action::Build {
|
||||
output: output.ok_or("build requires --output <new-file>")?,
|
||||
target: target.map_or_else(BuildTarget::host, Ok)?,
|
||||
},
|
||||
))
|
||||
}
|
||||
[tenant, migrate, root, tenant_id] if tenant == "tenant" && migrate == "migrate" => {
|
||||
let tenant_id = positive_tenant_id(tenant_id)?;
|
||||
Ok((root.into(), Action::TenantMigrate { tenant_id }))
|
||||
}
|
||||
[tenant, owner, root, tenant_id, site, user_id]
|
||||
if tenant == "tenant" && owner == "owner" =>
|
||||
{
|
||||
let tenant_id = positive_id(tenant_id, "tenant id")?;
|
||||
let site = site
|
||||
.to_str()
|
||||
.ok_or_else(|| "site must be UTF-8".to_owned())?
|
||||
.to_owned();
|
||||
let user_id = positive_id(user_id, "user id")?;
|
||||
Ok((
|
||||
root.into(),
|
||||
Action::TenantOwner {
|
||||
tenant_id,
|
||||
site,
|
||||
user_id,
|
||||
},
|
||||
))
|
||||
}
|
||||
[
|
||||
tenant,
|
||||
component_command,
|
||||
operation,
|
||||
root,
|
||||
tenant_id,
|
||||
component,
|
||||
] if tenant == "tenant"
|
||||
&& component_command == "component"
|
||||
&& (operation == "enable" || operation == "disable") =>
|
||||
{
|
||||
let tenant_id = positive_tenant_id(tenant_id)?;
|
||||
let component = component
|
||||
.to_str()
|
||||
.ok_or_else(|| "component must be UTF-8".to_owned())?
|
||||
.to_owned();
|
||||
Ok((
|
||||
root.into(),
|
||||
Action::TenantComponent {
|
||||
tenant_id,
|
||||
component,
|
||||
enabled: operation == "enable",
|
||||
},
|
||||
))
|
||||
}
|
||||
_ => Err(USAGE.into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn option_value<'a>(
|
||||
options: &mut impl Iterator<Item = &'a OsString>,
|
||||
option: &str,
|
||||
) -> Result<&'a OsString, String> {
|
||||
options
|
||||
.next()
|
||||
.filter(|value| !value.is_empty() && !value.to_string_lossy().starts_with("--"))
|
||||
.ok_or_else(|| format!("{option} requires a value"))
|
||||
}
|
||||
|
||||
fn parse_target(value: &OsString) -> Result<BuildTarget, String> {
|
||||
value.to_str().ok_or("build target must be UTF-8")?.parse()
|
||||
}
|
||||
|
||||
fn lib_arguments(arguments: &[OsString]) -> Result<(Vec<String>, BuildTarget), String> {
|
||||
let mut specs = Vec::new();
|
||||
let mut target = None;
|
||||
let mut arguments = arguments.iter();
|
||||
while let Some(argument) = arguments.next() {
|
||||
if argument == "--target" && target.is_none() {
|
||||
target = Some(parse_target(option_value(&mut arguments, "--target")?)?);
|
||||
} else {
|
||||
let spec = argument.to_str().ok_or("lib spec must be UTF-8")?;
|
||||
if spec.starts_with('-') {
|
||||
return Err("unknown or duplicate lib option".into());
|
||||
}
|
||||
specs.push(spec.to_owned());
|
||||
}
|
||||
}
|
||||
Ok((specs, target.map_or_else(BuildTarget::host, Ok)?))
|
||||
}
|
||||
|
||||
fn positive_tenant_id(value: &OsString) -> Result<i64, String> {
|
||||
positive_id(value, "tenant id")
|
||||
}
|
||||
|
||||
fn positive_id(value: &OsString, label: &str) -> Result<i64, String> {
|
||||
let id = value
|
||||
.to_str()
|
||||
.ok_or_else(|| format!("{label} must be a positive integer"))?
|
||||
.parse::<i64>()
|
||||
.map_err(|_| format!("{label} must be a positive integer"))?;
|
||||
if id <= 0 {
|
||||
return Err(format!("{label} must be a positive integer"));
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Action, arguments_for};
|
||||
use std::ffi::OsString;
|
||||
|
||||
#[test]
|
||||
fn run_accepts_only_one_named_json_command_after_separator() {
|
||||
let args = [
|
||||
"run",
|
||||
"project",
|
||||
"--",
|
||||
"user.account.greeting",
|
||||
"{\"name\":\"Ada\"}",
|
||||
]
|
||||
.map(OsString::from);
|
||||
let (root, action) = arguments_for(&args).unwrap();
|
||||
assert_eq!(root.to_string_lossy(), "project");
|
||||
let Action::Run {
|
||||
tenant_id,
|
||||
arguments,
|
||||
} = action
|
||||
else {
|
||||
panic!("expected run action")
|
||||
};
|
||||
assert_eq!(tenant_id, None);
|
||||
assert_eq!(arguments, args[3..]);
|
||||
assert!(arguments_for(&args[..4]).is_err());
|
||||
assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tenant_command_uses_a_separate_positive_control_plane_id() {
|
||||
let args = [
|
||||
"run",
|
||||
"project",
|
||||
"--tenant",
|
||||
"42",
|
||||
"--",
|
||||
"content.article.rebuild",
|
||||
"{\"force\":true}",
|
||||
]
|
||||
.map(OsString::from);
|
||||
let (root, action) = arguments_for(&args).unwrap();
|
||||
assert_eq!(root.to_string_lossy(), "project");
|
||||
let Action::Run {
|
||||
tenant_id,
|
||||
arguments,
|
||||
} = action
|
||||
else {
|
||||
panic!("expected run action")
|
||||
};
|
||||
assert_eq!(tenant_id, Some(42));
|
||||
assert_eq!(arguments, args[5..]);
|
||||
|
||||
for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] {
|
||||
let args = [
|
||||
"run",
|
||||
"project",
|
||||
"--tenant",
|
||||
invalid,
|
||||
"--",
|
||||
"content.article.rebuild",
|
||||
"{}",
|
||||
]
|
||||
.map(OsString::from);
|
||||
let error = arguments_for(&args)
|
||||
.err()
|
||||
.expect("invalid tenant id must fail");
|
||||
assert_eq!(error, "tenant id must be a positive integer");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tenant_migrate_requires_a_positive_integer_id() {
|
||||
let args = ["tenant", "migrate", "project", "42"].map(OsString::from);
|
||||
let (root, action) = arguments_for(&args).unwrap();
|
||||
assert_eq!(root.to_string_lossy(), "project");
|
||||
assert!(matches!(action, Action::TenantMigrate { tenant_id: 42 }));
|
||||
|
||||
for invalid in ["0", "-1", "tenant-1", "9223372036854775808"] {
|
||||
let args = ["tenant", "migrate", "project", invalid].map(OsString::from);
|
||||
let error = arguments_for(&args)
|
||||
.err()
|
||||
.expect("invalid tenant id must fail");
|
||||
assert_eq!(error, "tenant id must be a positive integer");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tenant_component_command_selects_one_manifest_component() {
|
||||
for (operation, expected) in [("enable", true), ("disable", false)] {
|
||||
let args =
|
||||
["tenant", "component", operation, "project", "42", "news"].map(OsString::from);
|
||||
let (root, action) = arguments_for(&args).unwrap();
|
||||
assert_eq!(root.to_string_lossy(), "project");
|
||||
let Action::TenantComponent {
|
||||
tenant_id,
|
||||
component,
|
||||
enabled,
|
||||
} = action
|
||||
else {
|
||||
panic!("expected tenant component action");
|
||||
};
|
||||
assert_eq!(tenant_id, 42);
|
||||
assert_eq!(component, "news");
|
||||
assert_eq!(enabled, expected);
|
||||
}
|
||||
|
||||
let invalid =
|
||||
["tenant", "component", "toggle", "project", "42", "news"].map(OsString::from);
|
||||
assert!(arguments_for(&invalid).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_accepts_exactly_one_project_root() {
|
||||
let args = ["clean", "project"].map(OsString::from);
|
||||
let (root, action) = arguments_for(&args).unwrap();
|
||||
assert_eq!(root.to_string_lossy(), "project");
|
||||
assert!(matches!(action, Action::Clean));
|
||||
assert!(arguments_for(&[args.as_slice(), &[OsString::from("extra")]].concat()).is_err());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,128 @@
|
||||
//! Create complete starter projects without replacing caller-owned paths.
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
macro_rules! template {
|
||||
($path:literal) => {
|
||||
include_str!(concat!("../../../skills/assets/template/", $path))
|
||||
};
|
||||
}
|
||||
|
||||
const FILES: &[(&str, &str)] = &[
|
||||
("config/setting.json", template!("config/setting.json")),
|
||||
("AGENTS.md", template!("AGENTS.md")),
|
||||
("README.md", template!("README.md")),
|
||||
(".gitignore", template!(".gitignore")),
|
||||
];
|
||||
|
||||
const SOURCES: &[(&str, &str, &str)] = &[
|
||||
(
|
||||
"module/hello/greeting/app",
|
||||
template!("module/hello/greeting/app.dever"),
|
||||
template!("module/hello/greeting/app.md"),
|
||||
),
|
||||
(
|
||||
"module/hello/greeting/api",
|
||||
template!("module/hello/greeting/api.dever"),
|
||||
template!("module/hello/greeting/api.md"),
|
||||
),
|
||||
(
|
||||
"test/hello/greeting/greet",
|
||||
template!("test/hello/greeting/greet.dever"),
|
||||
template!("test/hello/greeting/greet.md"),
|
||||
),
|
||||
];
|
||||
|
||||
/// Publish a new project beside its private staging directory. Existing targets,
|
||||
/// including empty directories and dangling symlinks, are never replaced.
|
||||
pub fn create(destination: &Path, markdown: bool) -> Result<(), String> {
|
||||
let parent = destination
|
||||
.parent()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.unwrap_or_else(|| Path::new("."));
|
||||
let name = destination
|
||||
.file_name()
|
||||
.ok_or_else(|| "new project requires a new directory name".to_owned())?;
|
||||
let parent = fs::canonicalize(parent)
|
||||
.map_err(|error| format!("cannot resolve new project parent: {error}"))?;
|
||||
let destination = parent.join(name);
|
||||
match fs::symlink_metadata(&destination) {
|
||||
Ok(_) => {
|
||||
return Err(format!(
|
||||
"project target already exists: {}",
|
||||
destination.display()
|
||||
));
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(format!("cannot inspect project target: {error}")),
|
||||
}
|
||||
let staging = Staging::create(&parent)?;
|
||||
for (path, content) in FILES {
|
||||
write(&staging.0, path, content)?;
|
||||
}
|
||||
for (path, source, document) in SOURCES {
|
||||
let (suffix, content) = if markdown {
|
||||
("dever.md", document.replace("{{source}}", source))
|
||||
} else {
|
||||
("dever", (*source).to_owned())
|
||||
};
|
||||
write(&staging.0, &format!("{path}.{suffix}"), &content)?;
|
||||
}
|
||||
publish(&staging.0, &destination)
|
||||
}
|
||||
|
||||
fn write(root: &Path, path: &str, content: &str) -> Result<(), String> {
|
||||
let path = root.join(path);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("cannot create template directory: {error}"))?;
|
||||
}
|
||||
fs::write(&path, content)
|
||||
.map_err(|error| format!("cannot write template {}: {error}", path.display()))
|
||||
}
|
||||
|
||||
struct Staging(PathBuf);
|
||||
|
||||
impl Staging {
|
||||
fn create(parent: &Path) -> Result<Self, String> {
|
||||
loop {
|
||||
let sequence = NEXT_STAGING.fetch_add(1, Ordering::Relaxed);
|
||||
let path = parent.join(format!(".dever-new-{}-{sequence}", std::process::id()));
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
match builder.create(&path) {
|
||||
Ok(()) => return Ok(Self(path)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(format!("cannot stage new project: {error}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Staging {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn publish(staging: &Path, destination: &Path) -> Result<(), String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{CWD, RenameFlags, renameat_with};
|
||||
renameat_with(CWD, staging, CWD, destination, RenameFlags::NOREPLACE)
|
||||
.map_err(|error| format!("cannot publish new project: {error}"))
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = (staging, destination);
|
||||
Err("new project publication is not available for this platform".into())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
use std::fs;
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use dever_core::hir::{Program, TestCase};
|
||||
use dever_core::source::SourceMap;
|
||||
|
||||
pub(super) fn execute(program: &Program, sources: &SourceMap) -> Result<ExitCode, String> {
|
||||
let mut stdout = io::stdout().lock();
|
||||
writeln!(stdout, "running {} tests", program.tests().len()).map_err(io_error)?;
|
||||
|
||||
let mut passed = 0;
|
||||
let mut failures = Vec::new();
|
||||
if !program.tests().is_empty() {
|
||||
match crate::compile::test_suite(program, sources) {
|
||||
Ok(native) => {
|
||||
for (index, test) in program.tests().iter().enumerate() {
|
||||
match run_case(program, test, &native, index) {
|
||||
Ok(()) => {
|
||||
passed += 1;
|
||||
writeln!(stdout, "test {} ... ok", test.name()).map_err(io_error)?;
|
||||
}
|
||||
Err(failure) => {
|
||||
writeln!(stdout, "test {} ... FAILED", test.name())
|
||||
.map_err(io_error)?;
|
||||
failures.push((test.name(), failure));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(message) => {
|
||||
for test in program.tests() {
|
||||
writeln!(stdout, "test {} ... FAILED", test.name()).map_err(io_error)?;
|
||||
failures.push((test.name(), CaseFailure::message(message.clone())));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (name, failure) in &failures {
|
||||
writeln!(stdout, "\n---- {name} failure ----").map_err(io_error)?;
|
||||
writeln!(stdout, "{}", failure.message).map_err(io_error)?;
|
||||
write_output(&mut stdout, "stdout", &failure.stdout)?;
|
||||
write_output(&mut stdout, "stderr", &failure.stderr)?;
|
||||
}
|
||||
|
||||
let failed = failures.len();
|
||||
let status = if failed == 0 { "ok" } else { "FAILED" };
|
||||
writeln!(
|
||||
stdout,
|
||||
"\ntest result: {status}. {passed} passed; {failed} failed; 0 not run"
|
||||
)
|
||||
.map_err(io_error)?;
|
||||
Ok(if failed == 0 {
|
||||
ExitCode::SUCCESS
|
||||
} else {
|
||||
ExitCode::FAILURE
|
||||
})
|
||||
}
|
||||
|
||||
fn run_case(
|
||||
program: &Program,
|
||||
test: &TestCase,
|
||||
native: &dever_core::native::NativeProgram,
|
||||
index: usize,
|
||||
) -> Result<(), CaseFailure> {
|
||||
let project = TemporaryProject::new().map_err(CaseFailure::message)?;
|
||||
if test.uses_database() {
|
||||
write_settings(project.path(), test).map_err(CaseFailure::message)?;
|
||||
let settings = dever_runtime::config::Settings::load_project(project.path())
|
||||
.map_err(CaseFailure::message)?;
|
||||
let profile = program
|
||||
.validate_database_settings(&settings)
|
||||
.map_err(CaseFailure::message)?;
|
||||
if !profile.sqlite || profile.postgres {
|
||||
return Err(CaseFailure::message(
|
||||
"generated test settings must select SQLite only".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let executable = project
|
||||
.path()
|
||||
.join(format!("test-program{}", std::env::consts::EXE_SUFFIX));
|
||||
native.save(&executable).map_err(|error| {
|
||||
CaseFailure::message(format!(
|
||||
"cannot stage test executable '{}': {error}",
|
||||
executable.display()
|
||||
))
|
||||
})?;
|
||||
let output = Command::new(&executable)
|
||||
.arg(index.to_string())
|
||||
.current_dir(project.path())
|
||||
.output()
|
||||
.map_err(|error| CaseFailure::message(format!("cannot run test process: {error}")))?;
|
||||
if output.status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(CaseFailure {
|
||||
message: format!("test process exited with {}", output.status),
|
||||
stdout: output.stdout,
|
||||
stderr: output.stderr,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn write_settings(root: &Path, test: &TestCase) -> Result<(), String> {
|
||||
let config = root.join("config");
|
||||
fs::create_dir(&config)
|
||||
.map_err(|error| format!("cannot create '{}': {error}", config.display()))?;
|
||||
let mut databases = serde_json::Map::new();
|
||||
for name in test.database_connections() {
|
||||
databases.insert(
|
||||
name.clone(),
|
||||
serde_json::json!({
|
||||
"type": "sqlite",
|
||||
"path": format!("data/db/{name}.db"),
|
||||
"max_connections": 1,
|
||||
"max_page_size": 100
|
||||
}),
|
||||
);
|
||||
}
|
||||
let document = serde_json::json!({ "database": databases });
|
||||
let path = config.join("setting.json");
|
||||
let bytes = serde_json::to_vec_pretty(&document)
|
||||
.map_err(|error| format!("cannot encode test settings: {error}"))?;
|
||||
fs::write(&path, bytes).map_err(|error| format!("cannot write '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn write_output(output: &mut impl Write, label: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
if bytes.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
writeln!(output, "---- {label} ----").map_err(io_error)?;
|
||||
output.write_all(bytes).map_err(io_error)?;
|
||||
if !bytes.ends_with(b"\n") {
|
||||
writeln!(output).map_err(io_error)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn io_error(error: io::Error) -> String {
|
||||
error.to_string()
|
||||
}
|
||||
|
||||
struct CaseFailure {
|
||||
message: String,
|
||||
stdout: Vec<u8>,
|
||||
stderr: Vec<u8>,
|
||||
}
|
||||
|
||||
impl CaseFailure {
|
||||
fn message(message: String) -> Self {
|
||||
Self {
|
||||
message,
|
||||
stdout: Vec::new(),
|
||||
stderr: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TemporaryProject(PathBuf);
|
||||
|
||||
impl TemporaryProject {
|
||||
fn new() -> Result<Self, String> {
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
loop {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"dever-application-test-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
match builder.create(&path) {
|
||||
Ok(()) => return Ok(Self(path)),
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => {
|
||||
return Err(format!(
|
||||
"cannot create temporary test project '{}': {error}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TemporaryProject {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
//! Explicit Linux first installation. The caller must already trust this installer.
|
||||
//! Release keys come from an independent administrator input, never the release.
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::release::{self, InstallLock};
|
||||
use super::{Artifact, Layout, MachineManager, ReleaseManifest, Version};
|
||||
|
||||
mod sandbox;
|
||||
mod service;
|
||||
mod transaction;
|
||||
pub use service::{RecoveryAction, recovery_actions};
|
||||
|
||||
pub(crate) const SERVICE: &str = "[Unit]\nDescription=Dever shared compiler service\nAfter=local-fs.target\n\n[Service]\nType=exec\nUser=root\nGroup=root\nExecStart=/opt/dever/bin/deverd --root /opt/dever\nRuntimeDirectory=dever\nRuntimeDirectoryMode=0755\nUMask=0077\nRestart=on-failure\nRestartSec=1\nKillMode=control-group\nTimeoutStopSec=30\nNoNewPrivileges=yes\nPrivateTmp=yes\nProtectHome=yes\nProtectSystem=strict\nReadWritePaths=/opt/dever/state /opt/dever/cache /run/dever\n\n[Install]\nWantedBy=multi-user.target\n";
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Settings {
|
||||
release: PathBuf,
|
||||
system_root: PathBuf,
|
||||
trusted_key: PathBuf,
|
||||
#[serde(default)]
|
||||
activate_service: bool,
|
||||
}
|
||||
|
||||
/// Private distribution entry; ordinary version install/update remains unchanged.
|
||||
pub fn install(installer_root: &Path) -> Result<Version, String> {
|
||||
require_root()?;
|
||||
absolute_root(installer_root)?;
|
||||
trusted_directory(installer_root)?;
|
||||
let config = installer_root.join("config/setting.json");
|
||||
trusted_file(&config)?;
|
||||
let settings: Settings =
|
||||
serde_json::from_slice(&read_bounded(&config, 1024 * 1024)?).map_err(|error| {
|
||||
format!(
|
||||
"invalid installer config at {}:{}",
|
||||
error.line(),
|
||||
error.column()
|
||||
)
|
||||
})?;
|
||||
absolute_root(&settings.system_root)?;
|
||||
trusted_directory(&settings.system_root)?;
|
||||
if settings.system_root == Path::new("/") {
|
||||
// Publishing an AppArmor file can trigger a later system-wide reload,
|
||||
// even when this installer does not load profiles or activate services.
|
||||
dever_sandbox::validate_host_policy()?;
|
||||
}
|
||||
if settings.activate_service && settings.system_root != Path::new("/") {
|
||||
return Err(
|
||||
"service activation requires the actual system root; image installs are offline".into(),
|
||||
);
|
||||
}
|
||||
let package = absolute_input(installer_root, &settings.release)?;
|
||||
let key_path = absolute_input(installer_root, &settings.trusted_key)?;
|
||||
trusted_file(&key_path)?;
|
||||
let package = fs::canonicalize(package).map_err(|error| error.to_string())?;
|
||||
if key_path.starts_with(&package) {
|
||||
return Err("trusted key must be supplied independently outside the release".into());
|
||||
}
|
||||
let key = release::decode_hex_file(&key_path, 32)?;
|
||||
create_directory(&settings.system_root, "opt")?;
|
||||
let layout = Layout::new(settings.system_root.join("opt/dever"));
|
||||
if layout.root().exists() {
|
||||
trusted_directory(layout.root())?;
|
||||
}
|
||||
for directory in [
|
||||
layout.bin(),
|
||||
layout.state(),
|
||||
layout.versions(),
|
||||
layout.downloads(),
|
||||
layout.staging(),
|
||||
layout.native_cache(),
|
||||
] {
|
||||
let relative = directory
|
||||
.strip_prefix(&settings.system_root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.to_str()
|
||||
.ok_or("installer directory is not UTF-8")?;
|
||||
create_directory(&settings.system_root, relative)?;
|
||||
}
|
||||
layout.validate_machine_permissions()?;
|
||||
let _lock = InstallLock::acquire(&layout)?;
|
||||
transaction::recover(&settings.system_root, &layout)?;
|
||||
let pinned = layout.state().join("trusted-release-key");
|
||||
if pinned.exists() {
|
||||
trusted_file(&pinned)?;
|
||||
if release::decode_hex_file(&pinned, 32)? != key {
|
||||
return Err("installer key differs from the machine's pinned release key".into());
|
||||
}
|
||||
}
|
||||
// The private copy is checked again by the version owner before any health
|
||||
// command executes. A writable download cannot race signed bytes into it.
|
||||
let manifest = MachineManager::verify_package(&package, None, &key)?;
|
||||
validate_payload(&package, &manifest.artifacts, true)?;
|
||||
pin_key(&layout, &key)?;
|
||||
let manager = MachineManager::new(layout.clone());
|
||||
if manager
|
||||
.active_version()?
|
||||
.is_some_and(|active| manifest.version < active)
|
||||
{
|
||||
return Err("bootstrap installation cannot downgrade the active compiler".into());
|
||||
}
|
||||
let installed = layout.versions().join(manifest.version.as_str());
|
||||
if installed.exists()
|
||||
&& super::sha256_file(&installed.join("manifest.json"))?
|
||||
!= super::sha256_file(&package.join("manifest.json"))?
|
||||
{
|
||||
return Err("installed version already has a different signed release identity".into());
|
||||
}
|
||||
manager.install_package_locked(&package, &manifest.version)?;
|
||||
transaction::install(
|
||||
&settings.system_root,
|
||||
&layout,
|
||||
&installed,
|
||||
&manifest,
|
||||
&key,
|
||||
settings.activate_service,
|
||||
)?;
|
||||
Ok(manifest.version)
|
||||
}
|
||||
|
||||
/// Recover only this installer's fixed, recorded targets. Normal version
|
||||
/// operations must finish a pending bootstrap transaction first.
|
||||
pub fn recover(system_root: &Path) -> Result<(), String> {
|
||||
require_root()?;
|
||||
absolute_root(system_root)?;
|
||||
trusted_directory(system_root)?;
|
||||
let layout = Layout::new(system_root.join("opt/dever"));
|
||||
layout.validate_machine_permissions()?;
|
||||
let _lock = InstallLock::acquire(&layout)?;
|
||||
transaction::recover(system_root, &layout)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_payload(
|
||||
root: &Path,
|
||||
artifacts: &[Artifact],
|
||||
current_template: bool,
|
||||
) -> Result<(), String> {
|
||||
let scoped = artifacts
|
||||
.iter()
|
||||
.filter_map(|artifact| {
|
||||
artifact
|
||||
.path
|
||||
.strip_prefix("bootstrap/")
|
||||
.map(|path| Artifact {
|
||||
path: path.into(),
|
||||
bytes: artifact.bytes,
|
||||
sha256: artifact.sha256.clone(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
validate_files(&root.join("bootstrap"), &scoped, current_template)
|
||||
}
|
||||
|
||||
fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Result<(), String> {
|
||||
let bytes = artifacts
|
||||
.iter()
|
||||
.try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes))
|
||||
.ok_or("bootstrap size overflow")?;
|
||||
if artifacts.len() > 64 || bytes > 64 * 1024 * 1024 {
|
||||
return Err("bootstrap closure exceeds its 64-file/64-MiB budget".into());
|
||||
}
|
||||
for artifact in artifacts {
|
||||
if !matches!(
|
||||
artifact.path.as_str(),
|
||||
"dever" | "deverd" | "deverd.service" | "dever-sandbox"
|
||||
) && !artifact.path.starts_with("lib/")
|
||||
{
|
||||
return Err("bootstrap has an unsupported artifact".into());
|
||||
}
|
||||
let path = super::runtime_pack::checked_path(root, &artifact.path, true)?;
|
||||
let bytes = read_bounded(&path, artifact.bytes)?;
|
||||
if bytes.len() as u64 != artifact.bytes || digest(&bytes) != artifact.sha256 {
|
||||
return Err(format!(
|
||||
"bootstrap artifact '{}' differs from its signature",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
}
|
||||
if !artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == "deverd.service")
|
||||
|| new_payload && read_bounded(&root.join("deverd.service"), 8192)? != SERVICE.as_bytes()
|
||||
{
|
||||
return Err(
|
||||
"bootstrap service definition is absent or differs from the supported contract".into(),
|
||||
);
|
||||
}
|
||||
if new_payload
|
||||
&& (!artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == "dever-sandbox")
|
||||
|| read_bounded(&root.join("dever-sandbox"), 8192)?
|
||||
!= dever_sandbox::APPARMOR_PROFILE.as_bytes())
|
||||
{
|
||||
return Err(
|
||||
"bootstrap AppArmor profile is absent or differs from the supported contract".into(),
|
||||
);
|
||||
}
|
||||
super::core_libraries::validate_entries(root, artifacts, &["dever", "deverd"])
|
||||
}
|
||||
|
||||
pub(super) fn health_check(path: &Path, flag: &str) -> Result<(), String> {
|
||||
if !service::run(
|
||||
std::process::Command::new(path).arg(flag),
|
||||
"health-check installed executable",
|
||||
)? {
|
||||
return Err(format!(
|
||||
"installed executable '{}' failed its health check",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn require_no_pending(layout: &Layout) -> Result<(), String> {
|
||||
if fs::symlink_metadata(layout.state().join("bootstrap-install.json")).is_ok() {
|
||||
return Err(
|
||||
"bootstrap installation is incomplete; rerun the explicit installer to recover it"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_bin(root: &Path, key: &[u8]) -> Result<Vec<u8>, String> {
|
||||
trusted_directory(root)?;
|
||||
let manifest = MachineManager::signed_manifest(root, key)?;
|
||||
let artifacts = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.filter_map(|artifact| {
|
||||
artifact
|
||||
.path
|
||||
.strip_prefix("bootstrap/")
|
||||
.map(|path| Artifact {
|
||||
path: path.into(),
|
||||
bytes: artifact.bytes,
|
||||
sha256: artifact.sha256.clone(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
validate_files(root, &artifacts, false)?;
|
||||
let expected = artifacts
|
||||
.iter()
|
||||
.map(|artifact| artifact.path.as_str())
|
||||
.chain(["manifest.json", "manifest.sig"])
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
let found = transaction::files(root)?;
|
||||
if found
|
||||
.iter()
|
||||
.map(String::as_str)
|
||||
.collect::<std::collections::BTreeSet<_>>()
|
||||
!= expected
|
||||
{
|
||||
return Err("installed bootstrap contains unsigned files".into());
|
||||
}
|
||||
read_bounded(&root.join("deverd.service"), 8192)
|
||||
}
|
||||
|
||||
fn pin_key(layout: &Layout, key: &[u8]) -> Result<(), String> {
|
||||
let path = layout.state().join("trusted-release-key");
|
||||
let next = layout.state().join(".trusted-release-key.next");
|
||||
if fs::symlink_metadata(&next).is_ok() {
|
||||
trusted_file(&next)?;
|
||||
if fs::metadata(&next)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len()
|
||||
> 65
|
||||
{
|
||||
return Err("temporary release key exceeds its fixed size".into());
|
||||
}
|
||||
fs::remove_file(&next).map_err(|error| error.to_string())?;
|
||||
release::sync_directory(&layout.state()).map_err(|error| error.to_string())?;
|
||||
}
|
||||
if fs::symlink_metadata(&path).is_ok() {
|
||||
trusted_file(&path)?;
|
||||
if release::decode_hex_file(&path, 32)? != key {
|
||||
return Err("installer key differs from the machine's pinned release key".into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
write_new(&next, format!("{}\n", hex(key)).as_bytes(), 0o644)?;
|
||||
rustix::fs::renameat_with(
|
||||
rustix::fs::CWD,
|
||||
&next,
|
||||
rustix::fs::CWD,
|
||||
&path,
|
||||
rustix::fs::RenameFlags::NOREPLACE,
|
||||
)
|
||||
.map_err(|error| format!("cannot publish initial release key: {error}"))?;
|
||||
release::sync_directory(&layout.state()).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn absolute_root(path: &Path) -> Result<(), String> {
|
||||
if !path.is_absolute()
|
||||
|| path
|
||||
.components()
|
||||
.any(|part| matches!(part, Component::ParentDir))
|
||||
{
|
||||
return Err("installer roots must be absolute paths without '..'".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_root() -> Result<(), String> {
|
||||
if rustix::process::geteuid().as_raw() != 0 {
|
||||
return Err("machine bootstrap installation requires root".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn absolute_input(root: &Path, path: &Path) -> Result<PathBuf, String> {
|
||||
let path = if path.is_absolute() {
|
||||
path.to_path_buf()
|
||||
} else {
|
||||
root.join(path)
|
||||
};
|
||||
if path
|
||||
.components()
|
||||
.any(|part| matches!(part, Component::ParentDir))
|
||||
{
|
||||
return Err("installer paths cannot contain '..'".into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn trusted_directory(path: &Path) -> Result<(), String> {
|
||||
trusted_path(path, true)
|
||||
}
|
||||
|
||||
fn trusted_file(path: &Path) -> Result<(), String> {
|
||||
trusted_path(path, false)
|
||||
}
|
||||
|
||||
fn trusted_path(path: &Path, directory: bool) -> Result<(), String> {
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| {
|
||||
format!(
|
||||
"cannot inspect installer path '{}': {error}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
if metadata.file_type().is_symlink()
|
||||
|| metadata.uid() != 0
|
||||
|| metadata.mode() & 0o022 != 0
|
||||
|| directory && !metadata.is_dir()
|
||||
|| !directory && (!metadata.is_file() || metadata.nlink() != 1)
|
||||
{
|
||||
return Err(format!(
|
||||
"installer path '{}' must be a real root-owned {} without shared write permission",
|
||||
path.display(),
|
||||
if directory { "directory" } else { "file" }
|
||||
));
|
||||
}
|
||||
for ancestor in path.ancestors().skip(1) {
|
||||
if ancestor.as_os_str().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let parent = fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?;
|
||||
// A root-owned sticky temporary ancestor is safe for an owned fixture;
|
||||
// the actual installation root itself may never be shared-writable.
|
||||
if !parent.is_dir()
|
||||
|| parent.file_type().is_symlink()
|
||||
|| parent.uid() != 0
|
||||
|| parent.mode() & 0o022 != 0 && parent.mode() & 0o1000 == 0
|
||||
{
|
||||
return Err("installer path crosses an untrusted ancestor".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_directory(root: &Path, relative: &str) -> Result<PathBuf, String> {
|
||||
let mut path = root.to_owned();
|
||||
for part in Path::new(relative).components() {
|
||||
let Component::Normal(part) = part else {
|
||||
return Err("invalid installer directory".into());
|
||||
};
|
||||
path.push(part);
|
||||
match fs::create_dir(&path) {
|
||||
Ok(()) => {
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
release::sync_directory(&path).map_err(|error| error.to_string())?;
|
||||
release::sync_directory(path.parent().ok_or("installer directory has no parent")?)
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
|
||||
Err(error) => return Err(error.to_string()),
|
||||
}
|
||||
trusted_directory(&path)?;
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn read_bounded(path: &Path, maximum: u64) -> Result<Vec<u8>, String> {
|
||||
let mut bytes = Vec::new();
|
||||
File::open(path)
|
||||
.and_then(|file| file.take(maximum.saturating_add(1)).read_to_end(&mut bytes))
|
||||
.map_err(|error| error.to_string())?;
|
||||
if bytes.len() as u64 > maximum {
|
||||
return Err("installer input exceeds its size bound".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn write_new(path: &Path, bytes: &[u8], mode: u32) -> Result<(), String> {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)
|
||||
.map_err(|error| error.to_string())?;
|
||||
file.write_all(bytes)
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| error.to_string())?;
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(mode))
|
||||
.map_err(|error| error.to_string())?;
|
||||
file.sync_all().map_err(|error| error.to_string())?;
|
||||
release::sync_directory(path.parent().ok_or("installer file has no parent")?)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn digest(bytes: &[u8]) -> String {
|
||||
format!("{:x}", Sha256::digest(bytes))
|
||||
}
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! Preserve signed content-addressed helpers across compiler upgrades/removal.
|
||||
use std::fs;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::Path;
|
||||
|
||||
use super::{
|
||||
MachineManager, ReleaseManifest, digest, read_bounded, release, trusted_directory, trusted_file,
|
||||
};
|
||||
|
||||
const MAX_HELPERS: usize = 24;
|
||||
const MAX_HELPER_BYTES: u64 = 32 * 1024 * 1024;
|
||||
|
||||
fn artifact_path() -> String {
|
||||
format!(
|
||||
"sandbox/{}/bin/bwrap",
|
||||
crate::toolchain::platform_identity()
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn stage(
|
||||
installed: &Path,
|
||||
next: &Path,
|
||||
source: &Path,
|
||||
manifest: &ReleaseManifest,
|
||||
key: &[u8],
|
||||
) -> Result<(), String> {
|
||||
fs::create_dir(next).map_err(|error| error.to_string())?;
|
||||
fs::set_permissions(next, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut hashes = std::collections::BTreeSet::new();
|
||||
if installed.exists() {
|
||||
trusted_directory(installed)?;
|
||||
for entry in fs::read_dir(installed).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let hash = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "sandbox helper identity is not UTF-8")?;
|
||||
if hashes.len() >= MAX_HELPERS || !valid_hash(&hash) {
|
||||
return Err("installed sandbox helper store has unsupported entries or exceeds its version bound".into());
|
||||
}
|
||||
validate_installed(&entry.path(), &hash, key)?;
|
||||
let destination = next.join(&hash);
|
||||
copy_helper(&entry.path().join("bwrap"), &entry.path(), &destination)?;
|
||||
hashes.insert(hash);
|
||||
}
|
||||
}
|
||||
if let Some(artifact) = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.path == artifact_path())
|
||||
&& !hashes.contains(&artifact.sha256)
|
||||
{
|
||||
if hashes.len() >= MAX_HELPERS {
|
||||
return Err("sandbox helper version store is full; retained applications must be retired before removing old helpers".into());
|
||||
}
|
||||
let bytes = read_bounded(&source.join(&artifact.path), MAX_HELPER_BYTES)?;
|
||||
if bytes.len() as u64 != artifact.bytes
|
||||
|| dever_sandbox::validate_bwrap(&bytes)? != artifact.sha256
|
||||
{
|
||||
return Err("sandbox helper differs from its signed release".into());
|
||||
}
|
||||
let destination = next.join(&artifact.sha256);
|
||||
copy_helper(&source.join(&artifact.path), source, &destination)?;
|
||||
validate_installed(&destination, &artifact.sha256, key)?;
|
||||
}
|
||||
release::sync_tree(next)
|
||||
}
|
||||
|
||||
fn copy_helper(helper: &Path, signed_release: &Path, destination: &Path) -> Result<(), String> {
|
||||
fs::create_dir(destination).map_err(|error| error.to_string())?;
|
||||
fs::set_permissions(destination, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
release::copy_file(helper, &destination.join("bwrap"), true)?;
|
||||
for name in ["manifest.json", "manifest.sig"] {
|
||||
release::copy_file(&signed_release.join(name), &destination.join(name), false)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn valid_hash(hash: &str) -> bool {
|
||||
hash.len() == 64
|
||||
&& hash
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
}
|
||||
|
||||
fn validate_installed(root: &Path, hash: &str, key: &[u8]) -> Result<(), String> {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
trusted_directory(root)?;
|
||||
let names = fs::read_dir(root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.map(|entry| entry.map(|entry| entry.file_name()))
|
||||
.collect::<Result<std::collections::BTreeSet<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let expected = ["bwrap", "manifest.json", "manifest.sig"]
|
||||
.map(std::ffi::OsString::from)
|
||||
.into_iter()
|
||||
.collect();
|
||||
if names != expected {
|
||||
return Err("installed sandbox helper contains unsigned entries".into());
|
||||
}
|
||||
for name in ["bwrap", "manifest.json", "manifest.sig"] {
|
||||
trusted_file(&root.join(name))?;
|
||||
}
|
||||
let helper = root.join("bwrap");
|
||||
if fs::metadata(&helper)
|
||||
.map_err(|error| error.to_string())?
|
||||
.mode()
|
||||
& 0o6111
|
||||
!= 0o111
|
||||
{
|
||||
return Err("installed sandbox helper has unsupported executable permissions".into());
|
||||
}
|
||||
match rustix::fs::lgetxattr(&helper, "security.capability", &mut [0u8; 64]) {
|
||||
Err(rustix::io::Errno::NODATA | rustix::io::Errno::OPNOTSUPP) => {}
|
||||
Ok(_) => return Err("installed sandbox helper must not have file capabilities".into()),
|
||||
Err(error) => {
|
||||
return Err(format!(
|
||||
"cannot inspect installed sandbox helper capabilities: {error}"
|
||||
));
|
||||
}
|
||||
}
|
||||
let manifest = MachineManager::signed_manifest(root, key)?;
|
||||
let artifact = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.path == artifact_path())
|
||||
.ok_or("installed sandbox helper has no signed artifact")?;
|
||||
let bytes = read_bounded(&helper, MAX_HELPER_BYTES)?;
|
||||
if artifact.sha256 != hash || artifact.bytes != bytes.len() as u64 || digest(&bytes) != hash {
|
||||
return Err(
|
||||
"installed sandbox helper differs from its signature or directory identity".into(),
|
||||
);
|
||||
}
|
||||
dever_sandbox::validate_bwrap(&bytes)?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
//! Bounded Linux platform actions. Image installation never calls systemctl.
|
||||
use std::io::Read;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
struct OwnedChild(Child);
|
||||
impl Drop for OwnedChild {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn run(command: &mut Command, operation: &str) -> Result<bool, String> {
|
||||
let mut child = OwnedChild(
|
||||
command
|
||||
.env_clear()
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.map_err(|error| format!("cannot {operation}: {error}"))?,
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
if let Some(status) = child
|
||||
.0
|
||||
.try_wait()
|
||||
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
|
||||
{
|
||||
return Ok(status.success());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("{operation} exceeded 30 seconds"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn active() -> Result<bool, String> {
|
||||
if !loaded()? {
|
||||
return Ok(false);
|
||||
}
|
||||
match property("ActiveState")?.as_str() {
|
||||
"inactive" | "failed" => Ok(false),
|
||||
"active" | "activating" | "deactivating" | "reloading" => Ok(true),
|
||||
_ => Err("Dever service has an unsupported active state".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn loaded() -> Result<bool, String> {
|
||||
match property("LoadState")?.as_str() {
|
||||
"loaded" => Ok(true),
|
||||
"not-found" => Ok(false),
|
||||
_ => Err("Dever service has an unsupported load state".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn property(name: &str) -> Result<String, String> {
|
||||
let mut command = Command::new("/usr/bin/systemctl");
|
||||
command
|
||||
.env_clear()
|
||||
.args(["show", "--value", "--property", name, "deverd.service"])
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null());
|
||||
let mut child = OwnedChild(
|
||||
command
|
||||
.spawn()
|
||||
.map_err(|error| format!("cannot inspect Dever service: {error}"))?,
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let status = loop {
|
||||
if let Some(status) = child.0.try_wait().map_err(|error| error.to_string())? {
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err("service state inspection exceeded 30 seconds".into());
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
child
|
||||
.0
|
||||
.stdout
|
||||
.take()
|
||||
.ok_or("service state output missing")?
|
||||
.take(4097)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
if bytes.len() > 4096 {
|
||||
return Err("service state output exceeded its bound".into());
|
||||
}
|
||||
let value = String::from_utf8(bytes)
|
||||
.map_err(|_| "service state output is not UTF-8")?
|
||||
.trim()
|
||||
.to_owned();
|
||||
if !status.success()
|
||||
&& !(status.code() == Some(1) && name == "LoadState" && value == "not-found")
|
||||
{
|
||||
return Err(format!("cannot inspect Dever service {name}: {status}"));
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum RecoveryAction {
|
||||
Stop,
|
||||
Reload,
|
||||
Restart,
|
||||
}
|
||||
|
||||
/// Files are restored between these two action lists. A service that never
|
||||
/// existed is not stopped after its new unit has already been removed.
|
||||
pub fn recovery_actions(
|
||||
start_requested: bool,
|
||||
was_active: bool,
|
||||
currently_loaded: bool,
|
||||
) -> (Vec<RecoveryAction>, Vec<RecoveryAction>) {
|
||||
if !start_requested {
|
||||
return (vec![], vec![]);
|
||||
}
|
||||
let before = if currently_loaded {
|
||||
vec![RecoveryAction::Stop]
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let mut after = vec![RecoveryAction::Reload];
|
||||
if was_active {
|
||||
after.push(RecoveryAction::Restart);
|
||||
}
|
||||
(before, after)
|
||||
}
|
||||
|
||||
pub(super) fn execute(actions: &[RecoveryAction]) -> Result<(), String> {
|
||||
for action in actions {
|
||||
let arguments = match action {
|
||||
RecoveryAction::Reload => vec!["daemon-reload"],
|
||||
RecoveryAction::Stop => vec!["stop", "deverd.service"],
|
||||
RecoveryAction::Restart => vec!["restart", "deverd.service"],
|
||||
};
|
||||
if !run(
|
||||
Command::new("/usr/bin/systemctl").args(&arguments),
|
||||
"restore Dever service",
|
||||
)? {
|
||||
return Err(format!("systemctl {} failed", arguments.join(" ")));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn apply(restart: bool) -> Result<(), String> {
|
||||
execute(&[
|
||||
RecoveryAction::Reload,
|
||||
if restart {
|
||||
RecoveryAction::Restart
|
||||
} else {
|
||||
RecoveryAction::Stop
|
||||
},
|
||||
])
|
||||
}
|
||||
|
||||
pub(super) fn ready(layout: &crate::toolchain::Layout) -> Result<(), String> {
|
||||
let deadline = Instant::now() + Duration::from_secs(15);
|
||||
loop {
|
||||
match crate::toolchain::cache_status(layout) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) if Instant::now() >= deadline => {
|
||||
return Err(format!("Dever service did not become ready: {error}"));
|
||||
}
|
||||
Err(_) => std::thread::sleep(Duration::from_millis(50)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,522 @@
|
||||
//! One bounded journal for the fixed bootstrap destinations. It contains
|
||||
//! identities, never caller-selected recovery paths or arbitrary commands.
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::os::unix::fs::{MetadataExt, symlink};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
Layout, ReleaseManifest, create_directory, digest, read_bounded, release, service,
|
||||
trusted_directory, trusted_file, validate_bin, write_new,
|
||||
};
|
||||
|
||||
const JOURNAL: &str = "bootstrap-install.json";
|
||||
const JOURNAL_NEXT: &str = ".bootstrap-install.next";
|
||||
const MAX_TREE_BYTES: u64 = 80 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum Target {
|
||||
Bin,
|
||||
Sandbox,
|
||||
Apparmor,
|
||||
Service,
|
||||
Entry,
|
||||
Enable,
|
||||
Active,
|
||||
}
|
||||
|
||||
const TARGETS: [Target; 7] = [
|
||||
Target::Sandbox,
|
||||
Target::Apparmor,
|
||||
Target::Bin,
|
||||
Target::Service,
|
||||
Target::Entry,
|
||||
Target::Enable,
|
||||
Target::Active,
|
||||
];
|
||||
|
||||
impl Target {
|
||||
fn relative(self) -> &'static str {
|
||||
match self {
|
||||
Self::Bin => "opt/dever/bin",
|
||||
Self::Sandbox => "opt/dever/sandbox",
|
||||
Self::Apparmor => "etc/apparmor.d/dever-sandbox",
|
||||
Self::Service => "etc/systemd/system/deverd.service",
|
||||
Self::Entry => "usr/local/bin/dever",
|
||||
Self::Enable => "etc/systemd/system/multi-user.target.wants/deverd.service",
|
||||
Self::Active => "opt/dever/state/active-version",
|
||||
}
|
||||
}
|
||||
fn path(self, root: &Path) -> PathBuf {
|
||||
root.join(self.relative())
|
||||
}
|
||||
fn next(self, root: &Path) -> PathBuf {
|
||||
let path = self.path(root);
|
||||
path.with_file_name(format!(
|
||||
".dever-bootstrap-{}.next",
|
||||
path.file_name().unwrap().to_string_lossy()
|
||||
))
|
||||
}
|
||||
fn link(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Entry => Some("/opt/dever/bin/dever"),
|
||||
Self::Enable => Some("../deverd.service"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum Phase {
|
||||
Preparing,
|
||||
Prepared,
|
||||
Committed,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Change {
|
||||
target: Target,
|
||||
before: Option<String>,
|
||||
after: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Journal {
|
||||
format: String,
|
||||
phase: Phase,
|
||||
service_was_active: Option<bool>,
|
||||
service_start_requested: bool,
|
||||
changes: Vec<Change>,
|
||||
}
|
||||
|
||||
pub(super) fn install(
|
||||
root: &Path,
|
||||
layout: &Layout,
|
||||
source: &Path,
|
||||
manifest: &ReleaseManifest,
|
||||
key: &[u8],
|
||||
activate_service: bool,
|
||||
) -> Result<(), String> {
|
||||
let previous_service = if fs::read_dir(layout.bin())
|
||||
.map_err(|error| error.to_string())?
|
||||
.next()
|
||||
.is_some()
|
||||
{
|
||||
Some(validate_bin(&layout.bin(), key)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
for target in TARGETS {
|
||||
let parent = Path::new(target.relative())
|
||||
.parent()
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
create_directory(root, parent)?;
|
||||
if fs::symlink_metadata(target.next(root)).is_ok() {
|
||||
return Err("unrecorded bootstrap staging destination already exists".into());
|
||||
}
|
||||
if let Some(link) = target.link() {
|
||||
if let Ok(metadata) = fs::symlink_metadata(target.path(root))
|
||||
&& (previous_service.is_none()
|
||||
|| metadata.uid() != 0
|
||||
|| !metadata.file_type().is_symlink()
|
||||
|| fs::read_link(target.path(root)).map_err(|error| error.to_string())?
|
||||
!= Path::new(link))
|
||||
{
|
||||
return Err(
|
||||
"bootstrap cannot replace an unrelated public entry or service link".into(),
|
||||
);
|
||||
}
|
||||
} else if target == Target::Apparmor && target.path(root).exists() {
|
||||
trusted_file(&target.path(root))?;
|
||||
let installed_profile = layout.bin().join("dever-sandbox");
|
||||
if previous_service.is_none()
|
||||
|| !installed_profile.exists()
|
||||
|| read_bounded(&target.path(root), 8192)?
|
||||
!= read_bounded(&installed_profile, 8192)?
|
||||
{
|
||||
return Err("bootstrap cannot replace an unrelated AppArmor profile".into());
|
||||
}
|
||||
} else if target == Target::Service && target.path(root).exists() {
|
||||
trusted_file(&target.path(root))?;
|
||||
if previous_service.as_deref()
|
||||
!= Some(read_bounded(&target.path(root), 8192)?.as_slice())
|
||||
{
|
||||
return Err(
|
||||
"bootstrap cannot replace an unrelated system service definition".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let changes = TARGETS
|
||||
.into_iter()
|
||||
.map(|target| {
|
||||
Ok(Change {
|
||||
target,
|
||||
before: identity(&target.path(root), target.link())?,
|
||||
after: None,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
let mut journal = Journal {
|
||||
format: "dever-bootstrap-transaction-v1".into(),
|
||||
phase: Phase::Preparing,
|
||||
service_was_active: if activate_service {
|
||||
Some(service::active()?)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
service_start_requested: false,
|
||||
changes,
|
||||
};
|
||||
save(layout, &journal)?;
|
||||
let result = (|| {
|
||||
stage(root, source, manifest, key)?;
|
||||
for change in &mut journal.changes {
|
||||
change.after = identity(&change.target.next(root), change.target.link())?;
|
||||
if change.after.is_none() {
|
||||
return Err("bootstrap staging is incomplete".into());
|
||||
}
|
||||
}
|
||||
journal.phase = Phase::Prepared;
|
||||
save(layout, &journal)?;
|
||||
for index in 0..journal.changes.len() {
|
||||
if journal.changes[index].target == Target::Active && activate_service {
|
||||
journal.service_start_requested = true;
|
||||
save(layout, &journal)?;
|
||||
service::apply(true)?;
|
||||
service::ready(layout)?;
|
||||
}
|
||||
publish(root, &journal.changes[index])?;
|
||||
}
|
||||
journal.phase = Phase::Committed;
|
||||
save(layout, &journal)
|
||||
})();
|
||||
if let Err(primary) = result {
|
||||
// The journal is still Prepared unless its commit was persisted. Read
|
||||
// the durable phase rather than trusting the in-memory assignment.
|
||||
return match recover(root, layout) {
|
||||
Ok(()) => Err(primary),
|
||||
Err(rollback) => Err(format!("{primary}; bootstrap recovery failed: {rollback}")),
|
||||
};
|
||||
}
|
||||
if let Err(error) = recover(root, layout) {
|
||||
// Publication is already committed. Never roll it back because an old
|
||||
// staging copy could not be removed; preserve the journal for retry.
|
||||
return Err(format!(
|
||||
"bootstrap publication committed, but cleanup requires recovery: {error}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stage(root: &Path, source: &Path, manifest: &ReleaseManifest, key: &[u8]) -> Result<(), String> {
|
||||
super::sandbox::stage(
|
||||
&Target::Sandbox.path(root),
|
||||
&Target::Sandbox.next(root),
|
||||
source,
|
||||
manifest,
|
||||
key,
|
||||
)?;
|
||||
write_new(
|
||||
&Target::Apparmor.next(root),
|
||||
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
|
||||
0o644,
|
||||
)?;
|
||||
let bin = Target::Bin.next(root);
|
||||
fs::create_dir(&bin).map_err(|error| error.to_string())?;
|
||||
for artifact in &manifest.artifacts {
|
||||
let Some(relative) = artifact.path.strip_prefix("bootstrap/") else {
|
||||
continue;
|
||||
};
|
||||
release::copy_file(
|
||||
&source.join(&artifact.path),
|
||||
&bin.join(relative),
|
||||
matches!(relative, "dever" | "deverd"),
|
||||
)?;
|
||||
}
|
||||
for file in ["manifest.json", "manifest.sig"] {
|
||||
release::copy_file(&source.join(file), &bin.join(file), false)?;
|
||||
}
|
||||
validate_bin(&bin, key)?;
|
||||
for name in ["dever", "deverd"] {
|
||||
if !service::run(
|
||||
Command::new(bin.join(name)).arg("--dever-bootstrap-health"),
|
||||
"health-check bootstrap",
|
||||
)? {
|
||||
return Err(format!("bootstrap {name} failed its health check"));
|
||||
}
|
||||
}
|
||||
write_new(
|
||||
&Target::Service.next(root),
|
||||
super::SERVICE.as_bytes(),
|
||||
0o644,
|
||||
)?;
|
||||
for target in [Target::Entry, Target::Enable] {
|
||||
symlink(target.link().unwrap(), target.next(root)).map_err(|error| error.to_string())?;
|
||||
sync_parent(&target.next(root))?;
|
||||
}
|
||||
let active = Target::Active.path(root);
|
||||
let next = Target::Active.next(root);
|
||||
if active.exists() {
|
||||
release::copy_file(&active, &next, false)?;
|
||||
}
|
||||
release::append_active_version(&next, &manifest.version).map_err(|error| error.to_string())?;
|
||||
release::sync_tree(&bin)?;
|
||||
sync_parent(&bin)
|
||||
}
|
||||
|
||||
fn publish(root: &Path, change: &Change) -> Result<(), String> {
|
||||
if change.target == Target::Apparmor && root == Path::new("/") {
|
||||
dever_sandbox::validate_host_policy()?;
|
||||
}
|
||||
let path = change.target.path(root);
|
||||
let next = change.target.next(root);
|
||||
if identity(&path, change.target.link())? != change.before
|
||||
|| identity(&next, change.target.link())? != change.after
|
||||
{
|
||||
return Err("bootstrap destination changed during installation".into());
|
||||
}
|
||||
if change.before == change.after {
|
||||
return Ok(());
|
||||
}
|
||||
if change.before.is_some() {
|
||||
exchange(&path, &next)?;
|
||||
} else {
|
||||
rustix::fs::renameat_with(
|
||||
rustix::fs::CWD,
|
||||
&next,
|
||||
rustix::fs::CWD,
|
||||
&path,
|
||||
rustix::fs::RenameFlags::NOREPLACE,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
sync_parent(&path)
|
||||
}
|
||||
|
||||
pub(super) fn recover(root: &Path, layout: &Layout) -> Result<(), String> {
|
||||
let temporary = layout.state().join(JOURNAL_NEXT);
|
||||
if fs::symlink_metadata(&temporary).is_ok() {
|
||||
trusted_file(&temporary)?;
|
||||
fs::remove_file(&temporary).map_err(|error| error.to_string())?;
|
||||
}
|
||||
let path = layout.state().join(JOURNAL);
|
||||
let bytes = match fs::symlink_metadata(&path) {
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error.to_string()),
|
||||
Ok(_) => {
|
||||
trusted_file(&path)?;
|
||||
read_bounded(&path, 64 * 1024)?
|
||||
}
|
||||
};
|
||||
let journal: Journal = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("invalid bootstrap recovery journal: {error}"))?;
|
||||
if journal.format != "dever-bootstrap-transaction-v1"
|
||||
|| journal
|
||||
.changes
|
||||
.iter()
|
||||
.map(|change| change.target)
|
||||
.collect::<Vec<_>>()
|
||||
!= TARGETS
|
||||
|| journal.service_was_active.is_some() && root != Path::new("/")
|
||||
|| journal.service_start_requested
|
||||
&& (journal.service_was_active.is_none() || journal.phase == Phase::Preparing)
|
||||
|| journal.changes.iter().any(|change| {
|
||||
change.before.iter().chain(&change.after).any(|hash| {
|
||||
hash.len() != 64
|
||||
|| !hash
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
})
|
||||
})
|
||||
|| journal.phase != Phase::Preparing
|
||||
&& journal.changes.iter().any(|change| change.after.is_none())
|
||||
{
|
||||
return Err("bootstrap recovery journal has unsupported targets or identities".into());
|
||||
}
|
||||
// Validate all affected parents before touching any target. The journal can
|
||||
// never turn an image-root path into a write through a substituted symlink.
|
||||
for change in &journal.changes {
|
||||
trusted_directory(change.target.path(root).parent().unwrap())?;
|
||||
}
|
||||
if journal.phase == Phase::Committed {
|
||||
for change in &journal.changes {
|
||||
if identity(&change.target.path(root), change.target.link())? != change.after {
|
||||
return Err("committed bootstrap destination changed before cleanup".into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let (before_files, after_files) = if let Some(was_active) = journal.service_was_active {
|
||||
service::recovery_actions(
|
||||
journal.service_start_requested,
|
||||
was_active,
|
||||
journal.service_start_requested && service::loaded()?,
|
||||
)
|
||||
} else {
|
||||
(vec![], vec![])
|
||||
};
|
||||
service::execute(&before_files)?;
|
||||
for change in journal.changes.iter().rev() {
|
||||
rollback(root, change, journal.phase)?;
|
||||
}
|
||||
service::execute(&after_files)?;
|
||||
}
|
||||
for change in &journal.changes {
|
||||
remove_owned(&change.target.next(root), change.target.link())?;
|
||||
}
|
||||
fs::remove_file(&path).map_err(|error| error.to_string())?;
|
||||
sync_parent(&path)
|
||||
}
|
||||
|
||||
fn rollback(root: &Path, change: &Change, phase: Phase) -> Result<(), String> {
|
||||
let path = change.target.path(root);
|
||||
let next = change.target.next(root);
|
||||
let current = identity(&path, change.target.link())?;
|
||||
if current == change.before {
|
||||
return Ok(());
|
||||
}
|
||||
if phase != Phase::Prepared || current != change.after {
|
||||
return Err("bootstrap recovery refuses a destination outside this transaction".into());
|
||||
}
|
||||
if let Some(before) = &change.before {
|
||||
if identity(&next, change.target.link())?.as_ref() != Some(before) {
|
||||
return Err("bootstrap rollback copy is missing or changed".into());
|
||||
}
|
||||
exchange(&path, &next)?;
|
||||
} else {
|
||||
remove_owned(&path, change.target.link())?;
|
||||
}
|
||||
sync_parent(&path)
|
||||
}
|
||||
|
||||
fn save(layout: &Layout, journal: &Journal) -> Result<(), String> {
|
||||
let next = layout.state().join(JOURNAL_NEXT);
|
||||
write_new(
|
||||
&next,
|
||||
&serde_json::to_vec(journal).map_err(|error| error.to_string())?,
|
||||
0o600,
|
||||
)?;
|
||||
fs::rename(&next, layout.state().join(JOURNAL)).map_err(|error| error.to_string())?;
|
||||
release::sync_directory(&layout.state()).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn exchange(path: &Path, next: &Path) -> Result<(), String> {
|
||||
rustix::fs::renameat_with(
|
||||
rustix::fs::CWD,
|
||||
path,
|
||||
rustix::fs::CWD,
|
||||
next,
|
||||
rustix::fs::RenameFlags::EXCHANGE,
|
||||
)
|
||||
.map_err(|error| format!("atomic bootstrap exchange failed: {error}"))
|
||||
}
|
||||
|
||||
fn sync_parent(path: &Path) -> Result<(), String> {
|
||||
release::sync_directory(path.parent().ok_or("bootstrap destination has no parent")?)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn remove_owned(path: &Path, link: Option<&str>) -> Result<(), String> {
|
||||
let Some(_) = identity(path, link)? else {
|
||||
return Ok(());
|
||||
};
|
||||
if fs::symlink_metadata(path)
|
||||
.map_err(|error| error.to_string())?
|
||||
.is_dir()
|
||||
{
|
||||
fs::remove_dir_all(path).map_err(|error| error.to_string())?;
|
||||
} else {
|
||||
fs::remove_file(path).map_err(|error| error.to_string())?;
|
||||
}
|
||||
sync_parent(path)
|
||||
}
|
||||
|
||||
fn identity(path: &Path, link: Option<&str>) -> Result<Option<String>, String> {
|
||||
let metadata = match fs::symlink_metadata(path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(error.to_string()),
|
||||
};
|
||||
if let Some(link) = link {
|
||||
if !metadata.file_type().is_symlink()
|
||||
|| metadata.uid() != 0
|
||||
|| fs::read_link(path).map_err(|error| error.to_string())? != Path::new(link)
|
||||
{
|
||||
return Err("bootstrap link has an unsupported owner or target".into());
|
||||
}
|
||||
return Ok(Some(digest(format!("link\0{link}").as_bytes())));
|
||||
}
|
||||
let mut tree = Tree::default();
|
||||
tree.visit(path, path, 0)?;
|
||||
Ok(Some(format!("{:x}", tree.hash.finalize())))
|
||||
}
|
||||
|
||||
pub(super) fn files(root: &Path) -> Result<Vec<String>, String> {
|
||||
let mut tree = Tree::default();
|
||||
tree.visit(root, root, 0)?;
|
||||
Ok(tree.files.into_iter().collect())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Tree {
|
||||
hash: Sha256,
|
||||
bytes: u64,
|
||||
entries: usize,
|
||||
files: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
fn visit(&mut self, root: &Path, path: &Path, depth: usize) -> Result<(), String> {
|
||||
self.entries += 1;
|
||||
if self.entries > 128 || depth > 4 {
|
||||
return Err("bootstrap tree exceeds its entry/depth bound".into());
|
||||
}
|
||||
let metadata = fs::symlink_metadata(path).map_err(|error| error.to_string())?;
|
||||
let name = path
|
||||
.strip_prefix(root)
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.ok_or("bootstrap path is not UTF-8")?;
|
||||
self.hash.update((name.len() as u64).to_le_bytes());
|
||||
self.hash.update(name.as_bytes());
|
||||
self.hash.update((metadata.mode() & 0o777).to_le_bytes());
|
||||
if metadata.is_dir() && !metadata.file_type().is_symlink() {
|
||||
trusted_directory(path)?;
|
||||
self.hash.update(b"directory");
|
||||
let mut entries = fs::read_dir(path)
|
||||
.map_err(|error| error.to_string())?
|
||||
.map(|entry| entry.map(|entry| entry.path()))
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
entries.sort();
|
||||
for entry in entries {
|
||||
self.visit(root, &entry, depth + 1)?;
|
||||
}
|
||||
} else {
|
||||
trusted_file(path)?;
|
||||
self.bytes = self
|
||||
.bytes
|
||||
.checked_add(metadata.len())
|
||||
.ok_or("bootstrap tree size overflow")?;
|
||||
if self.bytes > MAX_TREE_BYTES {
|
||||
return Err("bootstrap tree exceeds its byte bound".into());
|
||||
}
|
||||
let bytes = read_bounded(path, metadata.len())?;
|
||||
self.hash.update(b"file");
|
||||
self.hash.update((bytes.len() as u64).to_le_bytes());
|
||||
self.hash.update(bytes);
|
||||
self.files.insert(name.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::release::{Layout, Version, hex};
|
||||
|
||||
static NEXT_CACHE_OPERATION: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BuildIdentity<'a> {
|
||||
pub version: &'a Version,
|
||||
pub compiler: &'a [u8],
|
||||
pub runtime: &'a [u8],
|
||||
pub target: &'a str,
|
||||
pub generated: &'a [u8],
|
||||
pub options: &'a [u8],
|
||||
pub environment: &'a [u8],
|
||||
}
|
||||
|
||||
impl BuildIdentity<'_> {
|
||||
pub fn key(&self) -> String {
|
||||
let mut digest = Sha256::new();
|
||||
digest.update(b"dever-machine-cache-v1\0");
|
||||
for value in [
|
||||
self.version.as_str().as_bytes(),
|
||||
self.compiler,
|
||||
self.runtime,
|
||||
self.target.as_bytes(),
|
||||
self.generated,
|
||||
self.options,
|
||||
self.environment,
|
||||
] {
|
||||
digest.update((value.len() as u64).to_le_bytes());
|
||||
digest.update(value);
|
||||
}
|
||||
hex(&digest.finalize())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CacheEntry {
|
||||
pub format: String,
|
||||
pub key: String,
|
||||
pub version: Version,
|
||||
pub target: String,
|
||||
pub bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
|
||||
pub struct CacheStatus {
|
||||
#[serde(default)]
|
||||
pub verified: bool,
|
||||
pub entries: u64,
|
||||
pub bytes: u64,
|
||||
pub versions: BTreeMap<String, u64>,
|
||||
pub corrupt_entries: u64,
|
||||
pub staging_entries: u64,
|
||||
#[serde(default)]
|
||||
pub artifact_entries: u64,
|
||||
#[serde(default)]
|
||||
pub artifact_bytes: u64,
|
||||
#[serde(default)]
|
||||
pub corrupt_artifacts: u64,
|
||||
}
|
||||
|
||||
/// The service accepts content identities, never caller-selected cache paths.
|
||||
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ArtifactReceipt {
|
||||
pub sha256: String,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
pub const MAX_ARTIFACT_BYTES: u64 = 64 * 1024 * 1024;
|
||||
pub(super) const MAX_COMPILED_BYTES: u64 = 320 * 1024 * 1024;
|
||||
const MAX_ARTIFACT_CACHE_BYTES: u64 = 2 * 1024 * 1024 * 1024;
|
||||
const MAX_ARTIFACT_ENTRIES: usize = 4096;
|
||||
|
||||
impl ArtifactReceipt {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
validate_key(&self.sha256)?;
|
||||
if self.bytes > MAX_ARTIFACT_BYTES {
|
||||
return Err("shared artifact exceeds the 64 MiB transfer limit".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_compiled(&self) -> Result<(), String> {
|
||||
validate_key(&self.sha256)?;
|
||||
if self.bytes == 0 || self.bytes > MAX_COMPILED_BYTES {
|
||||
return Err("compiled output must be between 1 byte and 320 MiB".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct StoredArtifact {
|
||||
pub receipt: ArtifactReceipt,
|
||||
pub file: File,
|
||||
_operation: CacheOperation,
|
||||
}
|
||||
|
||||
/// Filesystem engine owned by the privileged cache process. It deliberately has
|
||||
/// no transport or caller-authentication semantics; project processes must use
|
||||
/// the authenticated `deverd` IPC client rather than constructing this store directly.
|
||||
pub struct CacheStore {
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
impl CacheStore {
|
||||
pub fn new(layout: &Layout) -> Self {
|
||||
Self {
|
||||
root: layout.native_cache(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Hold across input staging, compilation, publication and response transfer.
|
||||
pub(crate) fn operation(&self) -> Result<CacheOperation, String> {
|
||||
CacheOperation::acquire(&self.root)
|
||||
}
|
||||
|
||||
pub(crate) fn compilation_directory(&self) -> Result<CompilationDirectory, String> {
|
||||
let root = self.root.join("staging");
|
||||
create_directory(&root)?;
|
||||
let path = root.join(format!(
|
||||
"compile-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
builder
|
||||
.create(&path)
|
||||
.map_err(|error| format!("cannot stage compilation: {error}"))?;
|
||||
Ok(CompilationDirectory(path))
|
||||
}
|
||||
|
||||
/// Hash while receiving into an owned staging file. Only verified complete
|
||||
/// bytes are published, and concurrent identical uploads share one entry.
|
||||
pub fn publish_artifact(
|
||||
&self,
|
||||
expected: &ArtifactReceipt,
|
||||
input: &mut impl Read,
|
||||
) -> Result<ArtifactReceipt, String> {
|
||||
expected.validate()?;
|
||||
let _operation = CacheOperation::acquire(&self.root)?;
|
||||
let staging_root = self.root.join("staging");
|
||||
create_directory(&staging_root)?;
|
||||
let staging = staging_root.join(format!(
|
||||
"artifact-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let result = (|| {
|
||||
let mut options = OpenOptions::new();
|
||||
options.write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
let mut file = options
|
||||
.open(&staging)
|
||||
.map_err(|error| format!("cannot stage shared artifact: {error}"))?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut remaining = expected.bytes;
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
while remaining > 0 {
|
||||
let limit = remaining.min(buffer.len() as u64) as usize;
|
||||
input
|
||||
.read_exact(&mut buffer[..limit])
|
||||
.map_err(|error| format!("incomplete shared artifact upload: {error}"))?;
|
||||
file.write_all(&buffer[..limit])
|
||||
.map_err(|error| format!("cannot stage shared artifact: {error}"))?;
|
||||
digest.update(&buffer[..limit]);
|
||||
remaining -= limit as u64;
|
||||
}
|
||||
if hex(&digest.finalize()) != expected.sha256 {
|
||||
return Err("shared artifact upload failed SHA-256 verification".into());
|
||||
}
|
||||
file.sync_all()
|
||||
.map_err(|error| format!("cannot sync shared artifact: {error}"))?;
|
||||
drop(file);
|
||||
let artifacts = self.root.join("artifacts");
|
||||
create_directory(&artifacts)?;
|
||||
let _publication = ArtifactPublication::acquire(&self.root)?;
|
||||
let destination = artifacts.join(&expected.sha256);
|
||||
match fs::symlink_metadata(&destination) {
|
||||
Ok(_) => {
|
||||
self.read_artifact_file(expected)?;
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
let (used, count) = directory_entries(&artifacts)?.try_fold(
|
||||
(0_u64, 0_usize),
|
||||
|(sum, count), entry| {
|
||||
let metadata = fs::symlink_metadata(entry.path())
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !metadata.is_file() || metadata.file_type().is_symlink() {
|
||||
return Err(
|
||||
"shared artifact cache contains a non-regular file".into()
|
||||
);
|
||||
}
|
||||
Ok::<_, String>((sum.saturating_add(metadata.len()), count + 1))
|
||||
},
|
||||
)?;
|
||||
if count >= MAX_ARTIFACT_ENTRIES
|
||||
|| used.saturating_add(expected.bytes) > MAX_ARTIFACT_CACHE_BYTES
|
||||
{
|
||||
return Err("shared artifact cache is full; administrator must run 'dever cache clean'".into());
|
||||
}
|
||||
fs::rename(&staging, &destination)
|
||||
.map_err(|error| format!("cannot publish shared artifact: {error}"))?;
|
||||
sync_directory(&artifacts)
|
||||
.map_err(|error| format!("cannot sync shared artifacts: {error}"))?;
|
||||
}
|
||||
Err(error) => return Err(format!("cannot inspect shared artifact: {error}")),
|
||||
}
|
||||
Ok(expected.clone())
|
||||
})();
|
||||
if staging.exists() {
|
||||
let _ = fs::remove_file(&staging);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn artifact(&self, expected: &ArtifactReceipt) -> Result<StoredArtifact, String> {
|
||||
expected.validate()?;
|
||||
let operation = CacheOperation::acquire(&self.root)?;
|
||||
let file = self.read_artifact_file(expected)?;
|
||||
Ok(StoredArtifact {
|
||||
receipt: expected.clone(),
|
||||
file,
|
||||
_operation: operation,
|
||||
})
|
||||
}
|
||||
|
||||
fn read_artifact_file(&self, expected: &ArtifactReceipt) -> Result<File, String> {
|
||||
let path = self.artifact_path(expected)?;
|
||||
if sha256_file(&path)? != expected.sha256 {
|
||||
return Err("shared artifact failed SHA-256 verification".into());
|
||||
}
|
||||
File::open(&path).map_err(|error| format!("cannot read shared artifact: {error}"))
|
||||
}
|
||||
|
||||
fn artifact_path(&self, expected: &ArtifactReceipt) -> Result<PathBuf, String> {
|
||||
expected.validate()?;
|
||||
let directory = self.root.join("artifacts");
|
||||
ensure_real_directory(&directory)
|
||||
.map_err(|error| format!("shared artifact unavailable: {error}"))?;
|
||||
let path = directory.join(&expected.sha256);
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| format!("shared artifact unavailable: {error}"))?;
|
||||
if !metadata.is_file()
|
||||
|| metadata.file_type().is_symlink()
|
||||
|| metadata.len() != expected.bytes
|
||||
{
|
||||
return Err("shared artifact is incomplete or modified".into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let owner = fs::symlink_metadata(&self.root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.uid();
|
||||
if metadata.uid() != owner || metadata.mode() & 0o077 != 0 {
|
||||
return Err("shared artifact must be private to the service owner".into());
|
||||
}
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub fn publish(
|
||||
&self,
|
||||
identity: &BuildIdentity<'_>,
|
||||
executable: &Path,
|
||||
) -> Result<String, String> {
|
||||
let _operation = CacheOperation::acquire(&self.root)?;
|
||||
let _publication = ArtifactPublication::acquire(&self.root)?;
|
||||
let key = identity.key();
|
||||
let destination = self.root.join("entries").join(&key);
|
||||
if destination.exists() {
|
||||
self.read_entry(&key)?;
|
||||
return Ok(key);
|
||||
}
|
||||
let staging_root = self.root.join("staging");
|
||||
create_directory(&staging_root)?;
|
||||
create_directory(&self.root.join("entries"))?;
|
||||
let bytes = fs::symlink_metadata(executable)
|
||||
.map_err(|error| format!("cannot inspect compiled output: {error}"))?
|
||||
.len();
|
||||
if bytes == 0 || bytes > MAX_COMPILED_BYTES {
|
||||
return Err("compiled output must be between 1 byte and 320 MiB".into());
|
||||
}
|
||||
let (used, count) = directory_entries(&self.root.join("entries"))?.try_fold(
|
||||
(0_u64, 0_usize),
|
||||
|(used, count), entry| {
|
||||
let key = entry.file_name().to_string_lossy().into_owned();
|
||||
let manifest = self.inspect_entry(&key, false)?;
|
||||
Ok::<_, String>((used.saturating_add(manifest.bytes), count + 1))
|
||||
},
|
||||
)?;
|
||||
if count >= MAX_ARTIFACT_ENTRIES || used.saturating_add(bytes) > MAX_ARTIFACT_CACHE_BYTES {
|
||||
return Err("shared compilation cache quota exceeded (2 GiB / 4096 entries)".into());
|
||||
}
|
||||
let staging = staging_root.join(format!(
|
||||
"publish-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
|
||||
));
|
||||
create_directory(&staging)?;
|
||||
let result = (|| {
|
||||
let program = staging.join("program");
|
||||
copy_regular_file(executable, &program)?;
|
||||
let metadata = fs::metadata(&program).map_err(|error| error.to_string())?;
|
||||
let entry = CacheEntry {
|
||||
format: "dever-machine-cache-entry-v1".into(),
|
||||
key: key.clone(),
|
||||
version: identity.version.clone(),
|
||||
target: identity.target.to_owned(),
|
||||
bytes: metadata.len(),
|
||||
sha256: sha256_file(&program)?,
|
||||
};
|
||||
let manifest = serde_json::to_vec(&entry).map_err(|error| error.to_string())?;
|
||||
write_new(&staging.join("entry.json"), &manifest)?;
|
||||
sync_directory(&staging).map_err(|error| error.to_string())?;
|
||||
match fs::rename(&staging, &destination) {
|
||||
Ok(()) => {
|
||||
sync_directory(&self.root.join("entries")).map_err(|error| error.to_string())
|
||||
}
|
||||
Err(_error) if destination.exists() => {
|
||||
self.read_entry(&key)?;
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(format!("cannot publish shared cache entry: {error}")),
|
||||
}
|
||||
})();
|
||||
if staging.exists() {
|
||||
let _ = fs::remove_dir_all(&staging);
|
||||
}
|
||||
result.map(|()| key)
|
||||
}
|
||||
|
||||
pub fn restore(&self, identity: &BuildIdentity<'_>) -> Result<Option<CachedArtifact>, String> {
|
||||
let key = identity.key();
|
||||
let entry = match self.read_entry(&key) {
|
||||
Ok(entry) => entry,
|
||||
Err(error) if error.starts_with("shared cache miss:") => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if entry.version != *identity.version || entry.target != identity.target {
|
||||
return Err("shared cache identity collision".into());
|
||||
}
|
||||
let lease = CacheLease::acquire(&self.root, &key)?;
|
||||
let path = self.root.join("entries").join(&key).join("program");
|
||||
let bytes = read_regular_file(&path)?;
|
||||
if bytes.len() as u64 != entry.bytes || sha256(&bytes) != entry.sha256 {
|
||||
return Err(format!(
|
||||
"shared cache entry {key} is incomplete or modified"
|
||||
));
|
||||
}
|
||||
Ok(Some(CachedArtifact {
|
||||
bytes,
|
||||
_lease: lease,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn status(&self) -> Result<CacheStatus, String> {
|
||||
self.inspect_status(true)
|
||||
}
|
||||
|
||||
/// Unprivileged status is metadata-only: it cannot force hashing all
|
||||
/// libraries/programs. It never establishes an artifact integrity claim.
|
||||
pub fn summary(&self) -> Result<CacheStatus, String> {
|
||||
self.inspect_status(false)
|
||||
}
|
||||
|
||||
fn inspect_status(&self, verify_bytes: bool) -> Result<CacheStatus, String> {
|
||||
let mut status = CacheStatus {
|
||||
verified: verify_bytes,
|
||||
..CacheStatus::default()
|
||||
};
|
||||
for entry in directory_entries(&self.root.join("staging"))? {
|
||||
let _ = entry;
|
||||
status.staging_entries += 1;
|
||||
}
|
||||
for entry in directory_entries(&self.root.join("entries"))? {
|
||||
let key = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "shared cache key is not UTF-8".to_owned())?;
|
||||
match self.inspect_entry(&key, verify_bytes) {
|
||||
Ok(manifest) => {
|
||||
status.entries += 1;
|
||||
status.bytes = status.bytes.saturating_add(manifest.bytes);
|
||||
*status
|
||||
.versions
|
||||
.entry(manifest.version.to_string())
|
||||
.or_default() += 1;
|
||||
}
|
||||
Err(_) => status.corrupt_entries += 1,
|
||||
}
|
||||
}
|
||||
for entry in directory_entries(&self.root.join("artifacts"))? {
|
||||
let metadata = fs::symlink_metadata(entry.path()).map_err(|error| error.to_string())?;
|
||||
let expected = ArtifactReceipt {
|
||||
sha256: entry.file_name().to_string_lossy().into_owned(),
|
||||
bytes: metadata.len(),
|
||||
};
|
||||
let inspected = if verify_bytes {
|
||||
self.read_artifact_file(&expected).map(|_| ())
|
||||
} else {
|
||||
self.artifact_path(&expected).map(|_| ())
|
||||
};
|
||||
match inspected {
|
||||
Ok(_) => {
|
||||
status.artifact_entries += 1;
|
||||
status.artifact_bytes = status.artifact_bytes.saturating_add(expected.bytes);
|
||||
}
|
||||
Err(_) => status.corrupt_artifacts += 1,
|
||||
}
|
||||
}
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub fn clean(&self, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> {
|
||||
let _lock = CacheCleanLock::acquire(&self.root)?;
|
||||
if directory_entries(&self.root.join("operations"))?
|
||||
.next()
|
||||
.is_some()
|
||||
{
|
||||
return Err("shared cache is busy; clean did not remove active build state".into());
|
||||
}
|
||||
remove_children(&self.root.join("staging"))?;
|
||||
// Artifacts are a recoverable download cache. Explicit administrator
|
||||
// cleanup removes them only while no upload or reader is active.
|
||||
remove_children(&self.root.join("artifacts"))?;
|
||||
for entry in directory_entries(&self.root.join("entries"))? {
|
||||
let key = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "shared cache key is not UTF-8".to_owned())?;
|
||||
if self.has_lease(&key)? {
|
||||
continue;
|
||||
}
|
||||
let remove = self
|
||||
.read_entry(&key)
|
||||
.map(|entry| !installed.contains(&entry.version))
|
||||
.unwrap_or(true);
|
||||
if remove {
|
||||
remove_entry(entry.path())?;
|
||||
}
|
||||
}
|
||||
self.status()
|
||||
}
|
||||
|
||||
fn read_entry(&self, key: &str) -> Result<CacheEntry, String> {
|
||||
self.inspect_entry(key, true)
|
||||
}
|
||||
|
||||
fn inspect_entry(&self, key: &str, verify_bytes: bool) -> Result<CacheEntry, String> {
|
||||
validate_key(key)?;
|
||||
let root = self.root.join("entries").join(key);
|
||||
if !root.exists() {
|
||||
return Err(format!("shared cache miss: {key}"));
|
||||
}
|
||||
ensure_real_directory(&root)?;
|
||||
let manifest: CacheEntry =
|
||||
serde_json::from_slice(&read_regular_file(&root.join("entry.json"))?)
|
||||
.map_err(|error| format!("invalid shared cache manifest {key}: {error}"))?;
|
||||
if manifest.format != "dever-machine-cache-entry-v1"
|
||||
|| manifest.key != key
|
||||
|| manifest.bytes == 0
|
||||
|| manifest.bytes > MAX_COMPILED_BYTES
|
||||
{
|
||||
return Err(format!("invalid shared cache manifest {key}"));
|
||||
}
|
||||
let program = root.join("program");
|
||||
let metadata = fs::symlink_metadata(&program)
|
||||
.map_err(|error| format!("cannot inspect shared cache entry {key}: {error}"))?;
|
||||
if !metadata.file_type().is_file()
|
||||
|| metadata.len() != manifest.bytes
|
||||
|| (verify_bytes && sha256_file(&program)? != manifest.sha256)
|
||||
{
|
||||
return Err(format!(
|
||||
"shared cache entry {key} is incomplete or modified"
|
||||
));
|
||||
}
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
fn has_lease(&self, key: &str) -> Result<bool, String> {
|
||||
let directory = self.root.join("leases").join(key);
|
||||
Ok(directory_entries(&directory)?.next().is_some())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CachedArtifact {
|
||||
bytes: Vec<u8>,
|
||||
_lease: CacheLease,
|
||||
}
|
||||
|
||||
impl CachedArtifact {
|
||||
pub fn as_bytes(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
struct CacheLease {
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl CacheLease {
|
||||
fn acquire(root: &Path, key: &str) -> Result<Self, String> {
|
||||
if root.join("clean.lock").exists() {
|
||||
return Err("shared cache maintenance is active".into());
|
||||
}
|
||||
let directory = root.join("leases").join(key);
|
||||
create_directory(&directory)?;
|
||||
let path = directory.join(format!(
|
||||
"{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
|
||||
));
|
||||
write_new(&path, b"dever-cache-lease-v1\n")?;
|
||||
if root.join("clean.lock").exists() {
|
||||
let _ = fs::remove_file(&path);
|
||||
return Err("shared cache maintenance is active".into());
|
||||
}
|
||||
Ok(Self { path })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheLease {
|
||||
fn drop(&mut self) {
|
||||
let parent = self.path.parent().map(Path::to_path_buf);
|
||||
let _ = fs::remove_file(&self.path);
|
||||
if let Some(parent) = parent {
|
||||
let _ = fs::remove_dir(parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CacheCleanLock(PathBuf);
|
||||
|
||||
impl CacheCleanLock {
|
||||
fn acquire(root: &Path) -> Result<Self, String> {
|
||||
create_directory(root)?;
|
||||
let path = root.join("clean.lock");
|
||||
write_new(&path, format!("{}\n", std::process::id()).as_bytes())?;
|
||||
Ok(Self(path))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct CacheOperation(PathBuf);
|
||||
|
||||
pub(crate) struct CompilationDirectory(pub(crate) PathBuf);
|
||||
|
||||
impl Drop for CompilationDirectory {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl CacheOperation {
|
||||
fn acquire(root: &Path) -> Result<Self, String> {
|
||||
create_directory(root)?;
|
||||
if root.join("clean.lock").exists() {
|
||||
return Err("shared cache maintenance is active".into());
|
||||
}
|
||||
let directory = root.join("operations");
|
||||
create_directory(&directory)?;
|
||||
let path = directory.join(format!(
|
||||
"{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_CACHE_OPERATION.fetch_add(1, Ordering::Relaxed),
|
||||
));
|
||||
write_new(&path, b"dever-cache-operation-v1\n")?;
|
||||
if root.join("clean.lock").exists() {
|
||||
let _ = fs::remove_file(&path);
|
||||
return Err("shared cache maintenance is active".into());
|
||||
}
|
||||
Ok(Self(path))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheOperation {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for CacheCleanLock {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Serializes quota accounting and publication, not network reception.
|
||||
struct ArtifactPublication(File);
|
||||
|
||||
impl ArtifactPublication {
|
||||
fn acquire(root: &Path) -> Result<Self, String> {
|
||||
let path = root.join("artifact-publish.lock");
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true).write(true);
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => {}
|
||||
Ok(_) => return Err("artifact publication lock must be a regular file".into()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
options.create_new(true);
|
||||
}
|
||||
Err(error) => return Err(format!("cannot inspect artifact publication lock: {error}")),
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
let file = match options.open(&path) {
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot open artifact publication lock: {error}"))?,
|
||||
Err(error) => return Err(format!("cannot open artifact publication lock: {error}")),
|
||||
};
|
||||
fs2::FileExt::lock_exclusive(&file)
|
||||
.map_err(|error| format!("cannot lock artifact publication: {error}"))?;
|
||||
Ok(Self(file))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArtifactPublication {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs2::FileExt::unlock(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_key(key: &str) -> Result<(), String> {
|
||||
if key.len() != 64
|
||||
|| !key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err("invalid shared cache key".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn directory_entries(path: &Path) -> Result<impl Iterator<Item = fs::DirEntry>, String> {
|
||||
let entries = match fs::read_dir(path) {
|
||||
Ok(entries) => entries,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new().into_iter()),
|
||||
Err(error) => return Err(format!("cannot inspect '{}': {error}", path.display())),
|
||||
};
|
||||
let entries = entries
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
|
||||
Ok(entries.into_iter())
|
||||
}
|
||||
|
||||
fn remove_children(path: &Path) -> Result<(), String> {
|
||||
for entry in directory_entries(path)? {
|
||||
remove_entry(entry.path())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_entry(path: PathBuf) -> Result<(), String> {
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
|
||||
if metadata.is_dir() && !metadata.file_type().is_symlink() {
|
||||
fs::remove_dir_all(&path)
|
||||
} else {
|
||||
fs::remove_file(&path)
|
||||
}
|
||||
.map_err(|error| format!("cannot remove '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn create_directory(path: &Path) -> Result<(), String> {
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
builder.recursive(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
builder
|
||||
.create(path)
|
||||
.map_err(|error| format!("cannot create '{}': {error}", path.display()))?;
|
||||
ensure_real_directory(path)
|
||||
}
|
||||
|
||||
fn ensure_real_directory(path: &Path) -> Result<(), String> {
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
|
||||
if metadata.file_type().is_symlink() || !metadata.is_dir() {
|
||||
return Err(format!("'{}' must be a real directory", path.display()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn copy_regular_file(source: &Path, destination: &Path) -> Result<(), String> {
|
||||
let metadata = fs::symlink_metadata(source)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", source.display()))?;
|
||||
if !metadata.file_type().is_file() {
|
||||
return Err(format!("'{}' must be a regular file", source.display()));
|
||||
}
|
||||
fs::copy(source, destination)
|
||||
.map_err(|error| format!("cannot copy '{}': {error}", source.display()))?;
|
||||
fs::set_permissions(destination, metadata.permissions())
|
||||
.map_err(|error| format!("cannot preserve '{}': {error}", destination.display()))
|
||||
}
|
||||
|
||||
fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> {
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.map_err(|error| format!("cannot inspect '{}': {error}", path.display()))?;
|
||||
if !metadata.file_type().is_file() {
|
||||
return Err(format!("'{}' must be a regular file", path.display()));
|
||||
}
|
||||
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn write_new(path: &Path, value: &[u8]) -> Result<(), String> {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path)
|
||||
.map_err(|error| format!("cannot create '{}': {error}", path.display()))?;
|
||||
file.write_all(value)
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| format!("cannot write '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn sha256_file(path: &Path) -> Result<String, String> {
|
||||
let mut file =
|
||||
File::open(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))?;
|
||||
let mut digest = Sha256::new();
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let length = file
|
||||
.read(&mut buffer)
|
||||
.map_err(|error| format!("cannot hash '{}': {error}", path.display()))?;
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
digest.update(&buffer[..length]);
|
||||
}
|
||||
Ok(hex(&digest.finalize()))
|
||||
}
|
||||
|
||||
fn sha256(value: &[u8]) -> String {
|
||||
hex(&Sha256::digest(value))
|
||||
}
|
||||
|
||||
fn sync_directory(path: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
File::open(path)?.sync_all()
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let _ = path;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
//! Bounded source-only protocol shared by callers, the service and its worker.
|
||||
|
||||
use super::BuildTarget;
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose::STANDARD;
|
||||
use dever_core::native::EmbeddedResource;
|
||||
use dever_core::source::SourceMap;
|
||||
use dever_runtime::config::CompilationBindings;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
// Base64 plus source/metadata must fit while embedding a complete interpreter.
|
||||
pub const MAX_COMPILE_BYTES: usize = 384 * 1024 * 1024;
|
||||
const MAX_RESOURCE_BYTES: usize = 128 * 1024 * 1024;
|
||||
const MAX_RESOURCE_TOTAL: usize = 256 * 1024 * 1024;
|
||||
pub const WORKER_REQUEST_FILE: &str = "request.json";
|
||||
pub const WORKER_OUTPUT_FILE: &str = "program";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, Eq, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CompileKind {
|
||||
Application,
|
||||
TestSuite,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CompileRequest {
|
||||
pub version: String,
|
||||
pub kind: CompileKind,
|
||||
pub target: BuildTarget,
|
||||
pub sources: Vec<CompilationSource>,
|
||||
pub bindings: Option<CompilationBindings>,
|
||||
pub resources: Vec<CompilationResource>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CompilationSource {
|
||||
pub logical_path: String,
|
||||
pub is_test: bool,
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CompilationResource {
|
||||
pub path: String,
|
||||
#[serde(with = "resource_bytes")]
|
||||
pub bytes: Vec<u8>,
|
||||
pub sha256: String,
|
||||
pub executable: bool,
|
||||
}
|
||||
|
||||
impl CompileRequest {
|
||||
pub fn new(
|
||||
version: String,
|
||||
kind: CompileKind,
|
||||
sources: &SourceMap,
|
||||
bindings: Option<CompilationBindings>,
|
||||
resources: &[EmbeddedResource],
|
||||
target: BuildTarget,
|
||||
) -> Result<Self, String> {
|
||||
let request = Self {
|
||||
version,
|
||||
kind,
|
||||
target,
|
||||
sources: sources
|
||||
.compilation_sources()?
|
||||
.into_iter()
|
||||
.map(|source| CompilationSource {
|
||||
logical_path: source.logical_path,
|
||||
is_test: source.is_test,
|
||||
text: source.text,
|
||||
})
|
||||
.collect(),
|
||||
bindings,
|
||||
resources: resources
|
||||
.iter()
|
||||
.map(|resource| CompilationResource {
|
||||
path: resource.path.clone(),
|
||||
bytes: resource.bytes.clone(),
|
||||
sha256: resource.sha256.clone(),
|
||||
executable: resource.executable,
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
request.validate()?;
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
super::Version::parse(&self.version)?;
|
||||
if self.kind == CompileKind::TestSuite && self.target != BuildTarget::host()? {
|
||||
return Err("test compilation requires the compiler host target".into());
|
||||
}
|
||||
if self.sources.is_empty()
|
||||
|| self.sources.len() > 4096
|
||||
|| self
|
||||
.sources
|
||||
.iter()
|
||||
.map(|source| source.text.len())
|
||||
.sum::<usize>()
|
||||
> 16 * 1024 * 1024
|
||||
|| self
|
||||
.sources
|
||||
.iter()
|
||||
.any(|source| source.logical_path.len() > 4096)
|
||||
|| !self.sources.iter().any(|source| !source.is_test)
|
||||
{
|
||||
return Err("compilation requires application sources, at most 4096 files and 16 MiB of source text".into());
|
||||
}
|
||||
self.source_map()?;
|
||||
match self.kind {
|
||||
CompileKind::Application => {
|
||||
if self.sources.iter().any(|source| source.is_test) {
|
||||
return Err("application compilation cannot contain test sources".into());
|
||||
}
|
||||
self.bindings
|
||||
.as_ref()
|
||||
.ok_or("application compilation requires deployment bindings")?
|
||||
.validate()?;
|
||||
}
|
||||
CompileKind::TestSuite if self.bindings.is_some() || !self.resources.is_empty() => {
|
||||
return Err("test compilation derives case-local bindings and cannot contain deployment bindings or external resources".into());
|
||||
}
|
||||
CompileKind::TestSuite => {}
|
||||
}
|
||||
if self.resources.len() > 4096
|
||||
|| self
|
||||
.resources
|
||||
.iter()
|
||||
.map(|resource| resource.bytes.len())
|
||||
.sum::<usize>()
|
||||
> MAX_RESOURCE_TOTAL
|
||||
{
|
||||
return Err("compilation resources exceed 4096 files or 256 MiB".into());
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
for resource in &self.resources {
|
||||
relative_path(&resource.path)?;
|
||||
if !paths.insert(&resource.path) {
|
||||
return Err("duplicate compilation resource path".into());
|
||||
}
|
||||
if resource.bytes.len() > MAX_RESOURCE_BYTES
|
||||
|| format!("{:x}", Sha256::digest(&resource.bytes)) != resource.sha256
|
||||
{
|
||||
return Err(format!(
|
||||
"compilation resource '{}' exceeds 128 MiB or has an invalid digest",
|
||||
resource.path
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn source_map(&self) -> Result<SourceMap, String> {
|
||||
SourceMap::from_compilation_sources(
|
||||
&self
|
||||
.sources
|
||||
.iter()
|
||||
.map(|source| dever_core::source::CompilationSource {
|
||||
logical_path: source.logical_path.clone(),
|
||||
is_test: source.is_test,
|
||||
text: source.text.clone(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn embedded_resources(&self) -> Vec<EmbeddedResource> {
|
||||
self.resources
|
||||
.iter()
|
||||
.map(|resource| EmbeddedResource {
|
||||
path: resource.path.clone(),
|
||||
bytes: resource.bytes.clone(),
|
||||
sha256: resource.sha256.clone(),
|
||||
executable: resource.executable,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn encode(&self) -> Result<Vec<u8>, String> {
|
||||
self.validate()?;
|
||||
#[derive(Serialize)]
|
||||
struct Canonical<'a> {
|
||||
version: &'a str,
|
||||
kind: CompileKind,
|
||||
target: BuildTarget,
|
||||
sources: Vec<&'a CompilationSource>,
|
||||
bindings: &'a Option<CompilationBindings>,
|
||||
resources: Vec<&'a CompilationResource>,
|
||||
}
|
||||
// Sort references: interpreter payloads must not be cloned to order JSON.
|
||||
let mut sources = self.sources.iter().collect::<Vec<_>>();
|
||||
sources.sort_by(|left, right| {
|
||||
(left.is_test, &left.logical_path).cmp(&(right.is_test, &right.logical_path))
|
||||
});
|
||||
let mut resources = self.resources.iter().collect::<Vec<_>>();
|
||||
resources.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
let canonical = Canonical {
|
||||
version: &self.version,
|
||||
kind: self.kind,
|
||||
target: self.target,
|
||||
sources,
|
||||
bindings: &self.bindings,
|
||||
resources,
|
||||
};
|
||||
let bytes = serde_json::to_vec(&canonical)
|
||||
.map_err(|error| format!("cannot encode compile request: {error}"))?;
|
||||
if bytes.len() > MAX_COMPILE_BYTES {
|
||||
return Err("compile request exceeds 384 MiB".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn decode(bytes: &[u8]) -> Result<Self, String> {
|
||||
if bytes.len() > MAX_COMPILE_BYTES {
|
||||
return Err("compile request exceeds 384 MiB".into());
|
||||
}
|
||||
let request: Self = serde_json::from_slice(bytes)
|
||||
.map_err(|error| format!("invalid compile request: {error}"))?;
|
||||
// Requiring one representation rejects duplicate map keys as well as
|
||||
// ambiguous ordering/escapes before requests acquire a cache identity.
|
||||
if request.encode()? != bytes {
|
||||
return Err("compile request must use canonical JSON without duplicate keys".into());
|
||||
}
|
||||
Ok(request)
|
||||
}
|
||||
}
|
||||
|
||||
fn relative_path(path: &str) -> Result<(), String> {
|
||||
if path.is_empty()
|
||||
|| path.len() > 4096
|
||||
|| path.contains(['\\', ':', '\0'])
|
||||
|| path
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
{
|
||||
return Err("compilation resources require normalized relative paths".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Workers accept inputs only inside the service owner's private compile stage.
|
||||
/// This prevents the hidden entry from becoming a client-side managed fallback.
|
||||
pub fn worker_directory(layout: &super::Layout) -> Result<std::path::PathBuf, String> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::fs;
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
let directory = std::env::current_dir()
|
||||
.map_err(|error| format!("cannot locate compiler worker directory: {error}"))?;
|
||||
let staging = layout.native_cache().join("staging");
|
||||
if directory.parent() != Some(staging.as_path())
|
||||
|| !directory
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| {
|
||||
name.strip_prefix("compile-")
|
||||
.is_some_and(|suffix| !suffix.is_empty())
|
||||
})
|
||||
{
|
||||
return Err(
|
||||
"compile worker requires a service-owned native cache staging directory".into(),
|
||||
);
|
||||
}
|
||||
let owner = rustix::process::geteuid().as_raw();
|
||||
for (path, private) in [
|
||||
(layout.state(), false),
|
||||
(layout.cache(), false),
|
||||
(layout.native_cache(), false),
|
||||
(staging, true),
|
||||
(directory.clone(), true),
|
||||
] {
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| format!("cannot inspect compiler worker directory: {error}"))?;
|
||||
if metadata.file_type().is_symlink()
|
||||
|| !metadata.is_dir()
|
||||
|| metadata.uid() != owner
|
||||
|| metadata.mode() & 0o022 != 0
|
||||
|| private && metadata.mode() & 0o777 != 0o700
|
||||
{
|
||||
return Err("compiler worker directories must be real and service-owned without group/other writes; staging must be private (0700)".into());
|
||||
}
|
||||
}
|
||||
let request = fs::symlink_metadata(directory.join(WORKER_REQUEST_FILE))
|
||||
.map_err(|error| format!("cannot inspect compiler worker request: {error}"))?;
|
||||
if request.file_type().is_symlink()
|
||||
|| !request.is_file()
|
||||
|| request.uid() != owner
|
||||
|| request.mode() & 0o777 != 0o600
|
||||
{
|
||||
return Err(
|
||||
"compiler worker request must be a service-owned regular file with mode 0600"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
Ok(directory)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = layout;
|
||||
Err("trusted compiler worker directories are unavailable on this platform".into())
|
||||
}
|
||||
}
|
||||
|
||||
mod resource_bytes {
|
||||
use super::*;
|
||||
pub fn serialize<S: serde::Serializer>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> {
|
||||
serializer.serialize_str(&STANDARD.encode(bytes))
|
||||
}
|
||||
pub fn deserialize<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<Vec<u8>, D::Error> {
|
||||
let encoded = String::deserialize(deserializer)?;
|
||||
if encoded.len() > MAX_RESOURCE_BYTES * 4 / 3 + 4 {
|
||||
return Err(serde::de::Error::custom(
|
||||
"compilation resource exceeds 128 MiB",
|
||||
));
|
||||
}
|
||||
STANDARD.decode(encoded).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
//! Native compiler loader closure, shared by authoring and signed installation.
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use goblin::elf::{Elf, dynamic, header};
|
||||
|
||||
use super::{Artifact, runtime_pack};
|
||||
|
||||
const MAX_FILES: usize = 64;
|
||||
const MAX_BYTES: u64 = 512 * 1024 * 1024;
|
||||
|
||||
pub(super) fn library_name(name: &str) -> Result<(), String> {
|
||||
if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0', ':']) {
|
||||
return Err("compiler dependency must use a plain library name".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn os_library(name: &str) -> bool {
|
||||
// These are the declared GNU OS ABI, not host-discovered dependencies.
|
||||
matches!(
|
||||
name,
|
||||
"libc.so.6"
|
||||
| "libm.so.6"
|
||||
| "libdl.so.2"
|
||||
| "libpthread.so.0"
|
||||
| "librt.so.1"
|
||||
| "libresolv.so.2"
|
||||
| "libutil.so.1"
|
||||
) || name == loader_name()
|
||||
}
|
||||
|
||||
fn loader_name() -> &'static str {
|
||||
if cfg!(target_arch = "aarch64") {
|
||||
"ld-linux-aarch64.so.1"
|
||||
} else {
|
||||
"ld-linux-x86-64.so.2"
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn validate(root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
|
||||
validate_entries(root, artifacts, &["dever-core"])
|
||||
}
|
||||
|
||||
/// Both release components keep their complete non-OS loader closure in lib/.
|
||||
/// Their entry sets differ; the loader and file admission rules do not.
|
||||
pub(super) fn validate_entries(
|
||||
root: &Path,
|
||||
artifacts: &[Artifact],
|
||||
entries: &[&str],
|
||||
) -> Result<(), String> {
|
||||
runtime_pack::host_target()?;
|
||||
dever_sandbox::require_no_loader_preload()?;
|
||||
let mut files = BTreeMap::new();
|
||||
let mut total = 0u64;
|
||||
for artifact in artifacts {
|
||||
if !entries.contains(&artifact.path.as_str()) && !artifact.path.starts_with("lib/") {
|
||||
continue;
|
||||
}
|
||||
if let Some(name) = artifact.path.strip_prefix("lib/") {
|
||||
library_name(name)?;
|
||||
if os_library(name) {
|
||||
return Err("compiler pack must not override the GNU OS ABI libraries".into());
|
||||
}
|
||||
}
|
||||
total = total
|
||||
.checked_add(artifact.bytes)
|
||||
.ok_or("compiler closure size overflow")?;
|
||||
if total > MAX_BYTES || files.len() >= MAX_FILES {
|
||||
return Err("compiler loader closure exceeds its file or byte budget".into());
|
||||
}
|
||||
if files.insert(artifact.path.as_str(), artifact).is_some() {
|
||||
return Err("compiler loader closure has duplicate files".into());
|
||||
}
|
||||
}
|
||||
validate_library_directory(root, &files)?;
|
||||
let mut remaining = entries
|
||||
.iter()
|
||||
.map(|entry| (*entry).to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = remaining.pop() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let artifact = files.get(path.as_str()).ok_or_else(|| {
|
||||
format!("compiler dependency '{path}' is not supplied by the signed release")
|
||||
})?;
|
||||
let input = runtime_pack::checked_path(root, &path, true)?;
|
||||
let mut bytes = Vec::new();
|
||||
File::open(input)
|
||||
.and_then(|file| file.take(artifact.bytes + 1).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read compiler loader input: {error}"))?;
|
||||
if bytes.len() as u64 != artifact.bytes {
|
||||
return Err("compiler loader input size differs from its signed declaration".into());
|
||||
}
|
||||
remaining.extend(dependencies(
|
||||
&path,
|
||||
&bytes,
|
||||
entries.contains(&path.as_str()),
|
||||
)?);
|
||||
}
|
||||
if reached.len() != files.len() {
|
||||
return Err("compiler closure contains libraries unused by the core".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dependencies(path: &str, bytes: &[u8], executable: bool) -> Result<Vec<String>, String> {
|
||||
let elf = Elf::parse(bytes).map_err(|error| format!("invalid compiler ELF input: {error}"))?;
|
||||
let machine = if cfg!(target_arch = "aarch64") {
|
||||
header::EM_AARCH64
|
||||
} else {
|
||||
header::EM_X86_64
|
||||
};
|
||||
if !elf.is_64
|
||||
|| !elf.little_endian
|
||||
|| elf.header.e_machine != machine
|
||||
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
|
||||
{
|
||||
return Err("compiler ELF input has an incompatible target or object type".into());
|
||||
}
|
||||
// GNU auditing/config and ELF filter tags can load libraries outside
|
||||
// DT_NEEDED; they are not part of this release's loader contract.
|
||||
if elf.dynamic.as_ref().is_some_and(|table| {
|
||||
table.dyns.iter().any(|entry| {
|
||||
matches!(
|
||||
entry.d_tag,
|
||||
dynamic::DT_CONFIG
|
||||
| dynamic::DT_AUDIT
|
||||
| dynamic::DT_DEPAUDIT
|
||||
| 0x7fff_fffd
|
||||
| 0x7fff_ffff
|
||||
)
|
||||
})
|
||||
}) {
|
||||
return Err("compiler ELF input contains unsupported dynamic loader hooks".into());
|
||||
}
|
||||
if executable {
|
||||
let interpreter = if cfg!(target_arch = "aarch64") {
|
||||
"/lib/ld-linux-aarch64.so.1"
|
||||
} else {
|
||||
"/lib64/ld-linux-x86-64.so.2"
|
||||
};
|
||||
if elf.interpreter != Some(interpreter)
|
||||
|| elf.rpaths != ["$ORIGIN/lib"]
|
||||
|| !elf.runpaths.is_empty()
|
||||
{
|
||||
return Err("compiler requires the GNU loader and inherited RPATH=$ORIGIN/lib".into());
|
||||
}
|
||||
} else {
|
||||
if elf.header.e_type != header::ET_DYN || elf.soname != path.strip_prefix("lib/") {
|
||||
return Err("compiler library SONAME must match its signed filename".into());
|
||||
}
|
||||
// Upstream LLVM uses $ORIGIN/../lib, which resolves to this same
|
||||
// private lib directory. No search entry may reach the host tree.
|
||||
if elf
|
||||
.rpaths
|
||||
.iter()
|
||||
.chain(&elf.runpaths)
|
||||
.any(|path| !matches!(*path, "$ORIGIN" | "$ORIGIN/../lib"))
|
||||
{
|
||||
return Err("compiler library search path leaves its private library directory".into());
|
||||
}
|
||||
}
|
||||
let mut dependencies = Vec::new();
|
||||
for name in elf.libraries {
|
||||
library_name(name)?;
|
||||
if !os_library(name) {
|
||||
dependencies.push(format!("lib/{name}"));
|
||||
}
|
||||
}
|
||||
Ok(dependencies)
|
||||
}
|
||||
|
||||
fn validate_library_directory(
|
||||
root: &Path,
|
||||
files: &BTreeMap<&str, &Artifact>,
|
||||
) -> Result<(), String> {
|
||||
let mut expected = files
|
||||
.keys()
|
||||
.filter_map(|path| path.strip_prefix("lib/"))
|
||||
.collect::<BTreeSet<_>>();
|
||||
match std::fs::symlink_metadata(root.join("lib")) {
|
||||
Err(error) if expected.is_empty() && error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(());
|
||||
}
|
||||
Err(error) => return Err(format!("cannot inspect compiler lib directory: {error}")),
|
||||
Ok(_) => {}
|
||||
}
|
||||
let directory = runtime_pack::checked_path(root, "lib", false)?;
|
||||
for entry in std::fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let name = entry.file_name();
|
||||
if !entry
|
||||
.file_type()
|
||||
.map_err(|error| error.to_string())?
|
||||
.is_file()
|
||||
|| !name.to_str().is_some_and(|name| expected.remove(name))
|
||||
{
|
||||
return Err(
|
||||
"compiler lib directory contains an unsigned file, directory or link".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
if !expected.is_empty() {
|
||||
return Err("compiler lib directory is missing a signed library".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
use std::ffi::{OsStr, OsString};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use super::release::{Layout, MachineManager, Version, platform_contract};
|
||||
|
||||
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CommandResult {
|
||||
Exited(i32),
|
||||
Message(String),
|
||||
}
|
||||
|
||||
pub fn execute(executable: &Path, arguments: &[OsString]) -> Result<CommandResult, String> {
|
||||
if platform_contract(std::env::consts::OS).is_none() {
|
||||
return Err(format!(
|
||||
"Dever does not support platform '{}'",
|
||||
std::env::consts::OS
|
||||
));
|
||||
}
|
||||
let layout = Layout::from_launcher(executable)?;
|
||||
execute_in(&layout, arguments)
|
||||
}
|
||||
|
||||
pub fn execute_in(layout: &Layout, arguments: &[OsString]) -> Result<CommandResult, String> {
|
||||
layout.validate_machine_permissions()?;
|
||||
#[cfg(target_os = "linux")]
|
||||
super::bootstrap::require_no_pending(layout)?;
|
||||
let manager = MachineManager::new(layout.clone());
|
||||
match arguments {
|
||||
[command, requested] if command == "install" => {
|
||||
let requested = utf8(requested, "version")?;
|
||||
super::release_source::prepare(layout, requested)?;
|
||||
let version = manager.install_requested(requested)?;
|
||||
Ok(CommandResult::Message(format!("installed Dever {version}")))
|
||||
}
|
||||
[command] if command == "update" => {
|
||||
super::release_source::prepare(layout, "latest")?;
|
||||
let version = manager.update()?;
|
||||
Ok(CommandResult::Message(format!(
|
||||
"updated Dever and dever-language skill to {version}"
|
||||
)))
|
||||
}
|
||||
[command, version] if command == "use" => {
|
||||
let version = Version::parse(utf8(version, "version")?)?;
|
||||
manager.activate(&version)?;
|
||||
Ok(CommandResult::Message(format!("active Dever {version}")))
|
||||
}
|
||||
[command, version] if command == "uninstall" => {
|
||||
let version = Version::parse(utf8(version, "version")?)?;
|
||||
manager.uninstall(&version)?;
|
||||
Ok(CommandResult::Message(format!(
|
||||
"uninstalled Dever {version}"
|
||||
)))
|
||||
}
|
||||
[command] if command == "version" => {
|
||||
let version = manager
|
||||
.active_version()?
|
||||
.ok_or("no active Dever version; run 'dever install latest'")?;
|
||||
manager.resolve_core(&version)?;
|
||||
Ok(CommandResult::Message(version.to_string()))
|
||||
}
|
||||
[command, project] if command == "version" => {
|
||||
let root = PathBuf::from(project);
|
||||
let version = select_version(&manager, &root)?;
|
||||
Ok(CommandResult::Message(version.to_string()))
|
||||
}
|
||||
[cache, status] if cache == "cache" && status == "status" => {
|
||||
let status = crate::toolchain::cache_status(layout)?;
|
||||
Ok(CommandResult::Message(
|
||||
serde_json::to_string(&status).map_err(|error| error.to_string())?,
|
||||
))
|
||||
}
|
||||
[cache, clean] if cache == "cache" && clean == "clean" => {
|
||||
let status = crate::toolchain::cache_clean(
|
||||
layout,
|
||||
&manager.installed_versions()?.into_iter().collect(),
|
||||
)?;
|
||||
Ok(CommandResult::Message(
|
||||
serde_json::to_string(&status).map_err(|error| error.to_string())?,
|
||||
))
|
||||
}
|
||||
[skill, path] if skill == "skill" && path == "path" => Ok(CommandResult::Message(
|
||||
super::skill::path(&manager)?.display().to_string(),
|
||||
)),
|
||||
[skill, path, project] if skill == "skill" && path == "path" => {
|
||||
let version = select_version(&manager, Path::new(project))?;
|
||||
Ok(CommandResult::Message(
|
||||
manager.resolve_skill(&version)?.display().to_string(),
|
||||
))
|
||||
}
|
||||
[skill, install, directory] if skill == "skill" && install == "install" => {
|
||||
super::skill::install(&manager, Path::new(directory))?;
|
||||
Ok(CommandResult::Message(format!(
|
||||
"installed dever-language skill entry in '{}'",
|
||||
Path::new(directory).display()
|
||||
)))
|
||||
}
|
||||
[command, ..] if command == "new" => {
|
||||
let version = manager
|
||||
.active_version()?
|
||||
.ok_or("no active Dever version; run 'dever install latest'")?;
|
||||
dispatch_core(&manager, &version, arguments)
|
||||
}
|
||||
[help] if help == "--help" => Ok(CommandResult::Message(format!(
|
||||
"{USAGE}\n dever new <new-directory> [--markdown]\n dever skill path [project-root]\n dever skill install <new-directory>"
|
||||
))),
|
||||
[] => Err(USAGE.into()),
|
||||
_ => dispatch_project_command(&manager, arguments),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn project_version(project_root: &Path) -> Result<Option<Version>, String> {
|
||||
let path = project_root.join("config/setting.json");
|
||||
let text = fs::read_to_string(&path)
|
||||
.map_err(|error| format!("cannot read '{}': {error}", path.display()))?;
|
||||
let root = dever_runtime::wire::parse(&text)
|
||||
.map_err(|error| format!("invalid '{}': {error}", path.display()))?;
|
||||
let fields = root
|
||||
.object()
|
||||
.map_err(|_| format!("invalid '{}': expected a JSON object", path.display()))?;
|
||||
let Some(dever) = fields.get("dever") else {
|
||||
return Ok(None);
|
||||
};
|
||||
let fields = dever
|
||||
.fields(&["version"])
|
||||
.map_err(|error| format!("invalid '{}': dever: {error}", path.display()))?;
|
||||
let version = fields
|
||||
.get("version")
|
||||
.ok_or_else(|| format!("invalid '{}': dever.version is required", path.display()))?
|
||||
.text()
|
||||
.map_err(|error| format!("invalid '{}': dever.version: {error}", path.display()))?;
|
||||
Version::parse(version)
|
||||
.map(Some)
|
||||
.map_err(|error| format!("invalid '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn dispatch_project_command(
|
||||
manager: &MachineManager,
|
||||
arguments: &[OsString],
|
||||
) -> Result<CommandResult, String> {
|
||||
let root = project_root(arguments).ok_or_else(|| USAGE.to_owned())?;
|
||||
let version = select_version(manager, &root)?;
|
||||
dispatch_core(manager, &version, arguments)
|
||||
}
|
||||
|
||||
fn dispatch_core(
|
||||
manager: &MachineManager,
|
||||
version: &Version,
|
||||
arguments: &[OsString],
|
||||
) -> Result<CommandResult, String> {
|
||||
let core = manager.resolve_core(version)?;
|
||||
let status = Command::new(&core)
|
||||
.args(arguments)
|
||||
.env_clear()
|
||||
.status()
|
||||
.map_err(|error| format!("cannot run Dever {version} '{}': {error}", core.display()))?;
|
||||
Ok(CommandResult::Exited(status.code().unwrap_or(1)))
|
||||
}
|
||||
|
||||
fn select_version(manager: &MachineManager, project_root: &Path) -> Result<Version, String> {
|
||||
let version = match project_version(project_root)? {
|
||||
Some(version) => version,
|
||||
None => manager
|
||||
.active_version()?
|
||||
.ok_or("no active Dever version; run 'dever install latest'")?,
|
||||
};
|
||||
manager.resolve_core(&version)?;
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
fn project_root(arguments: &[OsString]) -> Option<PathBuf> {
|
||||
match arguments {
|
||||
[command, _operation, root, ..] if command == "lib" || command == "package" => {
|
||||
Some(root.into())
|
||||
}
|
||||
[command, root, ..]
|
||||
if matches!(
|
||||
command.to_str(),
|
||||
Some("check" | "api" | "fmt" | "test" | "run" | "build" | "clean")
|
||||
) =>
|
||||
{
|
||||
Some(root.into())
|
||||
}
|
||||
[tenant, operation, root, ..]
|
||||
if tenant == "tenant" && matches!(operation.to_str(), Some("migrate" | "owner")) =>
|
||||
{
|
||||
Some(root.into())
|
||||
}
|
||||
[tenant, component, _operation, root, ..]
|
||||
if tenant == "tenant" && component == "component" =>
|
||||
{
|
||||
Some(root.into())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn utf8<'a>(value: &'a OsStr, label: &str) -> Result<&'a str, String> {
|
||||
value
|
||||
.to_str()
|
||||
.ok_or_else(|| format!("{label} must be UTF-8"))
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#[cfg(target_os = "linux")]
|
||||
pub mod bootstrap;
|
||||
mod cache;
|
||||
pub mod compilation;
|
||||
mod core_libraries;
|
||||
mod launcher;
|
||||
pub mod packaging;
|
||||
mod release;
|
||||
mod release_source;
|
||||
pub mod runtime_pack;
|
||||
mod service;
|
||||
mod skill;
|
||||
mod target;
|
||||
|
||||
pub use target::BuildTarget;
|
||||
|
||||
pub use cache::{
|
||||
ArtifactReceipt, BuildIdentity, CacheEntry, CacheStatus, CacheStore, CachedArtifact,
|
||||
};
|
||||
pub use launcher::{CommandResult, execute, execute_in, project_version};
|
||||
pub use release::{
|
||||
Artifact, DownloadedReleases, Layout, MachineManager, PlatformContract, ReleaseManifest,
|
||||
Version, platform_contract, platform_identity, sha256_file,
|
||||
};
|
||||
pub use service::{artifact_get, artifact_put, cache_clean, cache_status, compile, serve};
|
||||
@@ -0,0 +1,520 @@
|
||||
//! Private authoring of signed native releases from explicitly prepared inputs.
|
||||
//! Compilation and tool discovery do not belong to this entry point.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, File};
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use ring::signature::Ed25519KeyPair;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::runtime_pack::{self, Manifest, Profile, Profiles, RuntimeArtifact};
|
||||
use super::{Artifact, BuildTarget, ReleaseManifest, Version, sha256_file};
|
||||
|
||||
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
mod bootstrap;
|
||||
mod builds;
|
||||
mod ecosystems;
|
||||
mod sandbox;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Settings {
|
||||
format: String,
|
||||
version: String,
|
||||
target: String,
|
||||
signing_key: String,
|
||||
core: Input,
|
||||
core_libraries: Vec<LinkInput>,
|
||||
skill: Vec<LinkInput>,
|
||||
#[serde(default)]
|
||||
bootstrap: Option<bootstrap::Inputs>,
|
||||
profiles: ArchiveInputs,
|
||||
start: Vec<LinkInput>,
|
||||
libraries: Vec<LinkInput>,
|
||||
end: Vec<LinkInput>,
|
||||
#[serde(default)]
|
||||
native_targets: BTreeMap<BuildTarget, NativeInputs>,
|
||||
#[serde(default)]
|
||||
runtimes: ecosystems::RuntimeInputs,
|
||||
#[serde(default)]
|
||||
sandbox: Option<Vec<LinkInput>>,
|
||||
#[serde(default)]
|
||||
builds: builds::Inputs,
|
||||
#[serde(default)]
|
||||
ecosystem_targets: BTreeMap<BuildTarget, ecosystems::RuntimeInputs>,
|
||||
#[serde(default)]
|
||||
build_targets: BTreeMap<BuildTarget, builds::Inputs>,
|
||||
#[serde(default)]
|
||||
sandbox_targets: BTreeMap<BuildTarget, Vec<LinkInput>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct NativeInputs {
|
||||
profiles: ArchiveInputs,
|
||||
start: Vec<LinkInput>,
|
||||
libraries: Vec<LinkInput>,
|
||||
end: Vec<LinkInput>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Input {
|
||||
source: String,
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct LinkInput {
|
||||
source: String,
|
||||
path: String,
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ArchiveInputs {
|
||||
base: Input,
|
||||
sqlite: Input,
|
||||
postgres: Input,
|
||||
both: Input,
|
||||
}
|
||||
|
||||
struct SigningKey {
|
||||
pair: Ed25519KeyPair,
|
||||
sha256: String,
|
||||
}
|
||||
|
||||
struct PayloadWriter<'a> {
|
||||
root: &'a Path,
|
||||
staging: &'a Path,
|
||||
key: &'a SigningKey,
|
||||
destinations: BTreeSet<String>,
|
||||
}
|
||||
|
||||
/// Create a new release directory. Inputs come only from author-root/config/setting.json.
|
||||
pub fn create(author_root: &Path, output: &Path) -> Result<ReleaseManifest, String> {
|
||||
let target = runtime_pack::host_target()?;
|
||||
let metadata = fs::symlink_metadata(author_root)
|
||||
.map_err(|error| format!("cannot inspect author root: {error}"))?;
|
||||
if !metadata.is_dir() || metadata.file_type().is_symlink() {
|
||||
return Err("author root must be a real directory".into());
|
||||
}
|
||||
let root = fs::canonicalize(author_root)
|
||||
.map_err(|error| format!("cannot resolve author root: {error}"))?;
|
||||
let config = runtime_pack::checked_path(&root, "config/setting.json", true)?;
|
||||
let bytes = read_bounded(&config, 16 * 1024 * 1024, "author configuration")?;
|
||||
// Do not echo configuration values: they include the private signing-key location.
|
||||
let settings: Settings = serde_json::from_slice(&bytes).map_err(|error| {
|
||||
format!(
|
||||
"invalid author configuration at line {}, column {}",
|
||||
error.line(),
|
||||
error.column()
|
||||
)
|
||||
})?;
|
||||
if settings.format != "dever-native-release-input-v1" || settings.target != target {
|
||||
return Err("unsupported native release input format or target".into());
|
||||
}
|
||||
let version = Version::parse(&settings.version)?;
|
||||
let key = read_signing_key(&root, &settings.signing_key)?;
|
||||
let output = output_path(output)?;
|
||||
let staging = create_staging(output.parent().expect("resolved output parent"))?;
|
||||
let result = assemble(&root, &staging, settings, version, &key)
|
||||
.and_then(|manifest| publish(&staging, &output).map(|()| manifest));
|
||||
if result.is_err()
|
||||
&& let Err(error) = fs::remove_dir_all(&staging)
|
||||
{
|
||||
return Err(format!(
|
||||
"{}; cannot remove owned release staging: {error}",
|
||||
result.expect_err("failed release creation")
|
||||
));
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn assemble(
|
||||
root: &Path,
|
||||
staging: &Path,
|
||||
settings: Settings,
|
||||
version: Version,
|
||||
key: &SigningKey,
|
||||
) -> Result<ReleaseManifest, String> {
|
||||
let host = BuildTarget::host()?;
|
||||
let mut writer = PayloadWriter {
|
||||
root,
|
||||
staging,
|
||||
key,
|
||||
destinations: BTreeSet::from(["manifest.json".to_owned(), "manifest.sig".to_owned()]),
|
||||
};
|
||||
let mut artifacts = Vec::new();
|
||||
artifacts.push(writer.copy(
|
||||
&settings.core.source,
|
||||
"dever-core",
|
||||
&settings.core.sha256,
|
||||
true,
|
||||
)?);
|
||||
for input in &settings.core_libraries {
|
||||
super::core_libraries::library_name(&input.path)?;
|
||||
artifacts.push(writer.copy(
|
||||
&input.source,
|
||||
&format!("lib/{}", input.path),
|
||||
&input.sha256,
|
||||
false,
|
||||
)?);
|
||||
}
|
||||
super::core_libraries::validate(staging, &artifacts)?;
|
||||
if settings.skill.len() > 256 || !settings.skill.iter().any(|input| input.path == "SKILL.md") {
|
||||
return Err("release skill requires SKILL.md and at most 256 files".into());
|
||||
}
|
||||
let mut skill_bytes = 0_u64;
|
||||
for input in settings.skill {
|
||||
let source = writer.source(&input.source, &input.path, &input.sha256)?;
|
||||
let bytes = fs::metadata(source)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len();
|
||||
skill_bytes = skill_bytes
|
||||
.checked_add(bytes)
|
||||
.ok_or("release skill size overflow")?;
|
||||
if bytes > 1024 * 1024 || skill_bytes > 8 * 1024 * 1024 {
|
||||
return Err("release skill exceeds its 1 MiB file or 8 MiB total limit".into());
|
||||
}
|
||||
artifacts.push(writer.copy(
|
||||
&input.source,
|
||||
&format!("skills/dever-language/{}", input.path),
|
||||
&input.sha256,
|
||||
false,
|
||||
)?);
|
||||
}
|
||||
if let Some(input) = settings.bootstrap {
|
||||
artifacts.extend(bootstrap::assemble(&mut writer, input)?);
|
||||
}
|
||||
artifacts.extend(assemble_native(
|
||||
&mut writer,
|
||||
&version,
|
||||
host,
|
||||
NativeInputs {
|
||||
profiles: settings.profiles,
|
||||
start: settings.start,
|
||||
libraries: settings.libraries,
|
||||
end: settings.end,
|
||||
},
|
||||
)?);
|
||||
for (target, inputs) in settings.native_targets {
|
||||
artifacts.extend(assemble_native(&mut writer, &version, target, inputs)?);
|
||||
}
|
||||
for (ecosystem, input) in settings.runtimes.entries() {
|
||||
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, host)?);
|
||||
}
|
||||
for (target, inputs) in settings.ecosystem_targets {
|
||||
for (ecosystem, input) in inputs.entries() {
|
||||
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, target)?);
|
||||
}
|
||||
}
|
||||
if let Some(inputs) = settings.sandbox {
|
||||
artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?);
|
||||
}
|
||||
for (target, inputs) in settings.sandbox_targets {
|
||||
artifacts.extend(sandbox::assemble(&mut writer, inputs, target)?);
|
||||
}
|
||||
for (ecosystem, input) in settings.builds.entries() {
|
||||
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, host)?);
|
||||
}
|
||||
for (target, inputs) in settings.build_targets {
|
||||
for (ecosystem, input) in inputs.entries() {
|
||||
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, target)?);
|
||||
}
|
||||
}
|
||||
artifacts.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
let manifest = ReleaseManifest::new(version, artifacts);
|
||||
let bytes = write_metadata(&staging.join("manifest.json"), &manifest)?;
|
||||
let signature = key.pair.sign(&bytes);
|
||||
let hex: String = signature
|
||||
.as_ref()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
fs::write(staging.join("manifest.sig"), format!("{hex}\n"))
|
||||
.map_err(|error| format!("cannot write release signature: {error}"))?;
|
||||
set_mode(&staging.join("manifest.sig"), false)?;
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
fn assemble_native(
|
||||
writer: &mut PayloadWriter<'_>,
|
||||
version: &Version,
|
||||
target: BuildTarget,
|
||||
inputs: NativeInputs,
|
||||
) -> Result<Vec<Artifact>, String> {
|
||||
let runtime_prefix = format!("runtime/{}", target.platform());
|
||||
let manifest_path = writer.reserve(&format!("{runtime_prefix}/manifest.json"))?;
|
||||
let archive_inputs = inputs.profiles;
|
||||
let mut files = Vec::new();
|
||||
for (name, input) in [
|
||||
("base", archive_inputs.base),
|
||||
("sqlite", archive_inputs.sqlite),
|
||||
("postgres", archive_inputs.postgres),
|
||||
("both", archive_inputs.both),
|
||||
] {
|
||||
let path = format!("archives/{name}.a");
|
||||
let artifact = writer.copy(
|
||||
&input.source,
|
||||
&format!("{runtime_prefix}/{path}"),
|
||||
&input.sha256,
|
||||
false,
|
||||
)?;
|
||||
files.push(RuntimeArtifact {
|
||||
path,
|
||||
bytes: artifact.bytes,
|
||||
sha256: artifact.sha256,
|
||||
});
|
||||
}
|
||||
for input in inputs
|
||||
.start
|
||||
.iter()
|
||||
.chain(&inputs.libraries)
|
||||
.chain(&inputs.end)
|
||||
{
|
||||
runtime_pack::relative_path(&input.path)?;
|
||||
let artifact = writer.copy(
|
||||
&input.source,
|
||||
&format!("{runtime_prefix}/{}", input.path),
|
||||
&input.sha256,
|
||||
false,
|
||||
)?;
|
||||
files.push(RuntimeArtifact {
|
||||
path: input.path.clone(),
|
||||
bytes: artifact.bytes,
|
||||
sha256: artifact.sha256,
|
||||
});
|
||||
}
|
||||
files.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
let link_paths = |inputs: &[LinkInput]| inputs.iter().map(|input| input.path.clone()).collect();
|
||||
let profile = |name| {
|
||||
Some(Profile {
|
||||
start: link_paths(&inputs.start),
|
||||
runtime: format!("archives/{name}.a"),
|
||||
libraries: link_paths(&inputs.libraries),
|
||||
end: link_paths(&inputs.end),
|
||||
})
|
||||
};
|
||||
let runtime = Manifest {
|
||||
format: "dever-native-runtime-v1".into(),
|
||||
compiler_version: version.to_string(),
|
||||
abi: 1,
|
||||
platform: target.platform().into(),
|
||||
target: target.triple().into(),
|
||||
profiles: Profiles {
|
||||
base: profile("base"),
|
||||
sqlite: profile("sqlite"),
|
||||
postgres: profile("postgres"),
|
||||
both: profile("both"),
|
||||
},
|
||||
files,
|
||||
};
|
||||
write_metadata(&manifest_path, &runtime)?;
|
||||
Ok(
|
||||
runtime_pack::validate(writer.staging, version.as_str(), target)?
|
||||
.into_iter()
|
||||
.map(|artifact| Artifact {
|
||||
path: artifact.path,
|
||||
bytes: artifact.bytes,
|
||||
sha256: artifact.sha256,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
impl PayloadWriter<'_> {
|
||||
fn reserve(&mut self, destination: &str) -> Result<PathBuf, String> {
|
||||
runtime_pack::relative_path(destination)?;
|
||||
if self.destinations.iter().any(|path| {
|
||||
path == destination
|
||||
|| path.starts_with(&format!("{destination}/"))
|
||||
|| destination.starts_with(&format!("{path}/"))
|
||||
}) {
|
||||
return Err("duplicate release destination or file/directory conflict".into());
|
||||
}
|
||||
self.destinations.insert(destination.to_owned());
|
||||
let output = self.staging.join(destination);
|
||||
create_directories(self.staging, output.parent().expect("payload parent"))?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn source(&self, source: &str, destination: &str, digest: &str) -> Result<PathBuf, String> {
|
||||
if digest.len() != 64
|
||||
|| !digest
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err("payload requires a lowercase SHA-256 digest".into());
|
||||
}
|
||||
if digest == self.key.sha256 {
|
||||
return Err("signing key cannot be included as a release payload".into());
|
||||
}
|
||||
let source = runtime_pack::checked_path(self.root, source, true)
|
||||
.map_err(|_| format!("invalid source for release payload '{destination}'"))?;
|
||||
if sha256_file(&source).map_err(|_| "cannot hash source payload")? != digest {
|
||||
return Err(format!(
|
||||
"release payload '{destination}' failed SHA-256 verification"
|
||||
));
|
||||
}
|
||||
Ok(source)
|
||||
}
|
||||
|
||||
fn copy(
|
||||
&mut self,
|
||||
source: &str,
|
||||
destination: &str,
|
||||
digest: &str,
|
||||
executable: bool,
|
||||
) -> Result<Artifact, String> {
|
||||
let output = self.reserve(destination)?;
|
||||
let source = self.source(source, destination, digest)?;
|
||||
fs::copy(&source, &output)
|
||||
.map_err(|error| format!("cannot copy release payload '{destination}': {error}"))?;
|
||||
if sha256_file(&output)? != digest {
|
||||
return Err(format!(
|
||||
"copied release payload '{destination}' failed SHA-256 verification"
|
||||
));
|
||||
}
|
||||
set_mode(&output, executable)?;
|
||||
Ok(Artifact {
|
||||
path: destination.to_owned(),
|
||||
bytes: fs::metadata(output)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len(),
|
||||
sha256: digest.to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn read_signing_key(root: &Path, relative: &str) -> Result<SigningKey, String> {
|
||||
let path = runtime_pack::checked_path(root, relative, true)
|
||||
.map_err(|_| "signing key must be a real file within the author root")?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let metadata = fs::metadata(&path).map_err(|_| "cannot inspect signing key")?;
|
||||
if metadata.permissions().mode() & 0o077 != 0 {
|
||||
return Err("signing key permissions must be private to its owner".into());
|
||||
}
|
||||
}
|
||||
let bytes = read_bounded(&path, 4096, "signing key")?;
|
||||
let pair = Ed25519KeyPair::from_pkcs8(&bytes)
|
||||
.map_err(|_| "signing key is not a valid Ed25519 PKCS#8 key")?;
|
||||
Ok(SigningKey {
|
||||
pair,
|
||||
sha256: format!("{:x}", Sha256::digest(bytes)),
|
||||
})
|
||||
}
|
||||
|
||||
fn read_bounded(path: &Path, limit: u64, label: &str) -> Result<Vec<u8>, String> {
|
||||
let mut bytes = Vec::new();
|
||||
File::open(path)
|
||||
.and_then(|file| file.take(limit + 1).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read {label}: {error}"))?;
|
||||
if bytes.len() as u64 > limit {
|
||||
return Err(format!("{label} exceeds its size limit"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn write_metadata(path: &Path, value: &impl serde::Serialize) -> Result<Vec<u8>, String> {
|
||||
let mut bytes = serde_json::to_vec_pretty(value).map_err(|error| error.to_string())?;
|
||||
bytes.push(b'\n');
|
||||
fs::write(path, &bytes).map_err(|error| format!("cannot write release metadata: {error}"))?;
|
||||
set_mode(path, false)?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn output_path(output: &Path) -> Result<PathBuf, String> {
|
||||
let name = output
|
||||
.file_name()
|
||||
.ok_or("release output requires a new directory name")?;
|
||||
let parent = output
|
||||
.parent()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new("."));
|
||||
let parent = fs::canonicalize(parent)
|
||||
.map_err(|error| format!("cannot resolve release output parent: {error}"))?;
|
||||
let output = parent.join(name);
|
||||
match fs::symlink_metadata(&output) {
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(output),
|
||||
Ok(_) => Err("release output already exists".into()),
|
||||
Err(error) => Err(format!("cannot inspect release output: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_staging(parent: &Path) -> Result<PathBuf, String> {
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
loop {
|
||||
let path = parent.join(format!(
|
||||
".native-release-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_STAGING.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
match builder.create(&path) {
|
||||
Ok(()) => return Ok(path),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(format!("cannot create release staging: {error}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn create_directories(root: &Path, directory: &Path) -> Result<(), String> {
|
||||
let mut current = root.to_owned();
|
||||
for component in directory
|
||||
.strip_prefix(root)
|
||||
.expect("owned staging path")
|
||||
.components()
|
||||
{
|
||||
current.push(component);
|
||||
if !current.exists() {
|
||||
fs::create_dir(¤t)
|
||||
.map_err(|error| format!("cannot create payload directory: {error}"))?;
|
||||
set_mode(¤t, true)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_mode(path: &Path, executable: bool) -> Result<(), String> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(
|
||||
path,
|
||||
fs::Permissions::from_mode(if executable { 0o755 } else { 0o644 }),
|
||||
)
|
||||
.map_err(|error| format!("cannot set release permissions: {error}"))?;
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = (path, executable);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn publish(staging: &Path, output: &Path) -> Result<(), String> {
|
||||
set_mode(staging, true)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{CWD, RenameFlags, renameat_with};
|
||||
renameat_with(CWD, staging, CWD, output, RenameFlags::NOREPLACE)
|
||||
.map_err(|error| format!("cannot publish new release directory: {error}"))
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = output;
|
||||
Err("native release publication is not available for this platform".into())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
//! The stable Linux entry points share a small, separately checked loader closure.
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{Artifact, Input, LinkInput, PayloadWriter};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Inputs {
|
||||
launcher: Input,
|
||||
daemon: Input,
|
||||
libraries: Vec<LinkInput>,
|
||||
}
|
||||
|
||||
pub(super) fn assemble(
|
||||
writer: &mut PayloadWriter<'_>,
|
||||
input: Inputs,
|
||||
) -> Result<Vec<Artifact>, String> {
|
||||
if !cfg!(target_os = "linux") {
|
||||
return Err("bootstrap authoring currently requires Linux".into());
|
||||
}
|
||||
let mut artifacts = Vec::new();
|
||||
for (name, input) in [("dever", input.launcher), ("deverd", input.daemon)] {
|
||||
artifacts.push(writer.copy(
|
||||
&input.source,
|
||||
&format!("bootstrap/{name}"),
|
||||
&input.sha256,
|
||||
true,
|
||||
)?);
|
||||
}
|
||||
for library in input.libraries {
|
||||
crate::toolchain::core_libraries::library_name(&library.path)?;
|
||||
artifacts.push(writer.copy(
|
||||
&library.source,
|
||||
&format!("bootstrap/lib/{}", library.path),
|
||||
&library.sha256,
|
||||
false,
|
||||
)?);
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use sha2::{Digest, Sha256};
|
||||
for (path, bytes) in [
|
||||
(
|
||||
"bootstrap/deverd.service",
|
||||
crate::toolchain::bootstrap::SERVICE.as_bytes(),
|
||||
),
|
||||
(
|
||||
"bootstrap/dever-sandbox",
|
||||
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
|
||||
),
|
||||
] {
|
||||
let destination = writer.reserve(path)?;
|
||||
std::fs::write(&destination, bytes).map_err(|error| error.to_string())?;
|
||||
super::set_mode(&destination, false)?;
|
||||
artifacts.push(Artifact {
|
||||
path: path.into(),
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: format!("{:x}", Sha256::digest(bytes)),
|
||||
});
|
||||
}
|
||||
crate::toolchain::bootstrap::validate_payload(writer.staging, &artifacts, true)?;
|
||||
}
|
||||
Ok(artifacts)
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
//! Assemble source-build tools against already prepared runtime identities.
|
||||
|
||||
use flate2::{Compression, GzBuilder};
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fs::{self, File};
|
||||
|
||||
use super::{Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, write_metadata};
|
||||
use crate::libs::build::pack::{self, Descriptor, Manifest};
|
||||
use crate::libs::{Ecosystem, InstalledRegistryPack};
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Inputs {
|
||||
pip: Option<Input>,
|
||||
npm: Option<Input>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Input {
|
||||
executables: Vec<String>,
|
||||
files: Vec<LinkInput>,
|
||||
}
|
||||
|
||||
impl Inputs {
|
||||
pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, Input)> {
|
||||
[(Ecosystem::Pip, self.pip), (Ecosystem::Npm, self.npm)]
|
||||
.into_iter()
|
||||
.filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input)))
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn assemble(
|
||||
writer: &mut PayloadWriter<'_>,
|
||||
ecosystem: Ecosystem,
|
||||
input: Input,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<Artifact>, String> {
|
||||
let target = target.platform().to_owned();
|
||||
let runtime = |ecosystem: &Ecosystem| -> Result<InstalledRegistryPack, String> {
|
||||
let path = writer.staging.join(format!(
|
||||
"runtime/{}/{target}/manifest.json",
|
||||
ecosystem.as_str()
|
||||
));
|
||||
let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded(
|
||||
&path,
|
||||
1024 * 1024,
|
||||
"build runtime descriptor",
|
||||
)?)
|
||||
.map_err(|error| format!("invalid build runtime descriptor: {error}"))?;
|
||||
manifest.validate(ecosystem, &target)?;
|
||||
Ok(manifest)
|
||||
};
|
||||
let runtime = runtime(&ecosystem)?;
|
||||
let python = if ecosystem == Ecosystem::Npm {
|
||||
let path = writer
|
||||
.staging
|
||||
.join(format!("runtime/pip/{target}/manifest.json"));
|
||||
let manifest: InstalledRegistryPack = serde_json::from_slice(&read_bounded(
|
||||
&path,
|
||||
1024 * 1024,
|
||||
"auxiliary Python runtime descriptor",
|
||||
)?)
|
||||
.map_err(|error| format!("invalid auxiliary Python runtime: {error}"))?;
|
||||
manifest.validate(&Ecosystem::Pip, &target)?;
|
||||
Some(manifest)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut descriptor = Descriptor {
|
||||
format: "dever-registry-build-v1".into(),
|
||||
ecosystem: ecosystem.clone(),
|
||||
target: target.clone(),
|
||||
runtime_sha256: runtime.runtime.pack.sha256.clone(),
|
||||
python_runtime_sha256: python
|
||||
.as_ref()
|
||||
.map(|runtime| runtime.runtime.pack.sha256.clone()),
|
||||
sha256: "0".repeat(64),
|
||||
};
|
||||
descriptor.validate(
|
||||
&runtime.runtime,
|
||||
python.as_ref().map(|manifest| &manifest.runtime),
|
||||
)?;
|
||||
let manifest = Manifest {
|
||||
format: "dever-lib-build-v1".into(),
|
||||
ecosystem: ecosystem.clone(),
|
||||
target: target.clone(),
|
||||
runtime_sha256: descriptor.runtime_sha256.clone(),
|
||||
python_runtime_sha256: descriptor.python_runtime_sha256.clone(),
|
||||
executables: input.executables,
|
||||
};
|
||||
if input.files.len() >= pack::MAX_FILES {
|
||||
return Err("build pack has too many leaves".into());
|
||||
}
|
||||
let metadata = serde_json::to_vec(&manifest).map_err(|error| error.to_string())?;
|
||||
let mut remaining = pack::MAX_EXPANDED_BYTES
|
||||
.checked_sub(metadata.len())
|
||||
.ok_or("build manifest is too large")? as u64;
|
||||
let mut files = vec![("dever-build.json".into(), metadata)];
|
||||
for input in input.files {
|
||||
let path = writer.source(&input.source, &input.path, &input.sha256)?;
|
||||
let bytes = read_bounded(&path, remaining, "build tool leaf")?;
|
||||
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
|
||||
return Err("build tool changed during packaging".into());
|
||||
}
|
||||
remaining -= bytes.len() as u64;
|
||||
files.push((input.path, bytes));
|
||||
}
|
||||
files.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
manifest.validate(&files, &descriptor)?;
|
||||
let mut assets = Vec::new();
|
||||
for directory in ["bin", "lib"] {
|
||||
for entry in fs::read_dir(writer.staging.join(format!("sandbox/{target}/{directory}")))
|
||||
.map_err(|error| format!("build pack requires prepared sandbox assets: {error}"))?
|
||||
{
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let name = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "sandbox asset name is not UTF-8")?;
|
||||
assets.push((
|
||||
format!("{directory}/{name}"),
|
||||
read_bounded(&entry.path(), 32 * 1024 * 1024, "sandbox build asset")?,
|
||||
));
|
||||
}
|
||||
}
|
||||
dever_sandbox::validate_assets_for_target(&assets, &target)?;
|
||||
pack::validate_tools(&manifest, &files, &assets)?;
|
||||
let prefix = format!("build/{}/{target}", ecosystem.as_str());
|
||||
let archive_name = format!("{prefix}/build.pack");
|
||||
let path = writer.reserve(&archive_name)?;
|
||||
let encoder = GzBuilder::new().mtime(0).operating_system(255).write(
|
||||
File::create(&path).map_err(|error| error.to_string())?,
|
||||
Compression::default(),
|
||||
);
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for (path, bytes) in files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(if manifest.executables.contains(&path) {
|
||||
0o755
|
||||
} else {
|
||||
0o644
|
||||
});
|
||||
header.set_uid(0);
|
||||
header.set_gid(0);
|
||||
header.set_mtime(0);
|
||||
header.set_cksum();
|
||||
archive
|
||||
.append_data(&mut header, path, bytes.as_slice())
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
archive
|
||||
.into_inner()
|
||||
.and_then(|encoder| encoder.finish())
|
||||
.map_err(|error| error.to_string())?;
|
||||
set_mode(&path, false)?;
|
||||
let bytes = fs::metadata(&path)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len();
|
||||
if bytes > pack::MAX_PACK_BYTES as u64 {
|
||||
return Err("compressed build pack exceeds 256 MiB".into());
|
||||
}
|
||||
descriptor.sha256 = super::sha256_file(&path)?;
|
||||
let descriptor_name = format!("{prefix}/manifest.json");
|
||||
let metadata = write_metadata(&writer.reserve(&descriptor_name)?, &descriptor)?;
|
||||
Ok(vec![
|
||||
Artifact {
|
||||
path: archive_name,
|
||||
bytes,
|
||||
sha256: descriptor.sha256,
|
||||
},
|
||||
Artifact {
|
||||
path: descriptor_name,
|
||||
bytes: metadata.len() as u64,
|
||||
sha256: format!("{:x}", Sha256::digest(&metadata)),
|
||||
},
|
||||
])
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
//! Deterministic ecosystem packs inside the same signed native release.
|
||||
|
||||
use std::fs::{self, File};
|
||||
|
||||
use flate2::{Compression, GzBuilder};
|
||||
use pep508_rs::MarkerEnvironment;
|
||||
use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{
|
||||
Artifact, LinkInput, PayloadWriter, read_bounded, set_mode, sha256_file, write_metadata,
|
||||
};
|
||||
use crate::libs::{Ecosystem, InstalledRegistryPack, RegistryRuntime, RuntimePack};
|
||||
use crate::workers::{MAX_TREE_BYTES, MAX_TREE_FILES, runtime};
|
||||
|
||||
#[derive(Default, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct RuntimeInputs {
|
||||
pip: Option<RuntimeInput>,
|
||||
npm: Option<RuntimeInput>,
|
||||
go: Option<RuntimeInput>,
|
||||
}
|
||||
|
||||
impl RuntimeInputs {
|
||||
pub(super) fn entries(self) -> impl Iterator<Item = (Ecosystem, RuntimeInput)> {
|
||||
[
|
||||
(Ecosystem::Pip, self.pip),
|
||||
(Ecosystem::Npm, self.npm),
|
||||
(Ecosystem::Go, self.go),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(ecosystem, input)| input.map(|input| (ecosystem, input)))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct RuntimeInput {
|
||||
name: String,
|
||||
version: String,
|
||||
python_markers: Option<MarkerEnvironment>,
|
||||
#[serde(default)]
|
||||
python_wheel_tags: Vec<String>,
|
||||
npm_libc: Option<String>,
|
||||
files: Vec<LinkInput>,
|
||||
}
|
||||
|
||||
pub(super) fn assemble(
|
||||
writer: &mut PayloadWriter<'_>,
|
||||
ecosystem: Ecosystem,
|
||||
input: RuntimeInput,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<Artifact>, String> {
|
||||
let target = target.platform().to_owned();
|
||||
let mut metadata = InstalledRegistryPack {
|
||||
format: "dever-registry-runtime-v1".into(),
|
||||
runtime: RegistryRuntime {
|
||||
pack: RuntimePack {
|
||||
name: input.name,
|
||||
version: input.version,
|
||||
sha256: "0".repeat(64),
|
||||
},
|
||||
target: target.clone(),
|
||||
python_markers: input.python_markers,
|
||||
python_wheel_tags: input.python_wheel_tags,
|
||||
npm_libc: input.npm_libc,
|
||||
},
|
||||
};
|
||||
metadata.validate(&ecosystem, &target)?;
|
||||
if input.files.len() > MAX_TREE_FILES {
|
||||
return Err("managed runtime pack has too many files".into());
|
||||
}
|
||||
let mut files = Vec::with_capacity(input.files.len());
|
||||
let mut remaining = MAX_TREE_BYTES as u64;
|
||||
for input in input.files {
|
||||
super::runtime_pack::relative_path(&input.path)?;
|
||||
// A runtime archive is built from its leaf files, never another completed pack.
|
||||
if input.path == "runtime.pack" {
|
||||
return Err("runtime pack cannot include another runtime.pack".into());
|
||||
}
|
||||
let source = writer.source(&input.source, &input.path, &input.sha256)?;
|
||||
let bytes = read_bounded(&source, remaining, "runtime payload")?;
|
||||
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
|
||||
return Err("runtime payload changed during packaging".into());
|
||||
}
|
||||
remaining -= bytes.len() as u64;
|
||||
files.push((input.path, bytes));
|
||||
}
|
||||
files.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
let manifest = runtime::validate(&files, &ecosystem, &target)?;
|
||||
if manifest.python_wheel_tags != metadata.runtime.python_wheel_tags
|
||||
|| manifest.python_markers != metadata.runtime.python_markers
|
||||
{
|
||||
return Err(
|
||||
"runtime pack Python metadata differs from its signed registry descriptor".into(),
|
||||
);
|
||||
}
|
||||
let prefix = format!("runtime/{}/{target}", ecosystem.as_str());
|
||||
let archive_name = format!("{prefix}/runtime.pack");
|
||||
let archive_path = writer.reserve(&archive_name)?;
|
||||
let output = File::create(&archive_path)
|
||||
.map_err(|error| format!("cannot create runtime pack: {error}"))?;
|
||||
let encoder = GzBuilder::new()
|
||||
.mtime(0)
|
||||
.operating_system(255)
|
||||
.write(output, Compression::default());
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for (path, bytes) in &files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(if manifest.executable(path) {
|
||||
0o755
|
||||
} else {
|
||||
0o644
|
||||
});
|
||||
header.set_uid(0);
|
||||
header.set_gid(0);
|
||||
header.set_mtime(0);
|
||||
header.set_cksum();
|
||||
archive
|
||||
.append_data(&mut header, path, bytes.as_slice())
|
||||
.map_err(|error| format!("cannot write runtime pack entry: {error}"))?;
|
||||
}
|
||||
archive
|
||||
.into_inner()
|
||||
.and_then(|encoder| encoder.finish())
|
||||
.map_err(|error| format!("cannot finish runtime pack: {error}"))?;
|
||||
set_mode(&archive_path, false)?;
|
||||
let size = fs::metadata(&archive_path)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len();
|
||||
if size > MAX_TREE_BYTES as u64 {
|
||||
return Err("compressed runtime pack exceeds 256 MiB".into());
|
||||
}
|
||||
metadata.runtime.pack.sha256 = sha256_file(&archive_path)?;
|
||||
let pack = Artifact {
|
||||
path: archive_name,
|
||||
bytes: size,
|
||||
sha256: metadata.runtime.pack.sha256.clone(),
|
||||
};
|
||||
let metadata_name = format!("{prefix}/manifest.json");
|
||||
let metadata_path = writer.reserve(&metadata_name)?;
|
||||
let bytes = write_metadata(&metadata_path, &metadata)?;
|
||||
Ok(vec![
|
||||
pack,
|
||||
Artifact {
|
||||
path: metadata_name,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: format!("{:x}", Sha256::digest(bytes)),
|
||||
},
|
||||
])
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
//! Sandbox tools and OS libraries are authenticated by the existing release.
|
||||
|
||||
use super::{Artifact, LinkInput, PayloadWriter, read_bounded};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
pub(super) fn assemble(
|
||||
writer: &mut PayloadWriter<'_>,
|
||||
inputs: Vec<LinkInput>,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<Artifact>, String> {
|
||||
if inputs.len() > 128 {
|
||||
return Err("sandbox pack contains too many files".into());
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
let mut remaining = 32 * 1024 * 1024;
|
||||
for input in inputs {
|
||||
super::runtime_pack::relative_path(&input.path)?;
|
||||
let source = writer.source(&input.source, &input.path, &input.sha256)?;
|
||||
let bytes = read_bounded(&source, remaining, "sandbox asset")?;
|
||||
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
|
||||
return Err("sandbox asset changed during packaging".into());
|
||||
}
|
||||
remaining -= bytes.len() as u64;
|
||||
files.push((input.path, bytes));
|
||||
}
|
||||
files.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
dever_sandbox::validate_assets_for_target(&files, target.platform())?;
|
||||
// Leaf artifacts avoid a second manifest/signature or extraction protocol.
|
||||
let mut artifacts = Vec::new();
|
||||
for (path, bytes) in files {
|
||||
let destination = format!("sandbox/{}/{path}", target.platform());
|
||||
let output = writer.reserve(&destination)?;
|
||||
std::fs::write(&output, &bytes)
|
||||
.map_err(|error| format!("cannot write sandbox asset: {error}"))?;
|
||||
super::set_mode(&output, dever_sandbox::asset_executable(&path))?;
|
||||
artifacts.push(Artifact {
|
||||
path: destination,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: format!("{:x}", Sha256::digest(bytes)),
|
||||
});
|
||||
}
|
||||
Ok(artifacts)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,387 @@
|
||||
//! Explicit installation/update downloads. Project commands never enter here.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::release::{
|
||||
InstallLock, Layout, MachineManager, ReleaseManifest, Version, decode_hex_file,
|
||||
ensure_real_directory, platform_identity, reject_symlink_ancestors,
|
||||
set_shared_file_permissions, sync_directory, sync_tree, validate_shared_file,
|
||||
};
|
||||
|
||||
const OFFICIAL_RELEASES: &str = "https://github.com/shemic/dever-main/releases";
|
||||
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const METADATA_LIMIT: u64 = 2 * 1024 * 1024;
|
||||
const RELEASE_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
|
||||
static NEXT_DOWNLOAD: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
struct ReleaseSource {
|
||||
base: String,
|
||||
agent: ureq::Agent,
|
||||
}
|
||||
|
||||
impl ReleaseSource {
|
||||
fn official() -> Self {
|
||||
Self::new(OFFICIAL_RELEASES.to_owned())
|
||||
}
|
||||
|
||||
fn new(base: String) -> Self {
|
||||
Self {
|
||||
base,
|
||||
agent: ureq::Agent::new_with_config(
|
||||
ureq::Agent::config_builder()
|
||||
.proxy(None)
|
||||
.max_redirects(0)
|
||||
.timeout_global(Some(DOWNLOAD_TIMEOUT))
|
||||
.build(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn asset(&self, requested: &str, suffix: &str) -> String {
|
||||
let selection = if requested == "latest" {
|
||||
"latest/download".to_owned()
|
||||
} else {
|
||||
format!("download/v{requested}")
|
||||
};
|
||||
format!(
|
||||
"{}/{selection}/dever-{}.{suffix}",
|
||||
self.base,
|
||||
platform_identity()
|
||||
)
|
||||
}
|
||||
|
||||
fn response(
|
||||
&self,
|
||||
mut address: String,
|
||||
started: Instant,
|
||||
) -> Result<ureq::http::Response<ureq::Body>, String> {
|
||||
for _ in 0..=5 {
|
||||
let remaining = DOWNLOAD_TIMEOUT
|
||||
.checked_sub(started.elapsed())
|
||||
.filter(|duration| !duration.is_zero())
|
||||
.ok_or("Dever release download exceeded its five-minute deadline")?;
|
||||
let response = self
|
||||
.agent
|
||||
.get(&address)
|
||||
.config()
|
||||
.timeout_global(Some(remaining))
|
||||
.build()
|
||||
.call()
|
||||
.map_err(|error| match error {
|
||||
ureq::Error::StatusCode(404) => {
|
||||
"the official Dever release is not published for this version/platform; no installed version was changed".to_owned()
|
||||
}
|
||||
ureq::Error::StatusCode(code) => {
|
||||
format!("official Dever release download returned HTTP {code}")
|
||||
}
|
||||
_ => "cannot download the official Dever release".to_owned(),
|
||||
})?;
|
||||
if response.status().is_success() {
|
||||
return Ok(response);
|
||||
}
|
||||
if !response.status().is_redirection() {
|
||||
return Err("unexpected official release HTTP status".into());
|
||||
}
|
||||
let mut locations = response.headers().get_all("location").iter();
|
||||
let location = locations
|
||||
.next()
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or("release redirect is missing a valid Location")?;
|
||||
if locations.next().is_some() {
|
||||
return Err("release redirect has multiple Locations".into());
|
||||
}
|
||||
address = redirect(&address, location)?;
|
||||
}
|
||||
Err("official release download exceeded its redirect limit".into())
|
||||
}
|
||||
|
||||
fn metadata(
|
||||
&self,
|
||||
requested: &str,
|
||||
suffix: &str,
|
||||
path: &Path,
|
||||
started: Instant,
|
||||
) -> Result<(), String> {
|
||||
let mut response = self.response(self.asset(requested, suffix), started)?;
|
||||
let mut bytes = Vec::new();
|
||||
response
|
||||
.body_mut()
|
||||
.as_reader()
|
||||
.take(METADATA_LIMIT + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|_| "cannot read official release metadata")?;
|
||||
if bytes.len() as u64 > METADATA_LIMIT {
|
||||
return Err("official release metadata exceeds its size limit".into());
|
||||
}
|
||||
write_new(path, &bytes)
|
||||
}
|
||||
}
|
||||
|
||||
fn redirect(current: &str, location: &str) -> Result<String, String> {
|
||||
let next = url::Url::parse(current)
|
||||
.and_then(|base| base.join(location))
|
||||
.map_err(|_| "invalid release redirect")?;
|
||||
if next.scheme() != "https"
|
||||
|| !next.username().is_empty()
|
||||
|| next.password().is_some()
|
||||
|| next.port_or_known_default() != Some(443)
|
||||
|| next.fragment().is_some()
|
||||
|| !matches!(
|
||||
next.host_str(),
|
||||
Some(
|
||||
"github.com"
|
||||
| "release-assets.githubusercontent.com"
|
||||
| "objects.githubusercontent.com"
|
||||
)
|
||||
)
|
||||
{
|
||||
return Err("release redirect must remain on the official HTTPS asset hosts".into());
|
||||
}
|
||||
Ok(next.into())
|
||||
}
|
||||
|
||||
pub(super) fn prepare(layout: &Layout, requested: &str) -> Result<Version, String> {
|
||||
prepare_from(layout, requested, &ReleaseSource::official())
|
||||
}
|
||||
|
||||
fn prepare_from(
|
||||
layout: &Layout,
|
||||
requested: &str,
|
||||
source: &ReleaseSource,
|
||||
) -> Result<Version, String> {
|
||||
if requested != "latest" {
|
||||
Version::parse(requested)?;
|
||||
}
|
||||
layout.validate_machine_permissions()?;
|
||||
for directory in [layout.downloads(), layout.staging()] {
|
||||
reject_symlink_ancestors(&directory)?;
|
||||
ensure_real_directory(&directory, "release download directory")?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let metadata = fs::metadata(&directory).map_err(|error| error.to_string())?;
|
||||
if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 {
|
||||
return Err(
|
||||
"release download directories must be root-owned and not publicly writable"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
let _lock = InstallLock::acquire(layout)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
super::bootstrap::require_no_pending(layout)?;
|
||||
let key_path = layout.state().join("trusted-release-key");
|
||||
validate_shared_file(&key_path, &layout.state(), "trusted release key", false)?;
|
||||
let key = decode_hex_file(&key_path, 32)?;
|
||||
let staging = Download::new(&layout.staging())?;
|
||||
let started = Instant::now();
|
||||
source.metadata(
|
||||
requested,
|
||||
"manifest.json",
|
||||
&staging.0.join("manifest.json"),
|
||||
started,
|
||||
)?;
|
||||
source.metadata(
|
||||
requested,
|
||||
"manifest.sig",
|
||||
&staging.0.join("manifest.sig"),
|
||||
started,
|
||||
)?;
|
||||
let manifest = MachineManager::signed_manifest(&staging.0, &key)?;
|
||||
if !manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == "skills/dever-language/SKILL.md")
|
||||
{
|
||||
return Err("official releases must include the matching dever-language skill".into());
|
||||
}
|
||||
if requested != "latest" && manifest.version.as_str() != requested {
|
||||
return Err("downloaded release version does not match the requested version".into());
|
||||
}
|
||||
validate_catalog(&manifest)?;
|
||||
let version = manifest.version.clone();
|
||||
let destination = layout.downloads().join(version.as_str());
|
||||
match fs::symlink_metadata(&destination) {
|
||||
Ok(_) => {
|
||||
MachineManager::verify_package(&destination, Some(&version), &key)?;
|
||||
if fs::read(destination.join("manifest.json")).map_err(|error| error.to_string())?
|
||||
!= fs::read(staging.0.join("manifest.json")).map_err(|error| error.to_string())?
|
||||
{
|
||||
return Err("published Dever version changed its signed manifest".into());
|
||||
}
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
let mut response =
|
||||
source.response(source.asset(version.as_str(), "tar.gz"), started)?;
|
||||
let compressed = response.body_mut().as_reader().take(RELEASE_LIMIT + 1);
|
||||
unpack(
|
||||
flate2::read::GzDecoder::new(compressed),
|
||||
&staging.0,
|
||||
&manifest,
|
||||
)?;
|
||||
MachineManager::verify_package(&staging.0, Some(&version), &key)?;
|
||||
sync_tree(&staging.0)?;
|
||||
publish(&staging.0, &destination)?;
|
||||
sync_directory(&layout.downloads()).map_err(|error| error.to_string())?;
|
||||
}
|
||||
Err(error) => return Err(format!("cannot inspect downloaded release: {error}")),
|
||||
}
|
||||
// Latest is only a catalog hint; the signed version and active journal own trust.
|
||||
if requested == "latest" {
|
||||
let latest = layout.downloads().join("latest");
|
||||
match fs::symlink_metadata(&latest) {
|
||||
Ok(_) => validate_shared_file(&latest, &layout.downloads(), "release catalog", false)?,
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(format!("cannot inspect release catalog: {error}")),
|
||||
}
|
||||
let temporary = Download::new(&layout.staging())?;
|
||||
let catalog = temporary.0.join("latest");
|
||||
write_new(&catalog, format!("{version}\n").as_bytes())?;
|
||||
fs::rename(catalog, latest)
|
||||
.map_err(|error| format!("cannot publish release catalog: {error}"))?;
|
||||
sync_directory(&layout.downloads()).map_err(|error| error.to_string())?;
|
||||
}
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
|
||||
if manifest.artifacts.is_empty() || manifest.artifacts.len() > 4096 {
|
||||
return Err("release artifact count is outside 1..=4096".into());
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
let mut size = 0_u64;
|
||||
for artifact in &manifest.artifacts {
|
||||
if artifact
|
||||
.path
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
|| artifact.path.contains(['\\', ':', '\0'])
|
||||
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|
||||
|| !paths.insert(&artifact.path)
|
||||
{
|
||||
return Err("release has an unsafe or duplicate artifact path".into());
|
||||
}
|
||||
size = size
|
||||
.checked_add(artifact.bytes)
|
||||
.ok_or("release size overflow")?;
|
||||
if size > RELEASE_LIMIT {
|
||||
return Err("release exceeds its two-GiB unpacked size limit".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unpack(reader: impl Read, root: &Path, manifest: &ReleaseManifest) -> Result<(), String> {
|
||||
validate_catalog(manifest)?;
|
||||
let expected: BTreeMap<_, _> = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.map(|artifact| (artifact.path.as_str(), artifact))
|
||||
.collect();
|
||||
let mut seen = BTreeSet::new();
|
||||
// Raw entries reject PAX/GNU extensions before tar can buffer arbitrary metadata.
|
||||
let mut archive = tar::Archive::new(reader.take(RELEASE_LIMIT + 4096 * 1024));
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|error| format!("invalid release archive: {error}"))?
|
||||
.raw(true)
|
||||
{
|
||||
let mut entry = entry.map_err(|error| format!("invalid release archive entry: {error}"))?;
|
||||
if !entry.header().entry_type().is_file() {
|
||||
return Err("release archives may contain only regular files".into());
|
||||
}
|
||||
let bytes = entry.path_bytes();
|
||||
let name = std::str::from_utf8(&bytes)
|
||||
.map_err(|_| "release archive path is not UTF-8")?
|
||||
.to_owned();
|
||||
let artifact = expected
|
||||
.get(name.as_str())
|
||||
.ok_or("release archive contains an unsigned file")?;
|
||||
if !seen.insert(name.clone()) || entry.size() != artifact.bytes {
|
||||
return Err("release archive contains a duplicate or wrong-sized file".into());
|
||||
}
|
||||
let path = root.join(&name);
|
||||
fs::create_dir_all(path.parent().expect("artifact has a parent"))
|
||||
.map_err(|error| format!("cannot create release artifact directory: {error}"))?;
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot create release artifact: {error}"))?;
|
||||
let bytes = io::copy(&mut entry, &mut file)
|
||||
.map_err(|error| format!("cannot extract release artifact: {error}"))?;
|
||||
if bytes != artifact.bytes {
|
||||
return Err("release archive contains a truncated file".into());
|
||||
}
|
||||
set_shared_file_permissions(&path, false)?;
|
||||
file.sync_all().map_err(|error| error.to_string())?;
|
||||
}
|
||||
if seen.len() != expected.len() {
|
||||
return Err("release archive is missing signed artifacts".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(path)
|
||||
.map_err(|error| format!("cannot create release metadata: {error}"))?;
|
||||
file.write_all(bytes)
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| error.to_string())?;
|
||||
set_shared_file_permissions(path, false)
|
||||
}
|
||||
|
||||
fn publish(source: &Path, destination: &Path) -> Result<(), String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{CWD, RenameFlags, renameat_with};
|
||||
renameat_with(CWD, source, CWD, destination, RenameFlags::NOREPLACE)
|
||||
.map_err(|error| format!("cannot publish downloaded release: {error}"))
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
let _ = (source, destination);
|
||||
Err("official release downloads currently require Linux".into())
|
||||
}
|
||||
}
|
||||
|
||||
struct Download(PathBuf);
|
||||
|
||||
impl Download {
|
||||
fn new(parent: &Path) -> Result<Self, String> {
|
||||
let path = parent.join(format!(
|
||||
"download-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_DOWNLOAD.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
builder
|
||||
.create(&path)
|
||||
.map_err(|error| format!("cannot create release download staging: {error}"))?;
|
||||
Ok(Self(path))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Download {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../../../../test/dever-cli-tests/release_source.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,464 @@
|
||||
//! Compiler-relative, explicitly supplied native link inputs. This is not a
|
||||
//! package resolver or a substitute for the managed compiler service.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use dever_runtime::config::RuntimeProfile;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{BuildTarget, sha256_file};
|
||||
|
||||
const MANIFEST_LIMIT: u64 = 1024 * 1024;
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Manifest {
|
||||
pub(super) format: String,
|
||||
pub(super) compiler_version: String,
|
||||
pub(super) abi: u32,
|
||||
pub(super) platform: String,
|
||||
pub(super) target: String,
|
||||
pub(super) profiles: Profiles,
|
||||
pub(super) files: Vec<RuntimeArtifact>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Profiles {
|
||||
pub(super) base: Option<Profile>,
|
||||
pub(super) sqlite: Option<Profile>,
|
||||
pub(super) postgres: Option<Profile>,
|
||||
pub(super) both: Option<Profile>,
|
||||
}
|
||||
|
||||
impl Profiles {
|
||||
fn all(&self) -> [(&'static str, Option<&Profile>); 4] {
|
||||
[
|
||||
("base", self.base.as_ref()),
|
||||
("sqlite", self.sqlite.as_ref()),
|
||||
("postgres", self.postgres.as_ref()),
|
||||
("both", self.both.as_ref()),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(super) struct Profile {
|
||||
pub(super) start: Vec<String>,
|
||||
pub(super) runtime: String,
|
||||
pub(super) libraries: Vec<String>,
|
||||
pub(super) end: Vec<String>,
|
||||
}
|
||||
|
||||
impl Profile {
|
||||
fn inputs(&self) -> impl Iterator<Item = &str> {
|
||||
self.start
|
||||
.iter()
|
||||
.chain(std::iter::once(&self.runtime))
|
||||
.chain(&self.libraries)
|
||||
.chain(&self.end)
|
||||
.map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Deserialize, Serialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RuntimeArtifact {
|
||||
pub path: String,
|
||||
pub bytes: u64,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
pub struct RuntimePack {
|
||||
root: PathBuf,
|
||||
pub target: BuildTarget,
|
||||
pub identity: String,
|
||||
profile: Profile,
|
||||
files: BTreeMap<String, RuntimeArtifact>,
|
||||
}
|
||||
|
||||
pub struct LinkInputs {
|
||||
pub start: Vec<PathBuf>,
|
||||
pub libraries: Vec<PathBuf>,
|
||||
pub end: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
struct LoadedManifest {
|
||||
root: PathBuf,
|
||||
target: BuildTarget,
|
||||
bytes: Vec<u8>,
|
||||
profiles: Profiles,
|
||||
files: BTreeMap<String, RuntimeArtifact>,
|
||||
}
|
||||
|
||||
fn load_manifest(
|
||||
directory: &Path,
|
||||
compiler_version: &str,
|
||||
selected: BuildTarget,
|
||||
) -> Result<LoadedManifest, String> {
|
||||
let target = selected.triple();
|
||||
let platform = selected.platform();
|
||||
let root = checked_path(directory, &format!("runtime/{platform}"), false)?;
|
||||
let path = checked_path(&root, "manifest.json", true)?;
|
||||
let mut bytes = Vec::new();
|
||||
fs::File::open(&path)
|
||||
.and_then(|file| file.take(MANIFEST_LIMIT + 1).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read runtime pack manifest: {error}"))?;
|
||||
if bytes.len() as u64 > MANIFEST_LIMIT {
|
||||
return Err("runtime pack manifest exceeds 1 MiB".into());
|
||||
}
|
||||
let manifest: Manifest = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("invalid runtime pack manifest: {error}"))?;
|
||||
if manifest.format != "dever-native-runtime-v1"
|
||||
|| manifest.compiler_version != compiler_version
|
||||
// A v1 manifest describes v1 runtime symbols. Even bridge metadata
|
||||
// references would link the native SDK into the launcher and daemon.
|
||||
|| manifest.abi != 1
|
||||
|| manifest.platform != platform
|
||||
|| manifest.target != target
|
||||
{
|
||||
return Err(
|
||||
"runtime pack format, compiler version, ABI or target does not match this compiler"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
let files = validate_files(manifest.files)?;
|
||||
validate_profiles(&manifest.profiles, &files)?;
|
||||
Ok(LoadedManifest {
|
||||
root,
|
||||
target: selected,
|
||||
bytes,
|
||||
profiles: manifest.profiles,
|
||||
files,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the entire runtime closure that must be covered by the signed release.
|
||||
pub fn validate(
|
||||
directory: &Path,
|
||||
compiler_version: &str,
|
||||
target: BuildTarget,
|
||||
) -> Result<Vec<RuntimeArtifact>, String> {
|
||||
let manifest = load_manifest(directory, compiler_version, target)?;
|
||||
let prefix = format!("runtime/{}", target.platform());
|
||||
let mut artifacts = vec![RuntimeArtifact {
|
||||
path: format!("{prefix}/manifest.json"),
|
||||
bytes: manifest.bytes.len() as u64,
|
||||
sha256: format!("{:x}", Sha256::digest(&manifest.bytes)),
|
||||
}];
|
||||
for (path, artifact) in manifest.files {
|
||||
verify_file(
|
||||
&checked_path(&manifest.root, &path, true)?,
|
||||
&artifact,
|
||||
target,
|
||||
)?;
|
||||
artifacts.push(RuntimeArtifact {
|
||||
path: format!("{prefix}/{path}"),
|
||||
..artifact
|
||||
});
|
||||
}
|
||||
Ok(artifacts)
|
||||
}
|
||||
|
||||
pub(super) fn host_target() -> Result<&'static str, String> {
|
||||
Ok(BuildTarget::host()?.triple())
|
||||
}
|
||||
|
||||
impl RuntimePack {
|
||||
pub fn load(
|
||||
directory: &Path,
|
||||
profile: RuntimeProfile,
|
||||
selected: BuildTarget,
|
||||
) -> Result<Self, String> {
|
||||
let LoadedManifest {
|
||||
root,
|
||||
target,
|
||||
bytes,
|
||||
profiles,
|
||||
files,
|
||||
} = load_manifest(directory, env!("CARGO_PKG_VERSION"), selected)?;
|
||||
let name = match (profile.sqlite, profile.postgres) {
|
||||
(false, false) => "base",
|
||||
(true, false) => "sqlite",
|
||||
(false, true) => "postgres",
|
||||
(true, true) => "both",
|
||||
};
|
||||
let profile = profiles
|
||||
.all()
|
||||
.into_iter()
|
||||
.find_map(|(key, value)| (key == name).then_some(value).flatten())
|
||||
.ok_or_else(|| format!("runtime pack is missing the required '{name}' profile"))?
|
||||
.clone();
|
||||
// Validate even on a cache hit: a corrupt installed pack is never
|
||||
// silently replaced with an old cached executable.
|
||||
for input in profile.inputs() {
|
||||
verify_file(&checked_path(&root, input, true)?, &files[input], selected)?;
|
||||
}
|
||||
Ok(Self {
|
||||
root,
|
||||
target,
|
||||
identity: format!("{name}:{:x}", Sha256::digest(bytes)),
|
||||
profile,
|
||||
files,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn snapshot(&self, directory: &Path) -> Result<LinkInputs, String> {
|
||||
let destination = directory.join("runtime");
|
||||
fs::create_dir(&destination)
|
||||
.map_err(|error| format!("cannot create runtime input snapshot: {error}"))?;
|
||||
let mut inputs = Vec::new();
|
||||
for (index, input) in self.profile.inputs().enumerate() {
|
||||
let source = checked_path(&self.root, input, true)?;
|
||||
let extension = Path::new(input)
|
||||
.extension()
|
||||
.expect("validated input extension");
|
||||
let output = destination
|
||||
.join(index.to_string())
|
||||
.with_extension(extension);
|
||||
fs::copy(&source, &output)
|
||||
.map_err(|error| format!("cannot snapshot runtime input '{input}': {error}"))?;
|
||||
verify_file(&output, &self.files[input], self.target)?;
|
||||
inputs.push(output);
|
||||
}
|
||||
let start = inputs.drain(..self.profile.start.len()).collect();
|
||||
let end = inputs.split_off(1 + self.profile.libraries.len());
|
||||
Ok(LinkInputs {
|
||||
start,
|
||||
libraries: inputs,
|
||||
end,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_files(
|
||||
artifacts: Vec<RuntimeArtifact>,
|
||||
) -> Result<BTreeMap<String, RuntimeArtifact>, String> {
|
||||
if artifacts.is_empty() || artifacts.len() > 4096 {
|
||||
return Err("runtime pack must declare 1..4096 files".into());
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
for artifact in artifacts {
|
||||
relative_path(&artifact.path)?;
|
||||
if artifact.bytes == 0
|
||||
|| artifact.bytes > 1024 * 1024 * 1024
|
||||
|| artifact.sha256.len() != 64
|
||||
|| !artifact
|
||||
.sha256
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err(format!(
|
||||
"invalid runtime file digest or size for '{}'",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
if files.insert(artifact.path.clone(), artifact).is_some() {
|
||||
return Err("duplicate runtime file path".into());
|
||||
}
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn validate_profiles(
|
||||
profiles: &Profiles,
|
||||
files: &BTreeMap<String, RuntimeArtifact>,
|
||||
) -> Result<(), String> {
|
||||
let mut referenced = BTreeSet::new();
|
||||
for (_, profile) in profiles.all() {
|
||||
let Some(profile) = profile else { continue };
|
||||
if profile.start.is_empty() || profile.end.is_empty() || profile.libraries.is_empty() {
|
||||
return Err("runtime profile requires CRT start/end and system libraries".into());
|
||||
}
|
||||
let mut unique = BTreeSet::new();
|
||||
for input in profile.inputs() {
|
||||
if !files.contains_key(input) || !unique.insert(input) {
|
||||
return Err(format!(
|
||||
"runtime profile has an undeclared or duplicate input '{input}'"
|
||||
));
|
||||
}
|
||||
referenced.insert(input);
|
||||
}
|
||||
for (paths, extension) in [
|
||||
(profile.start.as_slice(), "o"),
|
||||
(profile.end.as_slice(), "o"),
|
||||
(profile.libraries.as_slice(), "a"),
|
||||
(std::slice::from_ref(&profile.runtime), "a"),
|
||||
] {
|
||||
if paths.iter().any(|path| {
|
||||
Path::new(path).extension().and_then(|value| value.to_str()) != Some(extension)
|
||||
}) {
|
||||
return Err(
|
||||
"runtime profiles accept only explicit object files and static archives".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if referenced.len() != files.len() {
|
||||
return Err("runtime pack contains unreferenced files".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn relative_path(path: &str) -> Result<(), String> {
|
||||
if path.is_empty()
|
||||
|| path.len() > 4096
|
||||
|| path.contains(['\\', ':', '\0'])
|
||||
|| path
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
{
|
||||
return Err("runtime pack paths must be normalized relative paths".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn checked_path(root: &Path, relative: &str, file: bool) -> Result<PathBuf, String> {
|
||||
relative_path(relative)?;
|
||||
let mut path = root.to_owned();
|
||||
let mut segments = relative.split('/').peekable();
|
||||
while let Some(segment) = segments.next() {
|
||||
path.push(segment);
|
||||
let metadata = fs::symlink_metadata(&path).map_err(|error| {
|
||||
format!("cannot inspect runtime pack '{}': {error}", path.display())
|
||||
})?;
|
||||
let expect_file = file && segments.peek().is_none();
|
||||
if metadata.file_type().is_symlink()
|
||||
|| if expect_file {
|
||||
!metadata.is_file()
|
||||
} else {
|
||||
!metadata.is_dir()
|
||||
}
|
||||
{
|
||||
return Err(format!(
|
||||
"runtime pack '{}' must be a real {}",
|
||||
path.display(),
|
||||
if expect_file { "file" } else { "directory" }
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn verify_file(path: &Path, artifact: &RuntimeArtifact, target: BuildTarget) -> Result<(), String> {
|
||||
let length = fs::metadata(path)
|
||||
.map_err(|error| format!("cannot inspect runtime file '{}': {error}", path.display()))?
|
||||
.len();
|
||||
if length != artifact.bytes || sha256_file(path)? != artifact.sha256 {
|
||||
return Err(format!(
|
||||
"runtime file '{}' failed size or SHA-256 verification",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
let mut prefix = [0_u8; 8];
|
||||
fs::File::open(path)
|
||||
.and_then(|mut file| file.read_exact(&mut prefix))
|
||||
.map_err(|error| format!("cannot read runtime input header: {error}"))?;
|
||||
let valid = match Path::new(&artifact.path)
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
{
|
||||
Some("a") => &prefix == b"!<arch>\n",
|
||||
Some("o") => &prefix[..4] == b"\x7fELF",
|
||||
_ => false,
|
||||
};
|
||||
if !valid {
|
||||
return Err(format!(
|
||||
"runtime input '{}' is not an ELF object or self-contained archive; linker scripts and thin archives are forbidden",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
validate_target(path, length, target).map_err(|error| {
|
||||
format!(
|
||||
"runtime input '{}' failed target verification: {error}",
|
||||
artifact.path
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_target(path: &Path, length: u64, target: BuildTarget) -> Result<(), String> {
|
||||
let mut file = fs::File::open(path).map_err(|error| error.to_string())?;
|
||||
if path.extension().is_some_and(|extension| extension == "o") {
|
||||
return validate_object(&mut file, length, target);
|
||||
}
|
||||
// Walk archive members without loading large runtime archives into memory.
|
||||
let mut offset = 8_u64;
|
||||
while offset < length {
|
||||
file.seek(SeekFrom::Start(offset))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut header = [0; 60];
|
||||
file.read_exact(&mut header)
|
||||
.map_err(|_| "truncated runtime archive member")?;
|
||||
if &header[58..] != b"`\n" {
|
||||
return Err("invalid runtime archive member header".into());
|
||||
}
|
||||
let size = std::str::from_utf8(&header[48..58])
|
||||
.ok()
|
||||
.and_then(|size| size.trim().parse::<u64>().ok())
|
||||
.ok_or("invalid runtime archive member size")?;
|
||||
let start = offset
|
||||
.checked_add(60)
|
||||
.ok_or("runtime archive size overflow")?;
|
||||
let end = start
|
||||
.checked_add(size)
|
||||
.filter(|end| *end <= length)
|
||||
.ok_or("truncated runtime archive member")?;
|
||||
let name = std::str::from_utf8(&header[..16])
|
||||
.map_err(|_| "invalid runtime archive member name")?
|
||||
.trim();
|
||||
if !matches!(
|
||||
name,
|
||||
"/" | "//" | "/SYM64/" | "__.SYMDEF" | "__.SYMDEF SORTED"
|
||||
) {
|
||||
let name_bytes = match name.strip_prefix("#1/") {
|
||||
Some(bytes) => bytes
|
||||
.parse::<u64>()
|
||||
.map_err(|_| "invalid archive extended name")?,
|
||||
None => 0,
|
||||
};
|
||||
let object_size = size
|
||||
.checked_sub(name_bytes)
|
||||
.ok_or("invalid archive extended name size")?;
|
||||
file.seek(SeekFrom::Start(start + name_bytes))
|
||||
.map_err(|error| error.to_string())?;
|
||||
validate_object(&mut file, object_size, target)?;
|
||||
}
|
||||
offset = end
|
||||
.checked_add(size % 2)
|
||||
.filter(|end| *end <= length)
|
||||
.ok_or("truncated runtime archive padding")?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_object(file: &mut fs::File, length: u64, target: BuildTarget) -> Result<(), String> {
|
||||
let mut header = [0; 20];
|
||||
if length < header.len() as u64 {
|
||||
return Err("runtime input requires a complete target ELF object header".into());
|
||||
}
|
||||
file.read_exact(&mut header)
|
||||
.map_err(|error| format!("cannot read runtime object: {error}"))?;
|
||||
let machine = match target {
|
||||
BuildTarget::LinuxX86_64 => 62,
|
||||
BuildTarget::LinuxAarch64 => 183,
|
||||
};
|
||||
if &header[..4] != b"\x7fELF"
|
||||
|| header[4] != 2
|
||||
|| header[5] != 1
|
||||
|| header[6] != 1
|
||||
|| u16::from_le_bytes([header[16], header[17]]) != 1
|
||||
|| u16::from_le_bytes([header[18], header[19]]) != machine
|
||||
{
|
||||
return Err(format!(
|
||||
"runtime input is not a relocatable ELF object for {}",
|
||||
target.platform()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
//! The small authenticated boundary around the machine cache.
|
||||
//!
|
||||
//! `CacheStore` remains an on-disk engine owned by this process. Clients use
|
||||
//! this module's length-delimited local IPC instead of constructing the store
|
||||
//! in the launcher. Platform installers can replace the fixture socket with
|
||||
//! the documented system endpoint without changing the request contract.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
#[cfg(unix)]
|
||||
use std::fs::File;
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::Path;
|
||||
#[cfg(unix)]
|
||||
use std::sync::{Arc, Mutex, mpsc};
|
||||
|
||||
#[cfg(unix)]
|
||||
use fs2::FileExt;
|
||||
use ring::rand::{SecureRandom, SystemRandom};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::cache::{ArtifactReceipt, CacheStatus, CacheStore};
|
||||
use super::release::{Layout, Version};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[cfg(unix)]
|
||||
mod compile;
|
||||
|
||||
pub use self::compile_request::compile;
|
||||
|
||||
mod compile_request;
|
||||
|
||||
const TOKEN_BYTES: usize = 32;
|
||||
const MAX_MESSAGE: usize = 1024 * 1024;
|
||||
#[cfg(unix)]
|
||||
const CLIENT_WORKERS: usize = 8;
|
||||
|
||||
#[cfg(unix)]
|
||||
struct Client {
|
||||
stream: std::os::unix::net::UnixStream,
|
||||
caller: Option<u32>,
|
||||
#[cfg(target_os = "linux")]
|
||||
_admission: Option<ForeignAdmission>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
struct ForeignAdmission {
|
||||
uid: u32,
|
||||
users: Arc<Mutex<std::collections::BTreeMap<u32, usize>>>,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
impl ForeignAdmission {
|
||||
fn acquire(
|
||||
users: &Arc<Mutex<std::collections::BTreeMap<u32, usize>>>,
|
||||
uid: u32,
|
||||
) -> Option<Self> {
|
||||
let mut counts = users.lock().unwrap();
|
||||
// One stalled caller cannot fill the worker pool. Two service slots
|
||||
// stay available to the owner even when foreign users are saturated.
|
||||
if counts.values().sum::<usize>() >= CLIENT_WORKERS - 2
|
||||
|| counts.get(&uid).copied().unwrap_or(0) >= 2
|
||||
{
|
||||
return None;
|
||||
}
|
||||
*counts.entry(uid).or_default() += 1;
|
||||
Some(Self {
|
||||
uid,
|
||||
users: Arc::clone(users),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
impl Drop for ForeignAdmission {
|
||||
fn drop(&mut self) {
|
||||
let mut counts = self.users.lock().unwrap();
|
||||
if let Some(count) = counts.get_mut(&self.uid) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
counts.remove(&self.uid);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Request {
|
||||
#[serde(default)]
|
||||
token: String,
|
||||
operation: Operation,
|
||||
#[serde(default)]
|
||||
installed: Vec<Version>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case", deny_unknown_fields)]
|
||||
enum Operation {
|
||||
Status,
|
||||
Clean,
|
||||
ArtifactPut(ArtifactReceipt),
|
||||
ArtifactGet(ArtifactReceipt),
|
||||
Compile { bytes: u64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Response {
|
||||
ok: bool,
|
||||
status: Option<CacheStatus>,
|
||||
error: Option<String>,
|
||||
#[serde(default)]
|
||||
artifact: Option<ArtifactReceipt>,
|
||||
}
|
||||
|
||||
/// Ensure a per-machine authentication token exists. The token is not an
|
||||
/// environment variable and is never accepted from a command line argument.
|
||||
pub fn ensure_token(layout: &Layout) -> Result<(), String> {
|
||||
if layout.service_token().exists() {
|
||||
validate_token_file(layout)?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut token = [0_u8; TOKEN_BYTES];
|
||||
SystemRandom::new()
|
||||
.fill(&mut token)
|
||||
.map_err(|_| "cannot generate deverd authentication token".to_owned())?;
|
||||
let mut options = OpenOptions::new();
|
||||
options.write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
let mut file = options
|
||||
.open(layout.service_token())
|
||||
.map_err(|error| format!("cannot create deverd authentication token: {error}"))?;
|
||||
file.write_all(hex(&token).as_bytes())
|
||||
.and_then(|_| file.write_all(b"\n"))
|
||||
.and_then(|_| file.sync_all())
|
||||
.map_err(|error| format!("cannot write deverd authentication token: {error}"))?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(layout.service_token(), fs::Permissions::from_mode(0o600))
|
||||
.map_err(|error| format!("cannot protect deverd authentication token: {error}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Serve requests until the process is stopped. This is intentionally a
|
||||
/// local service primitive; OS service registration and privilege adapters
|
||||
/// belong to the distribution layer.
|
||||
pub fn serve(layout: &Layout) -> Result<(), String> {
|
||||
#[cfg(unix)]
|
||||
let _service_lock = acquire_service_lock(layout)?;
|
||||
ensure_token(layout)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::FileTypeExt;
|
||||
use std::os::unix::net::UnixListener;
|
||||
if let Some(parent) = layout.service_socket().parent() {
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("cannot create deverd state directory: {error}"))?;
|
||||
}
|
||||
match fs::symlink_metadata(layout.service_socket()) {
|
||||
Ok(metadata)
|
||||
if metadata.file_type().is_symlink() || !metadata.file_type().is_socket() =>
|
||||
{
|
||||
return Err("deverd IPC endpoint must be a real Unix socket".into());
|
||||
}
|
||||
Ok(_) => fs::remove_file(layout.service_socket())
|
||||
.map_err(|error| format!("cannot replace deverd IPC endpoint: {error}"))?,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(format!("cannot inspect deverd IPC endpoint: {error}")),
|
||||
}
|
||||
let listener = UnixListener::bind(layout.service_socket())
|
||||
.map_err(|error| format!("cannot bind deverd IPC endpoint: {error}"))?;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
// Linux obtains the calling uid from the kernel. The cache directories
|
||||
// remain private; only the socket is usable by other machine users.
|
||||
let mode = if cfg!(target_os = "linux") {
|
||||
0o666
|
||||
} else {
|
||||
0o600
|
||||
};
|
||||
fs::set_permissions(layout.service_socket(), fs::Permissions::from_mode(mode))
|
||||
.map_err(|error| format!("cannot protect deverd IPC endpoint: {error}"))?;
|
||||
let (sender, receiver) = mpsc::sync_channel(CLIENT_WORKERS);
|
||||
let receiver = Arc::new(Mutex::new(receiver));
|
||||
#[cfg(target_os = "linux")]
|
||||
let (owner, foreign_users) = {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let owner = fs::symlink_metadata(layout.state())
|
||||
.map_err(|error| format!("cannot inspect deverd state: {error}"))?
|
||||
.uid();
|
||||
if owner != rustix::process::geteuid().as_raw() {
|
||||
return Err("deverd must run as the state directory owner".into());
|
||||
}
|
||||
(
|
||||
owner,
|
||||
Arc::new(Mutex::new(std::collections::BTreeMap::new())),
|
||||
)
|
||||
};
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..CLIENT_WORKERS {
|
||||
let receiver = Arc::clone(&receiver);
|
||||
scope.spawn(move || {
|
||||
loop {
|
||||
let next = { receiver.lock().unwrap().recv() };
|
||||
match next {
|
||||
Ok(client) => {
|
||||
let Client {
|
||||
stream,
|
||||
caller,
|
||||
#[cfg(target_os = "linux")]
|
||||
_admission,
|
||||
} = client;
|
||||
serve_client(layout, stream, caller);
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
let result = loop {
|
||||
match listener.accept() {
|
||||
Ok((stream, _)) => {
|
||||
#[cfg(target_os = "linux")]
|
||||
let caller = match rustix::net::sockopt::socket_peercred(&stream) {
|
||||
Ok(credentials) => Some(credentials.uid.as_raw()),
|
||||
Err(_) => continue,
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let caller = None;
|
||||
#[cfg(target_os = "linux")]
|
||||
let admission = if caller != Some(owner) {
|
||||
match ForeignAdmission::acquire(
|
||||
&foreign_users,
|
||||
caller.expect("checked kernel uid"),
|
||||
) {
|
||||
Some(admission) => Some(admission),
|
||||
None => continue,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let client = Client {
|
||||
stream,
|
||||
caller,
|
||||
#[cfg(target_os = "linux")]
|
||||
_admission: admission,
|
||||
};
|
||||
let _ = sender.try_send(client);
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => break Err(format!("deverd IPC accept failed: {error}")),
|
||||
}
|
||||
};
|
||||
drop(sender);
|
||||
result
|
||||
})
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = layout;
|
||||
Err("deverd authenticated IPC is not implemented for this platform yet".into())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cache_status(layout: &Layout) -> Result<CacheStatus, String> {
|
||||
request(layout, Operation::Status, &[]).and_then(|response| {
|
||||
response.status.ok_or_else(|| {
|
||||
response
|
||||
.error
|
||||
.unwrap_or_else(|| "deverd returned no cache status".into())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn cache_clean(layout: &Layout, installed: &BTreeSet<Version>) -> Result<CacheStatus, String> {
|
||||
request(
|
||||
layout,
|
||||
Operation::Clean,
|
||||
&installed.iter().cloned().collect::<Vec<_>>(),
|
||||
)
|
||||
.and_then(|response| {
|
||||
response.status.ok_or_else(|| {
|
||||
response
|
||||
.error
|
||||
.unwrap_or_else(|| "deverd returned no cache status".into())
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn artifact_put(layout: &Layout, bytes: &[u8]) -> Result<ArtifactReceipt, String> {
|
||||
let expected = ArtifactReceipt {
|
||||
sha256: hex(&Sha256::digest(bytes)),
|
||||
bytes: bytes.len() as u64,
|
||||
};
|
||||
expected.validate()?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let (mut stream, token) = connect(layout)?;
|
||||
write_frame(
|
||||
&mut stream,
|
||||
&Request {
|
||||
token,
|
||||
operation: Operation::ArtifactPut(expected.clone()),
|
||||
installed: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
require_artifact(read_frame(&mut stream)?, &expected)?;
|
||||
stream
|
||||
.write_all(bytes)
|
||||
.map_err(|error| format!("deverd artifact upload failed: {error}"))?;
|
||||
require_artifact(read_frame(&mut stream)?, &expected)?;
|
||||
Ok(expected)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = layout;
|
||||
Err("deverd artifact IPC is not implemented for this platform".into())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>, String> {
|
||||
let expected = ArtifactReceipt {
|
||||
sha256: sha256.to_owned(),
|
||||
bytes,
|
||||
};
|
||||
expected.validate()?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let (mut stream, token) = connect(layout)?;
|
||||
write_frame(
|
||||
&mut stream,
|
||||
&Request {
|
||||
token,
|
||||
operation: Operation::ArtifactGet(expected.clone()),
|
||||
installed: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
require_artifact(read_frame(&mut stream)?, &expected)?;
|
||||
let mut bytes = vec![0; expected.bytes as usize];
|
||||
stream
|
||||
.read_exact(&mut bytes)
|
||||
.map_err(|error| format!("deverd artifact download is incomplete: {error}"))?;
|
||||
if hex(&Sha256::digest(&bytes)) != expected.sha256 {
|
||||
return Err("deverd artifact download failed SHA-256 verification".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = layout;
|
||||
Err("deverd artifact IPC is not implemented for this platform".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn require_artifact(response: Response, expected: &ArtifactReceipt) -> Result<(), String> {
|
||||
if !response.ok {
|
||||
return Err(response
|
||||
.error
|
||||
.unwrap_or_else(|| "deverd artifact request failed".into()));
|
||||
}
|
||||
if response.artifact.as_ref() != Some(expected) {
|
||||
return Err("deverd returned a different artifact identity".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn connect(layout: &Layout) -> Result<(std::os::unix::net::UnixStream, String), String> {
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt};
|
||||
use std::os::unix::net::UnixStream;
|
||||
|
||||
let state = fs::symlink_metadata(layout.state())
|
||||
.map_err(|error| format!("cannot inspect deverd state directory: {error}"))?;
|
||||
if !state.is_dir() || state.file_type().is_symlink() || state.mode() & 0o022 != 0 {
|
||||
return Err("deverd state directory must not be writable by other users".into());
|
||||
}
|
||||
let endpoint = fs::symlink_metadata(layout.service_socket())
|
||||
.map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?;
|
||||
if !endpoint.file_type().is_socket() || endpoint.uid() != state.uid() {
|
||||
return Err("deverd IPC endpoint does not belong to the service owner".into());
|
||||
}
|
||||
let stream = UnixStream::connect(layout.service_socket())
|
||||
.map_err(|error| format!("authenticated deverd cache service unavailable: {error}"))?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if rustix::net::sockopt::socket_peercred(&stream)
|
||||
.map_err(|error| format!("cannot authenticate deverd OS peer: {error}"))?
|
||||
.uid
|
||||
.as_raw()
|
||||
!= state.uid()
|
||||
{
|
||||
return Err("deverd OS peer does not match the service owner".into());
|
||||
}
|
||||
stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
|
||||
.and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5))))
|
||||
.map_err(|error| format!("cannot configure deverd IPC timeout: {error}"))?;
|
||||
#[cfg(target_os = "linux")]
|
||||
if rustix::process::geteuid().as_raw() != state.uid() {
|
||||
return Ok((stream, String::new()));
|
||||
}
|
||||
validate_token_file(layout)?;
|
||||
let token = String::from_utf8(read_file(&layout.service_token())?)
|
||||
.map_err(|_| "invalid deverd authentication token".to_owned())?;
|
||||
Ok((stream, token.trim().to_owned()))
|
||||
}
|
||||
|
||||
fn request(
|
||||
layout: &Layout,
|
||||
operation: Operation,
|
||||
installed: &[Version],
|
||||
) -> Result<Response, String> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let (mut stream, token) = connect(layout)?;
|
||||
let request = Request {
|
||||
token,
|
||||
operation,
|
||||
installed: installed.to_owned(),
|
||||
};
|
||||
write_frame(&mut stream, &request)?;
|
||||
read_frame(&mut stream)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (layout, operation, installed);
|
||||
Err(
|
||||
"authenticated deverd cache service unavailable: platform IPC is not implemented"
|
||||
.into(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn handle(
|
||||
layout: &Layout,
|
||||
stream: &mut std::os::unix::net::UnixStream,
|
||||
caller: Option<u32>,
|
||||
) -> Response {
|
||||
let request: Result<Request, _> = read_frame(stream);
|
||||
let request = match request {
|
||||
Ok(request) => request,
|
||||
Err(error) => return failure(error),
|
||||
};
|
||||
if let Err(error) = validate_token_file(layout) {
|
||||
return failure(error);
|
||||
}
|
||||
let bytes = match read_file(&layout.service_token()) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(error) => return failure(error),
|
||||
};
|
||||
let expected = match String::from_utf8(bytes) {
|
||||
Ok(token) => token.trim().as_bytes().to_vec(),
|
||||
Err(_) => return failure("invalid deverd authentication token".into()),
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
let foreign = match caller {
|
||||
Some(uid) => {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
match fs::symlink_metadata(layout.state()) {
|
||||
Ok(state) => uid != state.uid(),
|
||||
Err(error) => return failure(format!("cannot inspect deverd state: {error}")),
|
||||
}
|
||||
}
|
||||
None => return failure("deverd OS caller credentials are unavailable".into()),
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let foreign = {
|
||||
let _ = caller;
|
||||
false
|
||||
};
|
||||
if foreign {
|
||||
if matches!(request.operation, Operation::Clean) {
|
||||
return failure("deverd cache clean requires the service owner".into());
|
||||
}
|
||||
} else if !constant_time_equal(request.token.as_bytes(), &expected) {
|
||||
return failure("deverd authentication failed".into());
|
||||
}
|
||||
let store = CacheStore::new(layout);
|
||||
let result = match request.operation {
|
||||
Operation::Status => {
|
||||
if foreign {
|
||||
store.summary()
|
||||
} else {
|
||||
store.status()
|
||||
}
|
||||
}
|
||||
Operation::Clean => {
|
||||
if request.installed.len() > 256 {
|
||||
return failure("too many installed Dever versions".into());
|
||||
}
|
||||
store.clean(&request.installed.into_iter().collect())
|
||||
}
|
||||
Operation::ArtifactPut(expected) => {
|
||||
if let Err(error) = expected.validate() {
|
||||
return failure(error);
|
||||
}
|
||||
if let Err(error) = write_frame(stream, &artifact_response(expected.clone())) {
|
||||
return failure(error);
|
||||
}
|
||||
let mut input = TransferReader {
|
||||
input: stream,
|
||||
started: std::time::Instant::now(),
|
||||
};
|
||||
return match store.publish_artifact(&expected, &mut input) {
|
||||
Ok(receipt) => artifact_response(receipt),
|
||||
Err(error) => failure(error),
|
||||
};
|
||||
}
|
||||
Operation::ArtifactGet(expected) => {
|
||||
let mut artifact = match store.artifact(&expected) {
|
||||
Ok(artifact) => artifact,
|
||||
Err(error) => return failure(error),
|
||||
};
|
||||
if let Err(error) = write_frame(stream, &artifact_response(artifact.receipt)) {
|
||||
return failure(error);
|
||||
}
|
||||
if let Err(error) = std::io::copy(&mut artifact.file, stream) {
|
||||
return failure(format!("deverd artifact transfer failed: {error}"));
|
||||
}
|
||||
return Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
artifact: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
Operation::Compile { bytes } => {
|
||||
return match compile::serve(layout, stream, bytes) {
|
||||
Ok(()) => Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
artifact: None,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => failure(error),
|
||||
};
|
||||
}
|
||||
};
|
||||
match result {
|
||||
Ok(status) => Response {
|
||||
ok: true,
|
||||
status: Some(status),
|
||||
error: None,
|
||||
artifact: None,
|
||||
},
|
||||
Err(error) => failure(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn failure(error: String) -> Response {
|
||||
Response {
|
||||
ok: false,
|
||||
status: None,
|
||||
error: Some(error),
|
||||
artifact: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact_response(artifact: ArtifactReceipt) -> Response {
|
||||
Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
error: None,
|
||||
artifact: Some(artifact),
|
||||
}
|
||||
}
|
||||
|
||||
struct TransferReader<'a, R> {
|
||||
input: &'a mut R,
|
||||
started: std::time::Instant,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for TransferReader<'_, R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||
if self.started.elapsed() >= std::time::Duration::from_secs(60) {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"deverd artifact transfer deadline exceeded",
|
||||
));
|
||||
}
|
||||
self.input.read(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn serve_client(layout: &Layout, mut stream: std::os::unix::net::UnixStream, caller: Option<u32>) {
|
||||
if stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(5)))
|
||||
.and_then(|()| stream.set_write_timeout(Some(std::time::Duration::from_secs(5))))
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
let response = handle(layout, &mut stream, caller);
|
||||
let _ = write_frame(&mut stream, &response);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn acquire_service_lock(layout: &Layout) -> Result<File, String> {
|
||||
use std::os::unix::fs::{MetadataExt, OpenOptionsExt};
|
||||
|
||||
let path = layout.state().join("deverd.lock");
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true).write(true);
|
||||
match fs::symlink_metadata(&path) {
|
||||
Ok(_) => {
|
||||
validate_owned_file(layout, &path, "deverd lock")?;
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
options.create_new(true).mode(0o600);
|
||||
}
|
||||
Err(error) => return Err(format!("cannot inspect deverd lock: {error}")),
|
||||
}
|
||||
let file = options
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot open deverd lock: {error}"))?;
|
||||
let opened = file
|
||||
.metadata()
|
||||
.map_err(|error| format!("cannot inspect deverd lock: {error}"))?;
|
||||
let current = validate_owned_file(layout, &path, "deverd lock")?;
|
||||
if opened.dev() != current.dev() || opened.ino() != current.ino() {
|
||||
return Err("deverd lock changed while opening".into());
|
||||
}
|
||||
FileExt::try_lock_exclusive(&file).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock {
|
||||
"deverd is already running for this machine root".into()
|
||||
} else {
|
||||
format!("cannot lock deverd state: {error}")
|
||||
}
|
||||
})?;
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
fn validate_token_file(layout: &Layout) -> Result<(), String> {
|
||||
let path = layout.service_token();
|
||||
validate_owned_file(layout, &path, "deverd authentication token")?;
|
||||
let token = read_file(&path)?;
|
||||
let token = token.strip_suffix(b"\n").unwrap_or(&token);
|
||||
if token.len() != TOKEN_BYTES * 2
|
||||
|| !token
|
||||
.iter()
|
||||
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
|
||||
{
|
||||
return Err("invalid deverd authentication token".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_owned_file(layout: &Layout, path: &Path, name: &str) -> Result<fs::Metadata, String> {
|
||||
let metadata =
|
||||
fs::symlink_metadata(path).map_err(|error| format!("cannot inspect {name}: {error}"))?;
|
||||
if metadata.file_type().is_symlink() || !metadata.is_file() {
|
||||
return Err(format!("{name} must be a regular file"));
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let state = fs::symlink_metadata(layout.state())
|
||||
.map_err(|error| format!("cannot inspect deverd state directory: {error}"))?;
|
||||
if !state.is_dir() || state.uid() != metadata.uid() {
|
||||
return Err(format!("{name} must have the state directory owner"));
|
||||
}
|
||||
if metadata.mode() & 0o077 != 0 {
|
||||
return Err(format!("{name} must be accessible only to its owner"));
|
||||
}
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
fn write_frame<T: Serialize>(stream: &mut impl Write, value: &T) -> Result<(), String> {
|
||||
let payload =
|
||||
serde_json::to_vec(value).map_err(|error| format!("deverd IPC encode failed: {error}"))?;
|
||||
if payload.len() > MAX_MESSAGE {
|
||||
return Err("deverd IPC message is too large".into());
|
||||
}
|
||||
stream
|
||||
.write_all(&(payload.len() as u32).to_be_bytes())
|
||||
.and_then(|_| stream.write_all(&payload))
|
||||
.and_then(|_| stream.flush())
|
||||
.map_err(|error| format!("deverd IPC write failed: {error}"))
|
||||
}
|
||||
|
||||
fn read_frame<T: for<'de> Deserialize<'de>>(stream: &mut impl Read) -> Result<T, String> {
|
||||
let mut length = [0_u8; 4];
|
||||
stream
|
||||
.read_exact(&mut length)
|
||||
.map_err(|error| format!("deverd IPC read failed: {error}"))?;
|
||||
let length = u32::from_be_bytes(length) as usize;
|
||||
if length > MAX_MESSAGE {
|
||||
return Err("deverd IPC message is too large".into());
|
||||
}
|
||||
let mut payload = vec![0; length];
|
||||
stream
|
||||
.read_exact(&mut payload)
|
||||
.map_err(|error| format!("deverd IPC read failed: {error}"))?;
|
||||
let text =
|
||||
std::str::from_utf8(&payload).map_err(|_| "deverd IPC payload must be UTF-8 JSON")?;
|
||||
dever_runtime::wire::parse(text)
|
||||
.map_err(|error| format!("deverd IPC decode failed: {error}"))?;
|
||||
serde_json::from_slice(&payload).map_err(|error| format!("deverd IPC decode failed: {error}"))
|
||||
}
|
||||
|
||||
fn read_file(path: &Path) -> Result<Vec<u8>, String> {
|
||||
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
|
||||
}
|
||||
|
||||
fn constant_time_equal(left: &[u8], right: &[u8]) -> bool {
|
||||
if left.len() != right.len() {
|
||||
return false;
|
||||
}
|
||||
let mut difference = 0_u8;
|
||||
for (left, right) in left.iter().zip(right) {
|
||||
difference |= left ^ right;
|
||||
}
|
||||
difference == 0
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
const DIGITS: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut output = String::with_capacity(bytes.len() * 2);
|
||||
for byte in bytes {
|
||||
output.push(DIGITS[usize::from(byte >> 4)] as char);
|
||||
output.push(DIGITS[usize::from(byte & 0x0f)] as char);
|
||||
}
|
||||
output
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
//! Only this service path may publish compiled entries. Artifact uploads remain
|
||||
//! opaque bytes and can never satisfy a compilation lookup.
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::super::cache::{ArtifactReceipt, BuildIdentity, CacheStore, MAX_COMPILED_BYTES};
|
||||
use super::super::compilation::{
|
||||
CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE,
|
||||
};
|
||||
use super::super::release::{InstalledCompiler, Layout, MachineManager, Version};
|
||||
use super::{Response, artifact_response, hex, write_frame};
|
||||
|
||||
const COMPILE_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
const MAX_DIAGNOSTIC_BYTES: u64 = 64 * 1024;
|
||||
static ACTIVE_COMPILATIONS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
struct Admission;
|
||||
|
||||
impl Admission {
|
||||
fn acquire() -> Result<Self, String> {
|
||||
ACTIVE_COMPILATIONS
|
||||
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| {
|
||||
(count < 2).then_some(count + 1)
|
||||
})
|
||||
.map_err(|_| "deverd compilation capacity is busy; retry later".to_owned())?;
|
||||
Ok(Self)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Admission {
|
||||
fn drop(&mut self) {
|
||||
ACTIVE_COMPILATIONS.fetch_sub(1, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn serve(layout: &Layout, stream: &mut UnixStream, bytes: u64) -> Result<(), String> {
|
||||
if bytes == 0 || bytes > MAX_COMPILE_BYTES as u64 {
|
||||
return Err("deverd compilation request exceeds its transfer limit".into());
|
||||
}
|
||||
let _admission = Admission::acquire()?;
|
||||
let started = Instant::now();
|
||||
let store = CacheStore::new(layout);
|
||||
let _operation = store.operation()?;
|
||||
write_frame(
|
||||
stream,
|
||||
&Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
artifact: None,
|
||||
error: None,
|
||||
},
|
||||
)?;
|
||||
let mut payload = vec![0; bytes as usize];
|
||||
super::TransferReader {
|
||||
input: stream,
|
||||
started,
|
||||
}
|
||||
.read_exact(&mut payload)
|
||||
.map_err(|error| format!("incomplete compilation request: {error}"))?;
|
||||
let request = CompileRequest::decode(&payload)?;
|
||||
let version = Version::parse(&request.version)?;
|
||||
let target = request.target;
|
||||
// Canonical bytes own the cache identity; decoded interpreter copies are no
|
||||
// longer needed by the service while the worker rereads its private request.
|
||||
drop(request);
|
||||
let compiler = MachineManager::new(layout.clone()).compilation(&version)?;
|
||||
verify_pack(&compiler, target)?;
|
||||
let identity = BuildIdentity {
|
||||
version: &version,
|
||||
compiler: compiler.identity.as_bytes(),
|
||||
runtime: compiler.identity.as_bytes(),
|
||||
target: target.platform(),
|
||||
generated: &payload,
|
||||
options: b"dever-trusted-compilation-v1",
|
||||
environment: b"",
|
||||
};
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_millis(1)))
|
||||
.map_err(|error| format!("cannot monitor compilation caller: {error}"))?;
|
||||
require_connected(stream, started)?;
|
||||
if let Some(artifact) = store.restore(&identity)? {
|
||||
return send_output(stream, artifact.as_bytes(), started);
|
||||
}
|
||||
let directory = store.compilation_directory()?;
|
||||
write_private(&directory.0.join(WORKER_REQUEST_FILE), &payload)?;
|
||||
run_worker(&compiler.core, &directory.0, stream, started)?;
|
||||
require_connected(stream, started)?;
|
||||
let output = directory.0.join(WORKER_OUTPUT_FILE);
|
||||
let metadata = fs::symlink_metadata(&output)
|
||||
.map_err(|error| format!("compiler produced no executable: {error}"))?;
|
||||
if !metadata.is_file()
|
||||
|| metadata.file_type().is_symlink()
|
||||
|| metadata.len() == 0
|
||||
|| metadata.len() > MAX_COMPILED_BYTES
|
||||
{
|
||||
return Err("compiler output must be a regular executable of at most 320 MiB".into());
|
||||
}
|
||||
store.publish(&identity, &output)?;
|
||||
let artifact = store
|
||||
.restore(&identity)?
|
||||
.ok_or("compiled output was not published")?;
|
||||
send_output(stream, artifact.as_bytes(), started)
|
||||
}
|
||||
|
||||
fn verify_pack(
|
||||
compiler: &InstalledCompiler,
|
||||
target: super::super::BuildTarget,
|
||||
) -> Result<(), String> {
|
||||
let root = compiler
|
||||
.core
|
||||
.parent()
|
||||
.ok_or("installed compiler has no directory")?;
|
||||
let artifacts =
|
||||
super::super::runtime_pack::validate(root, compiler.manifest.version.as_str(), target)?;
|
||||
for input in artifacts {
|
||||
if !compiler.manifest.artifacts.iter().any(|signed| {
|
||||
signed.path == input.path
|
||||
&& signed.bytes == input.bytes
|
||||
&& signed.sha256 == input.sha256
|
||||
}) {
|
||||
return Err(format!(
|
||||
"native runtime input '{}' is not covered by the signed release",
|
||||
input.path
|
||||
));
|
||||
}
|
||||
}
|
||||
// This is the embedded compiler's fixed Linux LLVM dependency, not a PATH lookup.
|
||||
if cfg!(target_os = "linux")
|
||||
&& !compiler
|
||||
.manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == "lib/libLLVM.so.18.1")
|
||||
{
|
||||
return Err("signed release is missing lib/libLLVM.so.18.1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn send_output(stream: &mut UnixStream, bytes: &[u8], started: Instant) -> Result<(), String> {
|
||||
let receipt = ArtifactReceipt {
|
||||
sha256: hex(ring::digest::digest(&ring::digest::SHA256, bytes).as_ref()),
|
||||
bytes: bytes.len() as u64,
|
||||
};
|
||||
receipt.validate_compiled()?;
|
||||
let timeout = || {
|
||||
COMPILE_TIMEOUT
|
||||
.checked_sub(started.elapsed())
|
||||
.filter(|remaining| !remaining.is_zero())
|
||||
.map(|remaining| remaining.min(Duration::from_secs(5)))
|
||||
.ok_or_else(|| "deverd compilation transfer deadline exceeded".to_owned())
|
||||
};
|
||||
stream
|
||||
.set_write_timeout(Some(timeout()?))
|
||||
.map_err(|error| error.to_string())?;
|
||||
write_frame(stream, &artifact_response(receipt))?;
|
||||
let mut remaining = bytes;
|
||||
while !remaining.is_empty() {
|
||||
stream
|
||||
.set_write_timeout(Some(timeout()?))
|
||||
.map_err(|error| error.to_string())?;
|
||||
match stream.write(&remaining[..remaining.len().min(64 * 1024)]) {
|
||||
Ok(0) => return Err("compiled output transfer closed".into()),
|
||||
Ok(written) => remaining = &remaining[written..],
|
||||
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
|
||||
Err(error) => return Err(format!("compiled output transfer failed: {error}")),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_connected(stream: &mut UnixStream, started: Instant) -> Result<(), String> {
|
||||
if started.elapsed() >= COMPILE_TIMEOUT {
|
||||
return Err("deverd compilation deadline exceeded".into());
|
||||
}
|
||||
let mut signal = [0_u8; 1];
|
||||
match stream.read(&mut signal) {
|
||||
// Any byte after the exact request body explicitly cancels this request.
|
||||
Ok(_) => Err("deverd compilation cancelled by caller".into()),
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
std::io::ErrorKind::WouldBlock
|
||||
| std::io::ErrorKind::TimedOut
|
||||
| std::io::ErrorKind::Interrupted
|
||||
) =>
|
||||
{
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(format!("deverd compilation caller disconnected: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
struct Worker(Child);
|
||||
|
||||
impl Drop for Worker {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn run_worker(
|
||||
core: &Path,
|
||||
directory: &Path,
|
||||
stream: &mut UnixStream,
|
||||
started: Instant,
|
||||
) -> Result<(), String> {
|
||||
let diagnostic = directory.join("diagnostic");
|
||||
let error_output = private_file(&diagnostic)?;
|
||||
let mut worker = Worker(
|
||||
Command::new(core)
|
||||
.arg("--dever-compile-worker")
|
||||
.env_clear()
|
||||
.current_dir(directory)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(error_output)
|
||||
.spawn()
|
||||
.map_err(|error| format!("cannot start trusted compile worker: {error}"))?,
|
||||
);
|
||||
loop {
|
||||
require_connected(stream, started)?;
|
||||
if fs::metadata(&diagnostic)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len()
|
||||
> MAX_DIAGNOSTIC_BYTES
|
||||
{
|
||||
return Err("compile worker diagnostic exceeded 64 KiB".into());
|
||||
}
|
||||
if let Some(status) = worker
|
||||
.0
|
||||
.try_wait()
|
||||
.map_err(|error| format!("cannot wait for compile worker: {error}"))?
|
||||
{
|
||||
if status.success() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
File::open(&diagnostic)
|
||||
.map_err(|error| error.to_string())?
|
||||
.take(MAX_DIAGNOSTIC_BYTES)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
return Err(format!(
|
||||
"trusted compile worker failed ({status}): {}",
|
||||
String::from_utf8_lossy(&bytes)
|
||||
));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
fn private_file(path: &Path) -> Result<File, String> {
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.mode(0o600)
|
||||
.open(path)
|
||||
.map_err(|error| format!("cannot create compilation input: {error}"))
|
||||
}
|
||||
|
||||
fn write_private(path: &Path, bytes: &[u8]) -> Result<(), String> {
|
||||
private_file(path)?
|
||||
.write_all(bytes)
|
||||
.map_err(|error| format!("cannot write compilation input: {error}"))
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
use super::super::compilation::CompileRequest;
|
||||
use super::super::release::Layout;
|
||||
|
||||
/// Submit source and compile-only bindings, never a client-chosen cache key or IR.
|
||||
pub fn compile(layout: &Layout, request: &CompileRequest) -> Result<Vec<u8>, String> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use super::{Operation, Request, Response, connect, read_frame, write_frame};
|
||||
use std::io::{Read, Write};
|
||||
use std::time::{Duration, Instant};
|
||||
let payload = request.encode()?;
|
||||
let (mut stream, token) = connect(layout)?;
|
||||
write_frame(
|
||||
&mut stream,
|
||||
&Request {
|
||||
token,
|
||||
operation: Operation::Compile {
|
||||
bytes: payload.len() as u64,
|
||||
},
|
||||
installed: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
let ready: Response = read_frame(&mut stream)?;
|
||||
if !ready.ok {
|
||||
return Err(ready
|
||||
.error
|
||||
.unwrap_or_else(|| "deverd rejected compilation".into()));
|
||||
}
|
||||
stream
|
||||
.write_all(&payload)
|
||||
.map_err(|error| format!("deverd compilation upload failed: {error}"))?;
|
||||
drop(payload);
|
||||
let mut download = CompilationDownload {
|
||||
stream: &mut stream,
|
||||
deadline: Instant::now() + Duration::from_secs(190),
|
||||
};
|
||||
let response: Response = read_frame(&mut download)?;
|
||||
if !response.ok {
|
||||
return Err(response
|
||||
.error
|
||||
.unwrap_or_else(|| "deverd compilation failed".into()));
|
||||
}
|
||||
let receipt = response
|
||||
.artifact
|
||||
.ok_or("deverd returned no compiled output identity")?;
|
||||
receipt.validate_compiled()?;
|
||||
let mut output = vec![0; receipt.bytes as usize];
|
||||
download
|
||||
.read_exact(&mut output)
|
||||
.map_err(|error| format!("deverd compilation download is incomplete: {error}"))?;
|
||||
if super::hex(ring::digest::digest(&ring::digest::SHA256, &output).as_ref())
|
||||
!= receipt.sha256
|
||||
{
|
||||
return Err("deverd compiled output failed SHA-256 verification".into());
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = (layout, request);
|
||||
Err("trusted deverd compilation is not implemented for this platform".into())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
struct CompilationDownload<'a> {
|
||||
stream: &'a mut std::os::unix::net::UnixStream,
|
||||
deadline: std::time::Instant,
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
impl std::io::Read for CompilationDownload<'_> {
|
||||
fn read(&mut self, bytes: &mut [u8]) -> std::io::Result<usize> {
|
||||
let remaining = self
|
||||
.deadline
|
||||
.checked_duration_since(std::time::Instant::now())
|
||||
.filter(|duration| !duration.is_zero())
|
||||
.ok_or_else(|| {
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"compiled output download deadline exceeded",
|
||||
)
|
||||
})?;
|
||||
self.stream.set_read_timeout(Some(remaining))?;
|
||||
self.stream.read(bytes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//! A stable AI entry resolves the same active-version journal as the launcher.
|
||||
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use super::release::MachineManager;
|
||||
|
||||
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub(super) fn path(manager: &MachineManager) -> Result<PathBuf, String> {
|
||||
let version = manager
|
||||
.active_version()?
|
||||
.ok_or("no active Dever version; install Dever first")?;
|
||||
// Like launcher dispatch, this is read-only and available to every machine user.
|
||||
manager.resolve_skill(&version)
|
||||
}
|
||||
|
||||
pub(super) fn install(manager: &MachineManager, destination: &Path) -> Result<(), String> {
|
||||
path(manager)?;
|
||||
let launcher = manager.layout().bin().join("dever");
|
||||
let launcher = launcher
|
||||
.to_str()
|
||||
.ok_or("launcher path must be UTF-8 for an AI skill")?;
|
||||
if launcher.chars().any(char::is_control) || launcher.contains('`') {
|
||||
return Err("launcher path cannot contain control characters or backticks".into());
|
||||
}
|
||||
let quoted = format!("'{}'", launcher.replace('\'', "'\\''"));
|
||||
let content = format!(
|
||||
"---\nname: dever-language\ndescription: Develop, review and debug Dever language applications (.dever and .dever.md), manage Dever Packages and Libs, and install or update Dever. Not the Go Dever framework.\n---\n\n# Dever language\n\nBefore every Dever task, run this read-only command:\n\n```sh\n{quoted} skill path\n```\n\nRead `SKILL.md` in the returned directory completely, then follow its instructions. Resolve its reference and asset paths against that directory. This entry follows the machine's active Dever version, so `dever update` and `dever use` update the compiler and development guidance together. If resolution fails, report the error; do not use stale guidance or infer language syntax. This entry grants no permission to install, update or change a project.\n"
|
||||
);
|
||||
let name = destination
|
||||
.file_name()
|
||||
.ok_or("skill destination requires a new directory name")?;
|
||||
let parent = destination
|
||||
.parent()
|
||||
.filter(|path| !path.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new("."));
|
||||
let parent = fs::canonicalize(parent)
|
||||
.map_err(|error| format!("cannot resolve skill destination parent: {error}"))?;
|
||||
let destination = parent.join(name);
|
||||
let staging = stage(&parent)?;
|
||||
let entry = staging.join("SKILL.md");
|
||||
let result = (|| {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&entry)
|
||||
.map_err(|error| format!("cannot create skill entry: {error}"))?;
|
||||
file.write_all(content.as_bytes())
|
||||
.and_then(|()| file.sync_all())
|
||||
.map_err(|error| format!("cannot persist skill entry: {error}"))?;
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use rustix::fs::{CWD, RenameFlags, renameat_with};
|
||||
renameat_with(CWD, &staging, CWD, &destination, RenameFlags::NOREPLACE)
|
||||
.map_err(|error| format!("cannot publish new skill directory: {error}"))
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
Err("skill installation is not available for this platform".into())
|
||||
}
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = fs::remove_file(&entry);
|
||||
let _ = fs::remove_dir(&staging);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn stage(parent: &Path) -> Result<PathBuf, String> {
|
||||
loop {
|
||||
let path = parent.join(format!(
|
||||
".dever-skill-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_STAGING.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
builder.mode(0o700);
|
||||
}
|
||||
match builder.create(&path) {
|
||||
Ok(()) => return Ok(path),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(format!("cannot stage skill entry: {error}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
//! Application deployment targets, independent of the compiler host identity.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)]
|
||||
pub enum BuildTarget {
|
||||
#[serde(rename = "linux-x86_64")]
|
||||
LinuxX86_64,
|
||||
#[serde(rename = "linux-aarch64")]
|
||||
LinuxAarch64,
|
||||
}
|
||||
|
||||
impl BuildTarget {
|
||||
pub fn host() -> Result<Self, String> {
|
||||
match (std::env::consts::OS, std::env::consts::ARCH) {
|
||||
("linux", "x86_64") => Ok(Self::LinuxX86_64),
|
||||
("linux", "aarch64") => Ok(Self::LinuxAarch64),
|
||||
_ => Err("native executable runtime packs are not available for this platform".into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn platform(self) -> &'static str {
|
||||
match self {
|
||||
Self::LinuxX86_64 => "linux-x86_64",
|
||||
Self::LinuxAarch64 => "linux-aarch64",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn triple(self) -> &'static str {
|
||||
match self {
|
||||
Self::LinuxX86_64 => "x86_64-unknown-linux-gnu",
|
||||
Self::LinuxAarch64 => "aarch64-unknown-linux-gnu",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for BuildTarget {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"linux-x86_64" => Ok(Self::LinuxX86_64),
|
||||
"linux-aarch64" => Ok(Self::LinuxAarch64),
|
||||
_ => Err(format!(
|
||||
"unsupported build target '{value}'; expected linux-x86_64 or linux-aarch64"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//! Hidden source compiler. It never prepares dependencies or executes a project.
|
||||
|
||||
use dever_cli::toolchain::compilation::{
|
||||
CompileKind, CompileRequest, MAX_COMPILE_BYTES, WORKER_OUTPUT_FILE, WORKER_REQUEST_FILE,
|
||||
worker_directory,
|
||||
};
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
|
||||
pub(super) fn execute() -> Result<(), String> {
|
||||
resource_limits()?;
|
||||
let (layout, version) = crate::compile::managed_compiler()?
|
||||
.ok_or("compile workers require an installed managed compiler")?;
|
||||
let directory = worker_directory(&layout)?;
|
||||
let path = directory.join(WORKER_REQUEST_FILE);
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| format!("cannot inspect compile request: {error}"))?;
|
||||
if !metadata.is_file()
|
||||
|| metadata.file_type().is_symlink()
|
||||
|| metadata.len() > MAX_COMPILE_BYTES as u64
|
||||
{
|
||||
return Err("compile request must be a bounded regular file".into());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
fs::File::open(&path)
|
||||
.and_then(|file| {
|
||||
file.take(MAX_COMPILE_BYTES as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
})
|
||||
.map_err(|error| format!("cannot read compile request: {error}"))?;
|
||||
let request = CompileRequest::decode(&bytes)?;
|
||||
drop(bytes);
|
||||
if request.version != version {
|
||||
return Err("compile request version does not match worker".into());
|
||||
}
|
||||
let sources = request.source_map()?;
|
||||
let checked = match &request.bindings {
|
||||
Some(bindings) => dever_core::check_with_bindings(&sources, bindings),
|
||||
None => dever_core::check(&sources),
|
||||
};
|
||||
let render = |errors: Vec<dever_core::diagnostic::Diagnostic>| {
|
||||
errors
|
||||
.iter()
|
||||
.map(|error| error.render(&sources))
|
||||
.collect::<String>()
|
||||
};
|
||||
let program = checked.map_err(render)?;
|
||||
if request.kind == CompileKind::Application {
|
||||
if request.sources.iter().any(|source| {
|
||||
source
|
||||
.logical_path
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.is_some_and(|name| name == "main.dever" || name == "main.dever.md")
|
||||
}) {
|
||||
return Err("application entry is generated from API, CMD, and Job declarations; remove module/main.dever".into());
|
||||
}
|
||||
let errors = program.application_errors()?;
|
||||
if !errors.is_empty() {
|
||||
return Err(render(errors));
|
||||
}
|
||||
}
|
||||
crate::compile::worker_compile(
|
||||
&program,
|
||||
&sources,
|
||||
&request,
|
||||
&directory,
|
||||
&directory.join(WORKER_OUTPUT_FILE),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn resource_limits() -> Result<(), String> {
|
||||
use rustix::process::{Resource, Rlimit, setrlimit};
|
||||
for (resource, limit) in [
|
||||
(Resource::Cpu, 120),
|
||||
(Resource::As, 2 * 1024 * 1024 * 1024),
|
||||
(Resource::Fsize, 1024 * 1024 * 1024),
|
||||
(Resource::Nofile, 128),
|
||||
(Resource::Core, 0),
|
||||
] {
|
||||
setrlimit(
|
||||
resource,
|
||||
Rlimit {
|
||||
current: Some(limit),
|
||||
maximum: Some(limit),
|
||||
},
|
||||
)
|
||||
.map_err(|error| format!("cannot restrict compiler worker resources: {error}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn resource_limits() -> Result<(), String> {
|
||||
Err("managed compiler worker resource limits are unavailable on this platform".into())
|
||||
}
|
||||
@@ -0,0 +1,833 @@
|
||||
//! Offline, per-Adapter execution trees for managed external Workers.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
use std::path::{Component, Path};
|
||||
|
||||
use dever_core::hir::{ExternalWorkerContract, Program};
|
||||
use dever_core::native::EmbeddedResource;
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::libs::{self, Ecosystem, LibSpec, LockFile, LockedLib};
|
||||
|
||||
mod command;
|
||||
mod elf;
|
||||
mod go_build;
|
||||
pub mod python_wheel;
|
||||
pub(crate) mod runtime;
|
||||
|
||||
pub(crate) const MAX_TREE_BYTES: usize = 256 * 1024 * 1024;
|
||||
pub(crate) const MAX_TREE_FILES: usize = 65_536;
|
||||
const IGNORED_SOURCE_DIRECTORIES: &[&str] = &["node_modules", "__pycache__", ".git", ".dever"];
|
||||
const PYTHON_SDK: &[u8] = include_bytes!("../../../sdk/python/dever_component.py");
|
||||
const JAVASCRIPT_SDK: &[u8] = include_bytes!("../../../sdk/javascript/dever_component.mjs");
|
||||
|
||||
const PYTHON_RUNNER: &str = r#"import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
base = Path(__file__).resolve().parent
|
||||
sys.stdout = sys.stderr
|
||||
sys.path.insert(0, str(base))
|
||||
from dever_component import Worker, main
|
||||
|
||||
sys.path.insert(0, str(base / "runtime" / "lib" / "python__PYTHON_VERSION__" / "site-packages"))
|
||||
sys.path.insert(0, str(base / "source"))
|
||||
spec = importlib.util.spec_from_file_location("dever_worker_entry", base / "source" / "__ENTRY__")
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("managed Worker entry cannot be loaded")
|
||||
entry = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(entry)
|
||||
manifest = json.loads((base / "contract.json").read_text(encoding="utf-8"))
|
||||
handlers = {operation: getattr(entry, operation) for operation in manifest["operations"]}
|
||||
main(Worker.from_manifest(manifest, handlers))
|
||||
"#;
|
||||
|
||||
const JAVASCRIPT_RUNNER: &str = r#"import { Console } from 'node:console';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { Worker, main } from './dever_component.mjs';
|
||||
|
||||
globalThis.console = new Console({ stdout: process.stderr, stderr: process.stderr });
|
||||
const manifest = JSON.parse(readFileSync(new URL('./contract.json', import.meta.url), 'utf8'));
|
||||
const entry = await import(new URL('./source/__ENTRY__', import.meta.url));
|
||||
const handlers = Object.fromEntries(manifest.operations.map(operation => [operation, entry[operation] ?? entry.default?.[operation]]));
|
||||
await main(Worker.fromManifest(manifest, handlers));
|
||||
"#;
|
||||
|
||||
pub fn prepare(
|
||||
project_root: &Path,
|
||||
program: &Program,
|
||||
resources: &[EmbeddedResource],
|
||||
) -> Result<Vec<EmbeddedResource>, String> {
|
||||
prepare_for_target(
|
||||
project_root,
|
||||
program,
|
||||
resources,
|
||||
crate::toolchain::BuildTarget::host()?,
|
||||
resources,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn prepare_for_target(
|
||||
project_root: &Path,
|
||||
program: &Program,
|
||||
resources: &[EmbeddedResource],
|
||||
target: crate::toolchain::BuildTarget,
|
||||
build_assets: &[EmbeddedResource],
|
||||
) -> Result<Vec<EmbeddedResource>, String> {
|
||||
let workers = program.external_worker_contracts();
|
||||
let commands = program.external_command_contracts();
|
||||
if workers.is_empty() && commands.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let lock = if workers.iter().any(|worker| worker.ecosystem != "exec") {
|
||||
let lock_path = project_root.join("dever.lock");
|
||||
if fs::symlink_metadata(&lock_path)
|
||||
.map_err(|error| format!("cannot inspect dever.lock: {error}"))?
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
{
|
||||
return Err("managed Worker lock file must not be a symbolic link".into());
|
||||
}
|
||||
let lock = LockFile::decode(
|
||||
&fs::read(&lock_path).map_err(|error| format!("cannot read dever.lock: {error}"))?,
|
||||
)?;
|
||||
libs::doctor(&lock)?;
|
||||
Some(lock)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut supplied = BTreeMap::new();
|
||||
for resource in resources {
|
||||
if supplied.insert(resource.path.as_str(), resource).is_some() {
|
||||
return Err(format!(
|
||||
"duplicate locked Worker resource '{}'",
|
||||
resource.path
|
||||
));
|
||||
}
|
||||
if digest(&resource.bytes) != resource.sha256 {
|
||||
return Err(format!(
|
||||
"locked Worker resource '{}' has a digest mismatch",
|
||||
resource.path
|
||||
));
|
||||
}
|
||||
}
|
||||
let mut output = Vec::new();
|
||||
let package_files = crate::packages::owned_files(project_root)?;
|
||||
for contract in commands {
|
||||
output.extend(command::prepare(
|
||||
project_root,
|
||||
&contract,
|
||||
&package_files,
|
||||
&supplied,
|
||||
target,
|
||||
)?);
|
||||
}
|
||||
for contract in workers {
|
||||
if contract.ecosystem == "exec" {
|
||||
output.extend(prepare_exec(&contract, &supplied, target)?);
|
||||
continue;
|
||||
}
|
||||
let lock = lock.as_ref().expect("managed Worker has a lock");
|
||||
let locked = lock
|
||||
.workers
|
||||
.iter()
|
||||
.find(|worker| worker.port == contract.port && worker.adapter == contract.adapter)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"managed Worker '{}.{}' is not locked",
|
||||
contract.port, contract.adapter
|
||||
)
|
||||
})?;
|
||||
if locked.entry != contract.entry
|
||||
|| locked.schema != contract.schema
|
||||
|| locked.ecosystem != contract.ecosystem
|
||||
|| locked.operations != contract.operations
|
||||
|| locked.capabilities != contract.capabilities
|
||||
{
|
||||
return Err(format!(
|
||||
"managed Worker '{}.{}' differs from its checked Port",
|
||||
contract.port, contract.adapter
|
||||
));
|
||||
}
|
||||
let requested = contract
|
||||
.libs
|
||||
.iter()
|
||||
.map(|value| value.parse::<LibSpec>())
|
||||
.collect::<Result<BTreeSet<_>, _>>()?;
|
||||
if requested != locked.libs.iter().cloned().collect() {
|
||||
return Err("managed Worker Lib roots differ from its checked Adapter".into());
|
||||
}
|
||||
output.extend(prepare_worker(
|
||||
project_root,
|
||||
&contract,
|
||||
lock,
|
||||
&supplied,
|
||||
&package_files,
|
||||
target,
|
||||
build_assets,
|
||||
)?);
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn prepare_exec(
|
||||
contract: &ExternalWorkerContract,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<EmbeddedResource>, String> {
|
||||
let entry = supplied
|
||||
.get(contract.entry.as_str())
|
||||
.ok_or("exec Worker is missing from verified resources")?;
|
||||
if !entry.executable || entry.bytes.is_empty() {
|
||||
return Err("exec Worker entry must be a nonempty executable resource".into());
|
||||
}
|
||||
runtime::validate_binary(&contract.entry, &entry.bytes, target.platform())?;
|
||||
let identity =
|
||||
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?;
|
||||
let tree = format!("workers/{}", digest(&identity));
|
||||
let executable = format!("{tree}/worker");
|
||||
let manifest = json!({
|
||||
"format": "dever-worker-launch-v1", "ecosystem": "exec", "entry": contract.entry,
|
||||
"executable": executable, "arguments": [], "working_directory": tree,
|
||||
});
|
||||
Ok(vec![
|
||||
resource(executable, entry.bytes.clone(), true),
|
||||
resource(
|
||||
format!("{}.dever-worker.json", contract.entry),
|
||||
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
fn prepare_worker(
|
||||
project_root: &Path,
|
||||
contract: &ExternalWorkerContract,
|
||||
lock: &LockFile,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
package_files: &BTreeMap<String, Vec<u8>>,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
build_assets: &[EmbeddedResource],
|
||||
) -> Result<Vec<EmbeddedResource>, String> {
|
||||
let ecosystem: Ecosystem = contract.ecosystem.parse()?;
|
||||
let identity =
|
||||
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?;
|
||||
let tree = format!("workers/{}", digest(&identity));
|
||||
let target = target.platform();
|
||||
let runtime_path = format!("lib/runtime/{}/{target}/runtime.pack", ecosystem.as_str());
|
||||
let runtime = supplied
|
||||
.get(runtime_path.as_str())
|
||||
.ok_or_else(|| format!("managed {} runtime pack is missing", ecosystem.as_str()))?;
|
||||
let locked_worker = lock
|
||||
.workers
|
||||
.iter()
|
||||
.find(|worker| worker.port == contract.port && worker.adapter == contract.adapter)
|
||||
.expect("validated Worker lock identity");
|
||||
let locked_runtime = locked_worker
|
||||
.runtime
|
||||
.as_ref()
|
||||
.ok_or("managed Worker lacks a locked runtime pack")?;
|
||||
if runtime.sha256 != locked_runtime.sha256 {
|
||||
return Err("managed Worker runtime pack differs from its lock".into());
|
||||
}
|
||||
let files = libs::archive_files("tgz", &runtime.bytes)?;
|
||||
let manifest = runtime::validate(&files, &ecosystem, target)?;
|
||||
let mut tree_files = Tree::new(tree.clone());
|
||||
let runtime_executable = if ecosystem == Ecosystem::Go {
|
||||
None
|
||||
} else {
|
||||
let executable = manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.expect("validated interpreter executable");
|
||||
for (path, bytes) in &files {
|
||||
tree_files.add(
|
||||
&format!("runtime/{path}"),
|
||||
bytes.clone(),
|
||||
path == executable,
|
||||
)?;
|
||||
}
|
||||
let path = format!("{tree}/runtime/{executable}");
|
||||
Some(path)
|
||||
};
|
||||
add_adapter_source(project_root, contract, package_files, &mut tree_files)?;
|
||||
let runner = match ecosystem {
|
||||
Ecosystem::Pip => {
|
||||
add_python_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?;
|
||||
tree_files.add("dever_component.py", PYTHON_SDK.to_vec(), false)?;
|
||||
tree_files.add(
|
||||
"runner.py",
|
||||
PYTHON_RUNNER
|
||||
.replace("__ENTRY__", entry_name(&contract.entry)?)
|
||||
.replace(
|
||||
"__PYTHON_VERSION__",
|
||||
&manifest
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.expect("validated Python markers")
|
||||
.python_version()
|
||||
.to_string(),
|
||||
)
|
||||
.into_bytes(),
|
||||
false,
|
||||
)?;
|
||||
"runner.py"
|
||||
}
|
||||
Ecosystem::Npm => {
|
||||
ensure_javascript_format(&contract.entry, &mut tree_files)?;
|
||||
add_javascript_dependencies(contract, lock, supplied, &manifest, &mut tree_files)?;
|
||||
tree_files.add("dever_component.mjs", JAVASCRIPT_SDK.to_vec(), false)?;
|
||||
tree_files.add(
|
||||
"runner.mjs",
|
||||
JAVASCRIPT_RUNNER
|
||||
.replace("__ENTRY__", entry_name(&contract.entry)?)
|
||||
.into_bytes(),
|
||||
false,
|
||||
)?;
|
||||
"runner.mjs"
|
||||
}
|
||||
Ecosystem::Go => {
|
||||
tree_files.add("contract.json", identity.clone(), false)?;
|
||||
let binary = go_build::compile_worker(
|
||||
contract,
|
||||
lock,
|
||||
supplied,
|
||||
go_build::Inputs {
|
||||
files: &files,
|
||||
manifest: manifest
|
||||
.build
|
||||
.as_ref()
|
||||
.expect("validated Go build descriptor"),
|
||||
sandbox: build_assets,
|
||||
},
|
||||
&mut tree_files,
|
||||
target,
|
||||
)?;
|
||||
tree_files.remove_prefix("source/");
|
||||
let launch = json!({
|
||||
"format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem,
|
||||
"entry": contract.entry, "executable": format!("{tree}/worker{}", std::env::consts::EXE_SUFFIX),
|
||||
"arguments": [], "working_directory": tree,
|
||||
});
|
||||
let mut output = tree_files.finish();
|
||||
output.push(resource(
|
||||
format!("{}.dever-worker.json", contract.entry),
|
||||
serde_json::to_vec(&launch).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
));
|
||||
debug_assert_eq!(binary, format!("worker{}", std::env::consts::EXE_SUFFIX));
|
||||
return Ok(output);
|
||||
}
|
||||
};
|
||||
tree_files.add("contract.json", identity, false)?;
|
||||
let runner_path = format!("{tree}/{runner}");
|
||||
let arguments = manifest
|
||||
.arguments
|
||||
.iter()
|
||||
.map(|value| json!({"literal": value}))
|
||||
.chain(std::iter::once(json!({"resource": runner_path})))
|
||||
.collect::<Vec<_>>();
|
||||
let launch = json!({
|
||||
"format": "dever-worker-launch-v1", "ecosystem": contract.ecosystem,
|
||||
"entry": contract.entry, "executable": runtime_executable.expect("interpreter branch"),
|
||||
"arguments": arguments, "working_directory": tree,
|
||||
});
|
||||
let mut output = tree_files.finish();
|
||||
output.push(resource(
|
||||
format!("{}.dever-worker.json", contract.entry),
|
||||
serde_json::to_vec(&launch).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
));
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn digest(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn resource(path: String, bytes: Vec<u8>, executable: bool) -> EmbeddedResource {
|
||||
EmbeddedResource {
|
||||
sha256: digest(&bytes),
|
||||
path,
|
||||
bytes,
|
||||
executable,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn valid_path(path: &str) -> Result<(), String> {
|
||||
let relative = Path::new(path);
|
||||
if path.is_empty()
|
||||
|| path.contains(['\\', ':', '\0'])
|
||||
|| relative.is_absolute()
|
||||
|| relative
|
||||
.components()
|
||||
.any(|part| !matches!(part, Component::Normal(_)))
|
||||
|| path.split('/').any(str::is_empty)
|
||||
{
|
||||
return Err(format!(
|
||||
"managed Worker path '{path}' is not portable and relative"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn entry_name(entry: &str) -> Result<&str, String> {
|
||||
valid_path(entry)?;
|
||||
let name = Path::new(entry)
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or("invalid managed Worker entry name")?;
|
||||
if !name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
|
||||
{
|
||||
return Err("managed Worker entry name contains an unsafe character".into());
|
||||
}
|
||||
Ok(name)
|
||||
}
|
||||
|
||||
fn ensure_javascript_format(entry: &str, tree: &mut Tree) -> Result<(), String> {
|
||||
if !entry_name(entry)?.ends_with(".js") {
|
||||
return Ok(());
|
||||
}
|
||||
let path = format!("{}/source/package.json", tree.prefix);
|
||||
if let Some(package) = tree.files.get(&path) {
|
||||
let metadata: serde_json::Value = serde_json::from_slice(&package.bytes)
|
||||
.map_err(|error| format!("invalid Adapter package.json: {error}"))?;
|
||||
if !matches!(
|
||||
metadata.get("type").and_then(|value| value.as_str()),
|
||||
Some("module" | "commonjs")
|
||||
) {
|
||||
return Err(
|
||||
"managed .js Worker entry requires package.json type module or commonjs".into(),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
tree.add(
|
||||
"source/package.json",
|
||||
b"{\"type\":\"module\"}\n".to_vec(),
|
||||
false,
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_adapter_source(
|
||||
project_root: &Path,
|
||||
contract: &ExternalWorkerContract,
|
||||
package_files: &BTreeMap<String, Vec<u8>>,
|
||||
tree: &mut Tree,
|
||||
) -> Result<(), String> {
|
||||
let name = entry_name(&contract.entry)?;
|
||||
let allowed: &[&str] = match contract.ecosystem.as_str() {
|
||||
"pip" if name.ends_with(".py") => &["py", "pyi", "json"],
|
||||
"npm" if name.ends_with(".js") || name.ends_with(".mjs") || name.ends_with(".cjs") => {
|
||||
&["js", "mjs", "cjs", "json"]
|
||||
}
|
||||
"go" if name.ends_with(".go") => &[],
|
||||
_ => return Err("managed Worker entry has an unsupported language extension".into()),
|
||||
};
|
||||
let package_owned = crate::packages::owns_worker(project_root, &contract.entry)?;
|
||||
if add_package_adapter_source(&contract.entry, allowed, package_files, tree, package_owned)? {
|
||||
return Ok(());
|
||||
}
|
||||
let entry = libs::owned_worker(project_root, &contract.entry)?;
|
||||
let parent = entry
|
||||
.parent()
|
||||
.ok_or("managed Worker entry has no source directory")?;
|
||||
let mut pending = vec![parent.to_path_buf()];
|
||||
while let Some(directory) = pending.pop() {
|
||||
let mut children = fs::read_dir(&directory)
|
||||
.map_err(|error| format!("cannot read Adapter source: {error}"))?
|
||||
.map(|result| {
|
||||
result
|
||||
.map(|entry| entry.path())
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
children.sort();
|
||||
for path in children {
|
||||
let metadata = fs::symlink_metadata(&path)
|
||||
.map_err(|error| format!("cannot inspect Adapter source: {error}"))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err("Adapter source contains a symbolic link".into());
|
||||
}
|
||||
let relative = path
|
||||
.strip_prefix(parent)
|
||||
.map_err(|error| error.to_string())?
|
||||
.to_str()
|
||||
.ok_or("Adapter source path is not UTF-8")?
|
||||
.replace(std::path::MAIN_SEPARATOR, "/");
|
||||
valid_path(&relative)?;
|
||||
if metadata.is_dir() {
|
||||
if !path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| IGNORED_SOURCE_DIRECTORIES.contains(&name))
|
||||
{
|
||||
pending.push(path);
|
||||
}
|
||||
} else if metadata.is_file() {
|
||||
if allowed.is_empty()
|
||||
|| path
|
||||
.extension()
|
||||
.and_then(|value| value.to_str())
|
||||
.is_some_and(|extension| allowed.contains(&extension))
|
||||
{
|
||||
tree.add(
|
||||
&format!("source/{relative}"),
|
||||
fs::read(&path)
|
||||
.map_err(|error| format!("cannot read Adapter source: {error}"))?,
|
||||
false,
|
||||
)?;
|
||||
}
|
||||
} else {
|
||||
return Err("Adapter source contains a non-regular file".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_package_adapter_source(
|
||||
entry: &str,
|
||||
allowed: &[&str],
|
||||
package_files: &BTreeMap<String, Vec<u8>>,
|
||||
tree: &mut Tree,
|
||||
package_owned: bool,
|
||||
) -> Result<bool, String> {
|
||||
// Ownership comes from the checked Package lock, not from file presence:
|
||||
// a missing entire source directory must never enable a local fallback.
|
||||
if !package_owned {
|
||||
return Ok(false);
|
||||
}
|
||||
if !package_files.contains_key(entry) {
|
||||
return Err(format!("locked Package Worker entry '{entry}' is missing"));
|
||||
}
|
||||
let parent = Path::new(entry)
|
||||
.parent()
|
||||
.and_then(|path| path.to_str())
|
||||
.ok_or("Package Worker entry has no source directory")?;
|
||||
let prefix = format!("{parent}/");
|
||||
for (path, bytes) in package_files {
|
||||
let Some(relative) = path.strip_prefix(&prefix) else {
|
||||
continue;
|
||||
};
|
||||
valid_path(relative)?;
|
||||
if relative
|
||||
.split('/')
|
||||
.any(|part| IGNORED_SOURCE_DIRECTORIES.contains(&part))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if allowed.is_empty()
|
||||
|| Path::new(relative)
|
||||
.extension()
|
||||
.and_then(|extension| extension.to_str())
|
||||
.is_some_and(|extension| allowed.contains(&extension))
|
||||
{
|
||||
tree.add(&format!("source/{relative}"), bytes.clone(), false)?;
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
fn dependency_closure<'a>(
|
||||
contract: &ExternalWorkerContract,
|
||||
lock: &'a LockFile,
|
||||
) -> Result<Vec<&'a LockedLib>, String> {
|
||||
let mut remaining = contract
|
||||
.libs
|
||||
.iter()
|
||||
.map(|value| value.parse::<LibSpec>())
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut visited = BTreeSet::new();
|
||||
let mut closure = Vec::new();
|
||||
while let Some(spec) = remaining.pop() {
|
||||
if !visited.insert(spec.clone()) {
|
||||
continue;
|
||||
}
|
||||
if spec.ecosystem.as_str() != contract.ecosystem {
|
||||
return Err("Worker Lib ecosystem differs from Adapter".into());
|
||||
}
|
||||
let lib = lock
|
||||
.libs
|
||||
.iter()
|
||||
.find(|lib| lib.spec == spec)
|
||||
.ok_or_else(|| format!("locked Worker Lib {} is missing", spec.key()))?;
|
||||
remaining.extend(
|
||||
lib.dependencies
|
||||
.iter()
|
||||
.map(|dependency| dependency.spec.clone()),
|
||||
);
|
||||
closure.push(lib);
|
||||
}
|
||||
closure.sort_by(|left, right| left.spec.cmp(&right.spec));
|
||||
Ok(closure)
|
||||
}
|
||||
|
||||
fn locked_archive<'a>(
|
||||
lib: &LockedLib,
|
||||
extension: &str,
|
||||
supplied: &'a BTreeMap<&str, &EmbeddedResource>,
|
||||
target: &str,
|
||||
) -> Result<&'a [u8], String> {
|
||||
let artifact = lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| {
|
||||
artifact.target == target && artifact.path.ends_with(&format!(".{extension}"))
|
||||
})
|
||||
.ok_or_else(|| format!("locked {} archive for {target} is missing", lib.spec.key()))?;
|
||||
let path = format!(
|
||||
"lib/{}/{}/{}/archive.{extension}",
|
||||
lib.spec.ecosystem.as_str(),
|
||||
digest(lib.spec.distribution_name().as_bytes()),
|
||||
lib.spec.version
|
||||
);
|
||||
if artifact.path != path {
|
||||
return Err(format!(
|
||||
"locked Worker archive path differs from the {} installation layout",
|
||||
lib.spec.key()
|
||||
));
|
||||
}
|
||||
let resource = supplied
|
||||
.get(path.as_str())
|
||||
.ok_or_else(|| format!("locked Worker archive '{path}' is missing"))?;
|
||||
if resource.sha256 != artifact.sha256 || resource.bytes.len() as u64 != artifact.bytes {
|
||||
return Err(format!(
|
||||
"locked Worker archive '{path}' differs from its lock"
|
||||
));
|
||||
}
|
||||
Ok(&resource.bytes)
|
||||
}
|
||||
|
||||
fn add_python_dependencies(
|
||||
contract: &ExternalWorkerContract,
|
||||
lock: &LockFile,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
runtime: &runtime::Manifest,
|
||||
tree: &mut Tree,
|
||||
) -> Result<(), String> {
|
||||
let mut installed_distributions = BTreeMap::<&str, &LockedLib>::new();
|
||||
let mut wheel_files = BTreeSet::new();
|
||||
for lib in dependency_closure(contract, lock)? {
|
||||
let wheel = python_wheel::Wheel::parse(
|
||||
&lib.spec,
|
||||
None,
|
||||
&runtime.python_wheel_tags,
|
||||
locked_archive(lib, "whl", supplied, &runtime.target)?,
|
||||
)?;
|
||||
wheel.validate_dependencies(
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.expect("validated Python markers"),
|
||||
)?;
|
||||
libs::build::validate_python_policy(
|
||||
lock,
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.expect("validated Python markers"),
|
||||
)?;
|
||||
if let Some(previous) = installed_distributions.insert(lib.spec.distribution_name(), lib) {
|
||||
if previous.spec.version != lib.spec.version || previous.artifacts != lib.artifacts {
|
||||
return Err("Python extras refer to conflicting distribution artifacts".into());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
for file in wheel.install(
|
||||
&format!(
|
||||
"runtime/{}",
|
||||
runtime
|
||||
.executable
|
||||
.as_deref()
|
||||
.expect("validated Python interpreter")
|
||||
),
|
||||
&runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.expect("validated Python markers")
|
||||
.python_version()
|
||||
.to_string(),
|
||||
)? {
|
||||
wheel_files.insert(file.path.clone());
|
||||
tree.add(&file.path, file.bytes, file.executable)?;
|
||||
}
|
||||
}
|
||||
validate_worker_native(
|
||||
runtime,
|
||||
tree,
|
||||
supplied,
|
||||
&wheel_files,
|
||||
&runtime.python_extension_suffixes,
|
||||
)
|
||||
}
|
||||
|
||||
fn validate_worker_native(
|
||||
runtime: &runtime::Manifest,
|
||||
tree: &Tree,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
entries: &BTreeSet<String>,
|
||||
suffixes: &[String],
|
||||
) -> Result<(), String> {
|
||||
let prefix = format!("{}/", tree.prefix);
|
||||
let mut files = tree
|
||||
.files
|
||||
.iter()
|
||||
.map(|(path, resource)| {
|
||||
(
|
||||
path.strip_prefix(&prefix)
|
||||
.expect("owned Worker tree")
|
||||
.to_owned(),
|
||||
resource.bytes.as_slice(),
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
files.extend(
|
||||
supplied
|
||||
.iter()
|
||||
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
|
||||
.map(|(path, resource)| ((*path).to_owned(), resource.bytes.as_slice())),
|
||||
);
|
||||
python_wheel::validate_native(
|
||||
&runtime.target,
|
||||
suffixes,
|
||||
&format!(
|
||||
"runtime/{}",
|
||||
runtime
|
||||
.executable
|
||||
.as_deref()
|
||||
.expect("validated interpreter")
|
||||
),
|
||||
&files,
|
||||
entries,
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn add_javascript_dependencies(
|
||||
contract: &ExternalWorkerContract,
|
||||
lock: &LockFile,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
runtime: &runtime::Manifest,
|
||||
tree: &mut Tree,
|
||||
) -> Result<(), String> {
|
||||
let requested = contract
|
||||
.libs
|
||||
.iter()
|
||||
.map(|value| value.parse())
|
||||
.collect::<Result<Vec<LibSpec>, _>>()?;
|
||||
let Some(environment) = libs::npm::environment(lock, &requested)? else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(receipt) = libs::build::npm_receipt(lock, environment)? {
|
||||
if receipt.output.target != runtime.target || receipt.tools.target != runtime.target {
|
||||
return Err("locked npm build receipt differs from the selected Worker target".into());
|
||||
}
|
||||
let resource = supplied
|
||||
.get(receipt.output.path.as_str())
|
||||
.ok_or("locked npm installation resource is missing")?;
|
||||
for (path, bytes, executable) in
|
||||
libs::build::npm_installation(lock, environment, &resource.bytes)?
|
||||
{
|
||||
tree.add(&path, bytes, executable)?;
|
||||
}
|
||||
let entries = receipt
|
||||
.native_entries
|
||||
.iter()
|
||||
.filter(|path| {
|
||||
libs::npm::installed_owner(path)
|
||||
.is_ok_and(|owner| environment.instances.iter().any(|node| node.path == owner))
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
return validate_worker_native(runtime, tree, supplied, &entries, &[]);
|
||||
}
|
||||
for (path, bytes) in libs::npm::files(environment, lock, |lib| {
|
||||
locked_archive(lib, "tgz", supplied, &runtime.target)
|
||||
})? {
|
||||
tree.add(&path, bytes, false)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Tree {
|
||||
prefix: String,
|
||||
files: BTreeMap<String, EmbeddedResource>,
|
||||
bytes: usize,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
fn new(prefix: String) -> Self {
|
||||
Self {
|
||||
prefix,
|
||||
files: BTreeMap::new(),
|
||||
bytes: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&mut self, relative: &str, bytes: Vec<u8>, executable: bool) -> Result<(), String> {
|
||||
valid_path(relative)?;
|
||||
let total = self
|
||||
.bytes
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("managed Worker tree size overflow")?;
|
||||
if total > MAX_TREE_BYTES || self.files.len() >= MAX_TREE_FILES {
|
||||
return Err("managed Worker tree exceeds file or byte budget".into());
|
||||
}
|
||||
let path = format!("{}/{relative}", self.prefix);
|
||||
if self.files.contains_key(&path) {
|
||||
return Err(format!("managed Worker tree repeats '{path}'"));
|
||||
}
|
||||
let mut ancestor = path.as_str();
|
||||
while let Some((parent, _)) = ancestor.rsplit_once('/') {
|
||||
if self.files.contains_key(parent) {
|
||||
return Err(format!(
|
||||
"managed Worker tree has a file/directory conflict at '{parent}'"
|
||||
));
|
||||
}
|
||||
ancestor = parent;
|
||||
}
|
||||
let descendant_prefix = format!("{path}/");
|
||||
if self
|
||||
.files
|
||||
.range(descendant_prefix.clone()..)
|
||||
.next()
|
||||
.is_some_and(|(existing, _)| existing.starts_with(&descendant_prefix))
|
||||
{
|
||||
return Err(format!(
|
||||
"managed Worker tree has a file/directory conflict at '{path}'"
|
||||
));
|
||||
}
|
||||
self.bytes = total;
|
||||
self.files
|
||||
.insert(path.clone(), resource(path, bytes, executable));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn finish(self) -> Vec<EmbeddedResource> {
|
||||
self.files.into_values().collect()
|
||||
}
|
||||
|
||||
fn remove_prefix(&mut self, relative: &str) {
|
||||
let prefix = format!("{}/{relative}", self.prefix);
|
||||
self.files.retain(|path, resource| {
|
||||
if path.starts_with(&prefix) {
|
||||
self.bytes -= resource.bytes.len();
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
//! Offline native command trees, including their explicitly supplied libraries.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File};
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use dever_core::hir::ExternalWorkerContract;
|
||||
use dever_core::native::EmbeddedResource;
|
||||
use goblin::elf::{dynamic, header};
|
||||
use serde_json::json;
|
||||
|
||||
use super::{MAX_TREE_BYTES, MAX_TREE_FILES, Tree, digest, elf, resource, runtime, valid_path};
|
||||
use crate::toolchain::BuildTarget;
|
||||
|
||||
pub(super) fn prepare(
|
||||
root: &Path,
|
||||
contract: &ExternalWorkerContract,
|
||||
package_files: &BTreeMap<String, Vec<u8>>,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
target: BuildTarget,
|
||||
) -> Result<Vec<EmbeddedResource>, String> {
|
||||
let prefix = format!(
|
||||
"commands/{}",
|
||||
digest(&serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?)
|
||||
);
|
||||
let mut tree = Tree::new(prefix.clone());
|
||||
let source = command_files(root, &contract.entry, package_files)?;
|
||||
let binary = &source["program/tool"];
|
||||
runtime::validate_binary(&contract.entry, binary, target.platform())?;
|
||||
let machine = match target {
|
||||
BuildTarget::LinuxX86_64 => header::EM_X86_64,
|
||||
BuildTarget::LinuxAarch64 => header::EM_AARCH64,
|
||||
};
|
||||
let executable = elf::object("program/tool", binary, machine)?;
|
||||
let interpreter = executable.interpreter.map(str::to_owned);
|
||||
if interpreter.is_none() && !executable.libraries.is_empty() {
|
||||
return Err("command executable has dynamic dependencies without an interpreter".into());
|
||||
}
|
||||
let mut files = source
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
files.extend(
|
||||
supplied
|
||||
.iter()
|
||||
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
|
||||
.map(|(path, entry)| ((*path).to_owned(), entry.bytes.as_slice())),
|
||||
);
|
||||
let closure = elf::Closure {
|
||||
files: &files,
|
||||
machine,
|
||||
directories: vec!["program/lib".into(), "sandbox/lib".into()],
|
||||
};
|
||||
for (path, bytes) in &source {
|
||||
let object = elf::object(path, bytes, machine)?;
|
||||
reject_loader_hooks(&object)?;
|
||||
// The logical program root becomes /worker in the sandbox. A path
|
||||
// escaping it could otherwise resolve a library from a file grant.
|
||||
for fields in [&object.rpaths, &object.runpaths] {
|
||||
for directory in elf::origin_directories(path, fields)? {
|
||||
if directory != "program" && !directory.starts_with("program/") {
|
||||
return Err("command library search path escapes its packaged tree".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(name) = path.strip_prefix("program/lib/") {
|
||||
if supplied.contains_key(format!("sandbox/lib/{name}").as_str()) {
|
||||
return Err(format!(
|
||||
"command library '{name}' overrides a signed sandbox library"
|
||||
));
|
||||
}
|
||||
if object.header.e_type != header::ET_DYN
|
||||
|| object.interpreter.is_some()
|
||||
|| object.soname.is_some_and(|soname| soname != name)
|
||||
{
|
||||
return Err(format!(
|
||||
"command library '{name}' is not a matching SONAME shared object"
|
||||
));
|
||||
}
|
||||
}
|
||||
closure.follow(path, Vec::new(), &BTreeMap::new())?;
|
||||
}
|
||||
for (path, bytes) in source {
|
||||
let name = path.strip_prefix("program/").expect("owned command tree");
|
||||
tree.add(name, bytes, name == "tool")?;
|
||||
}
|
||||
let (executable, arguments) = if let Some(interpreter) = interpreter {
|
||||
let name = interpreter
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.expect("validated GNU loader");
|
||||
let loader = supplied
|
||||
.get(format!("sandbox/lib/{name}").as_str())
|
||||
.ok_or("command requires the signed target GNU loader")?;
|
||||
runtime::validate_binary(name, &loader.bytes, target.platform())?;
|
||||
tree.add("loader", loader.bytes.clone(), true)?;
|
||||
(
|
||||
format!("{prefix}/loader"),
|
||||
vec![
|
||||
json!({"literal": "--inhibit-cache"}),
|
||||
json!({"literal": "--library-path"}),
|
||||
json!({"literal": "lib:/lib"}),
|
||||
json!({"resource": format!("{prefix}/tool")}),
|
||||
],
|
||||
)
|
||||
} else {
|
||||
(format!("{prefix}/tool"), Vec::new())
|
||||
};
|
||||
let manifest = json!({
|
||||
"format": "dever-command-launch-v1", "ecosystem": "command",
|
||||
"entry": contract.entry, "executable": executable,
|
||||
"arguments": arguments, "working_directory": prefix,
|
||||
});
|
||||
let mut output = tree.finish();
|
||||
output.push(resource(
|
||||
format!("{}.dever-command.json", contract.entry),
|
||||
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
));
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn reject_loader_hooks(elf: &goblin::elf::Elf<'_>) -> Result<(), String> {
|
||||
if elf.dynamic.as_ref().is_some_and(|table| {
|
||||
table.dyns.iter().any(|entry| {
|
||||
matches!(
|
||||
entry.d_tag,
|
||||
dynamic::DT_CONFIG
|
||||
| dynamic::DT_AUDIT
|
||||
| dynamic::DT_DEPAUDIT
|
||||
| 0x7fff_fffd
|
||||
| 0x7fff_ffff
|
||||
)
|
||||
})
|
||||
}) {
|
||||
return Err("command ELF contains unsupported dynamic loader hooks".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn command_files(
|
||||
root: &Path,
|
||||
entry: &str,
|
||||
package_files: &BTreeMap<String, Vec<u8>>,
|
||||
) -> Result<BTreeMap<String, Vec<u8>>, String> {
|
||||
valid_path(entry)?;
|
||||
let parent = entry
|
||||
.rsplit_once('/')
|
||||
.map(|(parent, _)| parent)
|
||||
.ok_or("command entry must belong to a domain")?;
|
||||
let library_prefix = format!("{parent}/lib/");
|
||||
let packaged = crate::packages::owns_worker(root, entry)?;
|
||||
let binary = if packaged {
|
||||
package_files
|
||||
.get(entry)
|
||||
.cloned()
|
||||
.ok_or("Package command executable is missing")?
|
||||
} else {
|
||||
read_file(root, entry)?
|
||||
};
|
||||
let mut files = BTreeMap::from([("program/tool".to_owned(), binary)]);
|
||||
let mut bytes = files["program/tool"].len();
|
||||
if bytes == 0 || bytes > MAX_TREE_BYTES {
|
||||
return Err("command executable is empty or exceeds the byte budget".into());
|
||||
}
|
||||
let names = if packaged {
|
||||
package_files
|
||||
.keys()
|
||||
.filter_map(|path| path.strip_prefix(&library_prefix))
|
||||
.map(str::to_owned)
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
library_names(&root.join(&library_prefix))?
|
||||
};
|
||||
for name in names {
|
||||
valid_path(&name)?;
|
||||
if name.contains('/') || files.len() >= MAX_TREE_FILES {
|
||||
return Err("command lib directory must contain bounded regular library files".into());
|
||||
}
|
||||
let path = format!("{library_prefix}{name}");
|
||||
let library = if packaged {
|
||||
package_files[&path].clone()
|
||||
} else {
|
||||
read_file(root, &path)?
|
||||
};
|
||||
bytes = bytes
|
||||
.checked_add(library.len())
|
||||
.ok_or("command library size overflow")?;
|
||||
if bytes > MAX_TREE_BYTES {
|
||||
return Err("command libraries exceed the byte budget".into());
|
||||
}
|
||||
files.insert(format!("program/lib/{name}"), library);
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn library_names(directory: &Path) -> Result<Vec<String>, String> {
|
||||
let metadata = match fs::symlink_metadata(directory) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
|
||||
Err(error) => return Err(format!("cannot inspect command lib directory: {error}")),
|
||||
};
|
||||
if !metadata.is_dir() || metadata.file_type().is_symlink() {
|
||||
return Err("command lib directory must be a real directory".into());
|
||||
}
|
||||
let mut names = Vec::new();
|
||||
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
if names.len() >= MAX_TREE_FILES
|
||||
|| !entry
|
||||
.file_type()
|
||||
.map_err(|error| error.to_string())?
|
||||
.is_file()
|
||||
{
|
||||
return Err("command lib directory must contain bounded regular library files".into());
|
||||
}
|
||||
names.push(
|
||||
entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "command library name is not UTF-8")?,
|
||||
);
|
||||
}
|
||||
names.sort();
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
fn read_file(root: &Path, entry: &str) -> Result<Vec<u8>, String> {
|
||||
let path = crate::libs::owned_worker(root, entry)?;
|
||||
let mut bytes = Vec::new();
|
||||
File::open(path)
|
||||
.and_then(|file| file.take(MAX_TREE_BYTES as u64 + 1).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read command resource '{entry}': {error}"))?;
|
||||
if bytes.len() > MAX_TREE_BYTES {
|
||||
return Err("command resource exceeds the byte budget".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Shared ELF architecture, loader search and versioned dependency closure.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
|
||||
use goblin::elf::{Elf, header};
|
||||
|
||||
pub(crate) fn object<'a>(path: &str, bytes: &'a [u8], machine: u16) -> Result<Elf<'a>, String> {
|
||||
let elf = Elf::parse(bytes).map_err(|error| format!("invalid native ELF '{path}': {error}"))?;
|
||||
if !elf.is_64
|
||||
|| !elf.little_endian
|
||||
|| elf.header.e_machine != machine
|
||||
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
|
||||
{
|
||||
return Err(format!(
|
||||
"native ELF '{path}' has an incompatible target or object type"
|
||||
));
|
||||
}
|
||||
Ok(elf)
|
||||
}
|
||||
|
||||
pub(crate) fn origin_directories(path: &str, fields: &[&str]) -> Result<Vec<String>, String> {
|
||||
let parent = path.rsplit_once('/').map_or("", |(parent, _)| parent);
|
||||
let mut result = Vec::new();
|
||||
for field in fields {
|
||||
for directory in field.split(':') {
|
||||
let relative = directory
|
||||
.strip_prefix("$ORIGIN")
|
||||
.or_else(|| directory.strip_prefix("${ORIGIN}"))
|
||||
.ok_or("native library search path must be relative to $ORIGIN")?;
|
||||
if !relative.is_empty() && !relative.starts_with('/')
|
||||
|| relative.contains(['$', '\\', '\0'])
|
||||
{
|
||||
return Err("native library search path has an unsupported token".into());
|
||||
}
|
||||
let combined = format!("{parent}{relative}");
|
||||
let mut normalized = Vec::new();
|
||||
for part in combined.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
normalized
|
||||
.pop()
|
||||
.ok_or("native library path escapes the installation tree")?;
|
||||
}
|
||||
part => normalized.push(part),
|
||||
}
|
||||
}
|
||||
let normalized = normalized.join("/");
|
||||
if normalized.is_empty() || normalized.starts_with("sandbox/") {
|
||||
return Err("native library path escapes its Worker installation".into());
|
||||
}
|
||||
if !result.contains(&normalized) {
|
||||
result.push(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn plain_library(name: &str) -> Result<(), String> {
|
||||
if name.is_empty() || matches!(name, "." | "..") || name.contains(['/', '\\', '\0']) {
|
||||
return Err("native DT_NEEDED must be a plain library name".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_versions(consumer: &Elf<'_>, library: &str, provider: &Elf<'_>) -> Result<(), String> {
|
||||
let mut provided = BTreeSet::new();
|
||||
if let Some(definitions) = &provider.verdef {
|
||||
for definition in definitions.iter() {
|
||||
for name in definition.iter() {
|
||||
provided.insert(
|
||||
provider
|
||||
.dynstrtab
|
||||
.get_at(name.vda_name)
|
||||
.ok_or("invalid ELF version definition")?
|
||||
.to_owned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(requirements) = &consumer.verneed {
|
||||
for requirement in requirements.iter() {
|
||||
if consumer.dynstrtab.get_at(requirement.vn_file) != Some(library) {
|
||||
continue;
|
||||
}
|
||||
for version in requirement.iter() {
|
||||
let name = consumer
|
||||
.dynstrtab
|
||||
.get_at(version.vna_name)
|
||||
.ok_or("invalid ELF version requirement")?;
|
||||
if version.vna_flags & 2 == 0 && !provided.contains(name) {
|
||||
return Err(format!(
|
||||
"packaged library '{library}' does not provide native ABI version '{name}'"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) struct Closure<'a> {
|
||||
pub(crate) files: &'a BTreeMap<String, &'a [u8]>,
|
||||
pub(crate) machine: u16,
|
||||
pub(crate) directories: Vec<String>,
|
||||
}
|
||||
|
||||
impl Closure<'_> {
|
||||
pub(crate) fn follow(
|
||||
&self,
|
||||
start: &str,
|
||||
inherited: Vec<String>,
|
||||
globals: &BTreeMap<String, String>,
|
||||
) -> Result<(BTreeMap<String, String>, BTreeSet<String>), String> {
|
||||
let mut loaded = globals.clone();
|
||||
let mut pending = VecDeque::from([(start.to_owned(), inherited)]);
|
||||
let mut seen = BTreeSet::new();
|
||||
while let Some((path, inherited)) = pending.pop_front() {
|
||||
if !seen.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let bytes = self
|
||||
.files
|
||||
.get(&path)
|
||||
.ok_or_else(|| format!("native dependency '{path}' is absent"))?;
|
||||
let elf = object(&path, bytes, self.machine)?;
|
||||
let rpaths = if elf.runpaths.is_empty() {
|
||||
origin_directories(&path, &elf.rpaths)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let runpaths = origin_directories(&path, &elf.runpaths)?;
|
||||
let mut inherited_next = rpaths.clone();
|
||||
inherited_next.extend(inherited.clone());
|
||||
let inherited_search = if elf.runpaths.is_empty() {
|
||||
inherited.as_slice()
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
for name in &elf.libraries {
|
||||
plain_library(name)?;
|
||||
let supplied = loaded
|
||||
.get(*name)
|
||||
.cloned()
|
||||
.or_else(|| {
|
||||
rpaths
|
||||
.iter()
|
||||
.chain(inherited_search.iter())
|
||||
.chain(runpaths.iter())
|
||||
.map(|directory| format!("{directory}/{name}"))
|
||||
.chain(
|
||||
self.directories
|
||||
.iter()
|
||||
.map(|directory| format!("{directory}/{name}")),
|
||||
)
|
||||
.find(|candidate| self.files.contains_key(candidate))
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
format!("native library '{name}' is not in the packaged dependency closure")
|
||||
})?;
|
||||
let provider = object(&supplied, self.files[&supplied], self.machine)?;
|
||||
validate_versions(&elf, name, &provider)?;
|
||||
loaded.insert((*name).to_owned(), supplied.clone());
|
||||
if let Some(soname) = provider.soname {
|
||||
plain_library(soname)?;
|
||||
if soname != *name {
|
||||
return Err("native library SONAME differs from DT_NEEDED".into());
|
||||
}
|
||||
}
|
||||
pending.push_back((supplied, inherited_next.clone()));
|
||||
}
|
||||
}
|
||||
Ok((loaded, seen))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,574 @@
|
||||
//! Compile a checked Go Adapter with the exact tools and archives in its locked pack.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use dever_core::hir::ExternalWorkerContract;
|
||||
use dever_core::native::EmbeddedResource;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
|
||||
use super::{Tree, dependency_closure, digest, entry_name, locked_archive, valid_path};
|
||||
use crate::libs::build::process::Stage;
|
||||
use crate::libs::{self, LockFile};
|
||||
|
||||
const SDK: &[u8] = include_bytes!("../../../../sdk/go/component.go");
|
||||
|
||||
#[derive(Clone, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct GoBuildManifest {
|
||||
host: crate::toolchain::BuildTarget,
|
||||
compiler: String,
|
||||
linker: String,
|
||||
analyzer: String,
|
||||
stdlib_importcfg: String,
|
||||
goos: String,
|
||||
goarch: String,
|
||||
go_version: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SelectedPackage {
|
||||
name: String,
|
||||
files: Vec<String>,
|
||||
imports: Vec<String>,
|
||||
embed: BTreeMap<String, Vec<String>>,
|
||||
}
|
||||
|
||||
struct Builder {
|
||||
stage: Stage,
|
||||
input: PathBuf,
|
||||
assets: PathBuf,
|
||||
grants: Vec<dever_sandbox::Grant>,
|
||||
tools: GoBuildManifest,
|
||||
entry: String,
|
||||
stdlib: BTreeMap<String, PathBuf>,
|
||||
modules: BTreeMap<String, PathBuf>,
|
||||
archives: BTreeMap<String, PathBuf>,
|
||||
visiting: BTreeSet<String>,
|
||||
}
|
||||
|
||||
pub(super) struct Inputs<'a> {
|
||||
pub files: &'a [(String, Vec<u8>)],
|
||||
pub manifest: &'a GoBuildManifest,
|
||||
pub sandbox: &'a [EmbeddedResource],
|
||||
}
|
||||
|
||||
pub(super) fn compile_worker(
|
||||
contract: &ExternalWorkerContract,
|
||||
lock: &LockFile,
|
||||
supplied: &BTreeMap<&str, &EmbeddedResource>,
|
||||
inputs: Inputs<'_>,
|
||||
tree: &mut Tree,
|
||||
target: &str,
|
||||
) -> Result<String, String> {
|
||||
let Inputs {
|
||||
files: pack_files,
|
||||
manifest: build,
|
||||
sandbox: build_assets,
|
||||
} = inputs;
|
||||
let references = build.validate_files(pack_files, target)?;
|
||||
if build.host != crate::toolchain::BuildTarget::host()? {
|
||||
return Err("Go build tools require a different build host".into());
|
||||
}
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
let assets = stage.0.join("sandbox");
|
||||
for resource in build_assets
|
||||
.iter()
|
||||
.filter(|resource| resource.path.starts_with("sandbox/"))
|
||||
{
|
||||
if digest(&resource.bytes) != resource.sha256 {
|
||||
return Err("Go build sandbox asset has a digest mismatch".into());
|
||||
}
|
||||
stage.write(&resource.path, &resource.bytes, resource.executable)?;
|
||||
}
|
||||
let build_directory = stage.0.join("build");
|
||||
fs::create_dir(&build_directory)
|
||||
.map_err(|error| format!("cannot create Go build output: {error}"))?;
|
||||
let grants = vec![dever_sandbox::Grant {
|
||||
source: build_directory,
|
||||
destination: "/data/build".into(),
|
||||
writable: true,
|
||||
}];
|
||||
for (path, bytes) in pack_files {
|
||||
stage.write(
|
||||
&format!("input/{path}"),
|
||||
bytes,
|
||||
path == &build.compiler || path == &build.linker || path == &build.analyzer,
|
||||
)?;
|
||||
}
|
||||
let stdlib = references
|
||||
.into_iter()
|
||||
.map(|(name, path)| (name, input.join(path)))
|
||||
.collect();
|
||||
let source_prefix = format!("{}/source/", tree.prefix);
|
||||
let mut entry_present = false;
|
||||
for (path, resource) in &tree.files {
|
||||
if let Some(relative) = path.strip_prefix(&source_prefix) {
|
||||
if relative == entry_name(&contract.entry)? {
|
||||
entry_present = true;
|
||||
}
|
||||
stage.write(&format!("input/source/{relative}"), &resource.bytes, false)?;
|
||||
}
|
||||
}
|
||||
if !entry_present {
|
||||
return Err("managed Go Worker entry source is missing".into());
|
||||
}
|
||||
if tree
|
||||
.files
|
||||
.contains_key(&format!("{}/source/dever_worker_runner.go", tree.prefix))
|
||||
{
|
||||
return Err("Adapter source conflicts with the generated Go Worker runner".into());
|
||||
}
|
||||
stage.write("input/sdk/component.go", SDK, false)?;
|
||||
let runner = runner_source(contract)?;
|
||||
stage.write(
|
||||
"input/source/dever_worker_runner.go",
|
||||
runner.as_bytes(),
|
||||
false,
|
||||
)?;
|
||||
let mut modules = BTreeMap::new();
|
||||
let authenticated =
|
||||
libs::sumdb::verify_chain(&lock.go_sumdb, &libs::sumdb::Verifier::official())?;
|
||||
for lib in dependency_closure(contract, lock)? {
|
||||
let root = format!("{}@v{}/", lib.spec.name, lib.spec.version);
|
||||
let directory = format!("modules/{}", digest(lib.spec.key().as_bytes()));
|
||||
let archive = locked_archive(lib, "zip", supplied, target)?;
|
||||
authenticated.verify_zip(&lib.spec.name, &lib.spec.version, archive)?;
|
||||
let files = libs::archive_files("zip", archive)?;
|
||||
if files.is_empty() {
|
||||
return Err(format!("locked Go module {} is empty", lib.spec.key()));
|
||||
}
|
||||
for (path, bytes) in files {
|
||||
let relative = path.strip_prefix(&root).ok_or_else(|| {
|
||||
format!(
|
||||
"locked Go module {} has a mismatched archive root",
|
||||
lib.spec.key()
|
||||
)
|
||||
})?;
|
||||
stage.write(&format!("input/{directory}/{relative}"), &bytes, false)?;
|
||||
}
|
||||
if modules
|
||||
.insert(lib.spec.name.clone(), input.join(directory))
|
||||
.is_some()
|
||||
{
|
||||
return Err("locked Go module graph selects multiple versions of one module".into());
|
||||
}
|
||||
}
|
||||
let mut builder = Builder {
|
||||
stage,
|
||||
input,
|
||||
assets,
|
||||
grants,
|
||||
tools: build.clone(),
|
||||
entry: entry_name(&contract.entry)?.into(),
|
||||
stdlib,
|
||||
modules,
|
||||
archives: BTreeMap::new(),
|
||||
visiting: BTreeSet::new(),
|
||||
};
|
||||
builder.verify_tools()?;
|
||||
builder.compile("dever-component")?;
|
||||
builder.compile("main")?;
|
||||
let binary_name = format!("worker{}", std::env::consts::EXE_SUFFIX);
|
||||
builder.link(&binary_name)?;
|
||||
let path = builder.stage.0.join("build").join(&binary_name);
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
|
||||
}
|
||||
let file = options
|
||||
.open(path)
|
||||
.map_err(|error| format!("cannot open linked Go Worker: {error}"))?;
|
||||
let metadata = file.metadata().map_err(|error| error.to_string())?;
|
||||
if !metadata.is_file() || metadata.len() > 128 * 1024 * 1024 {
|
||||
return Err("linked Go Worker is not a bounded regular file".into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
if metadata.nlink() != 1 {
|
||||
return Err("linked Go Worker must not have hard links".into());
|
||||
}
|
||||
}
|
||||
let mut binary = Vec::new();
|
||||
file.take(128 * 1024 * 1024 + 1)
|
||||
.read_to_end(&mut binary)
|
||||
.map_err(|error| format!("cannot read linked Go Worker: {error}"))?;
|
||||
if binary.len() > 128 * 1024 * 1024 {
|
||||
return Err("linked Go Worker exceeds the byte limit".into());
|
||||
}
|
||||
if binary.is_empty() {
|
||||
return Err("linked Go Worker is empty".into());
|
||||
}
|
||||
super::runtime::validate_binary("Go Worker", &binary, target)?;
|
||||
tree.add(&binary_name, binary, true)?;
|
||||
Ok(binary_name)
|
||||
}
|
||||
|
||||
impl GoBuildManifest {
|
||||
pub(super) fn executable(&self, path: &str) -> bool {
|
||||
[&self.compiler, &self.linker, &self.analyzer]
|
||||
.iter()
|
||||
.any(|tool| tool.as_str() == path)
|
||||
}
|
||||
|
||||
pub(super) fn validate_files(
|
||||
&self,
|
||||
files: &[(String, Vec<u8>)],
|
||||
target: &str,
|
||||
) -> Result<BTreeMap<String, String>, String> {
|
||||
self.validate(target)?;
|
||||
let mut pack = BTreeMap::new();
|
||||
for (path, bytes) in files {
|
||||
if pack.insert(path.as_str(), bytes.as_slice()).is_some() {
|
||||
return Err("Go build pack repeats a file".into());
|
||||
}
|
||||
}
|
||||
for tool in [&self.compiler, &self.linker, &self.analyzer] {
|
||||
if !tool.starts_with("bin/")
|
||||
|| !pack
|
||||
.get(tool.as_str())
|
||||
.is_some_and(|bytes| !bytes.is_empty())
|
||||
{
|
||||
return Err(format!("locked Go build tool '{tool}' is missing"));
|
||||
}
|
||||
super::runtime::validate_binary(tool, pack[tool.as_str()], self.host.platform())?;
|
||||
}
|
||||
if self.stdlib_importcfg != "stdlib/importcfg" {
|
||||
return Err("locked Go standard library importcfg path is invalid".into());
|
||||
}
|
||||
let importcfg = pack
|
||||
.get(self.stdlib_importcfg.as_str())
|
||||
.ok_or("locked Go standard library importcfg is missing")?;
|
||||
let importcfg = std::str::from_utf8(importcfg)
|
||||
.map_err(|_| "locked Go standard library importcfg is not UTF-8")?;
|
||||
let mut stdlib = BTreeMap::new();
|
||||
for line in importcfg.lines() {
|
||||
let (name, path) = line
|
||||
.strip_prefix("packagefile ")
|
||||
.and_then(|line| line.split_once('='))
|
||||
.ok_or("invalid locked Go standard library importcfg")?;
|
||||
valid_import(name)?;
|
||||
if path != format!("stdlib/{name}.a")
|
||||
|| !pack.get(path).is_some_and(|bytes| !bytes.is_empty())
|
||||
|| stdlib.insert(name.into(), path.into()).is_some()
|
||||
{
|
||||
return Err(
|
||||
"locked Go standard library importcfg has a missing or repeated archive".into(),
|
||||
);
|
||||
}
|
||||
let archive = pack[path];
|
||||
let version = self
|
||||
.go_version
|
||||
.split_whitespace()
|
||||
.nth(2)
|
||||
.ok_or("invalid Go version")?;
|
||||
let object_identity = format!("go object {} {} {version} ", self.goos, self.goarch);
|
||||
if !archive.starts_with(b"!<arch>\n__.PKGDEF")
|
||||
|| !archive
|
||||
.get(68..)
|
||||
.is_some_and(|bytes| bytes.starts_with(object_identity.as_bytes()))
|
||||
{
|
||||
return Err(format!(
|
||||
"locked Go standard library archive '{name}' differs from its target or compiler version"
|
||||
));
|
||||
}
|
||||
}
|
||||
if !["runtime", "context", "encoding/json"]
|
||||
.iter()
|
||||
.all(|name| stdlib.contains_key(*name))
|
||||
{
|
||||
return Err("locked Go standard library importcfg is incomplete".into());
|
||||
}
|
||||
Ok(stdlib)
|
||||
}
|
||||
|
||||
fn validate(&self, target: &str) -> Result<(), String> {
|
||||
let expected = match target {
|
||||
"linux-x86_64" => ("linux", "amd64"),
|
||||
"linux-aarch64" => ("linux", "arm64"),
|
||||
_ => return Err("managed Go build target is unsupported".into()),
|
||||
};
|
||||
if (self.goos.as_str(), self.goarch.as_str()) != expected
|
||||
|| !self.go_version.starts_with("go version go")
|
||||
|| !self.go_version.ends_with(match self.host {
|
||||
crate::toolchain::BuildTarget::LinuxX86_64 => " linux/amd64",
|
||||
crate::toolchain::BuildTarget::LinuxAarch64 => " linux/arm64",
|
||||
})
|
||||
{
|
||||
return Err("managed Go build tools do not match the selected target".into());
|
||||
}
|
||||
for path in [
|
||||
&self.compiler,
|
||||
&self.linker,
|
||||
&self.analyzer,
|
||||
&self.stdlib_importcfg,
|
||||
] {
|
||||
valid_path(path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
fn verify_tools(&self) -> Result<(), String> {
|
||||
let version = self
|
||||
.tools
|
||||
.go_version
|
||||
.split_whitespace()
|
||||
.nth(2)
|
||||
.ok_or("locked Go build version is invalid")?;
|
||||
for (kind, path) in [
|
||||
("compile", &self.tools.compiler),
|
||||
("link", &self.tools.linker),
|
||||
] {
|
||||
let output = self.run(&self.input.join(path), &["-V"])?;
|
||||
if output != format!("{kind} version {version}\n").as_bytes() {
|
||||
return Err(format!(
|
||||
"locked Go {kind} tool version differs from its pack"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn package_dir(&self, name: &str) -> Result<PathBuf, String> {
|
||||
if name == "main" {
|
||||
return Ok(self.input.join("source"));
|
||||
}
|
||||
if name == "dever-component" {
|
||||
return Ok(self.input.join("sdk"));
|
||||
}
|
||||
if let Some(relative) = name.strip_prefix("dever-worker/") {
|
||||
valid_path(relative)?;
|
||||
return Ok(self.input.join("source").join(relative));
|
||||
}
|
||||
let (module, root) = self
|
||||
.modules
|
||||
.iter()
|
||||
.filter(|(module, _)| {
|
||||
name == module.as_str() || name.starts_with(&format!("{module}/"))
|
||||
})
|
||||
.max_by_key(|(module, _)| module.len())
|
||||
.ok_or_else(|| format!("Go import '{name}' is not in the locked module graph"))?;
|
||||
let suffix = name
|
||||
.strip_prefix(module)
|
||||
.expect("selected module prefix")
|
||||
.trim_start_matches('/');
|
||||
if !suffix.is_empty() {
|
||||
valid_path(suffix)?;
|
||||
}
|
||||
Ok(root.join(suffix))
|
||||
}
|
||||
|
||||
fn compile(&mut self, name: &str) -> Result<(), String> {
|
||||
if self.stdlib.contains_key(name) || name == "unsafe" || self.archives.contains_key(name) {
|
||||
return Ok(());
|
||||
}
|
||||
valid_import(name)?;
|
||||
if !self.visiting.insert(name.to_owned()) {
|
||||
return Err(format!("Go import cycle includes '{name}'"));
|
||||
}
|
||||
let directory = self.package_dir(name)?;
|
||||
let analyzer = self.input.join(&self.tools.analyzer);
|
||||
let output = self.run(
|
||||
&analyzer,
|
||||
&[
|
||||
self.tools.goos.as_str(),
|
||||
self.tools.goarch.as_str(),
|
||||
directory.to_str().ok_or("Go package path is not UTF-8")?,
|
||||
],
|
||||
)?;
|
||||
let selected: SelectedPackage = serde_json::from_slice(&output)
|
||||
.map_err(|error| format!("invalid locked Go analyzer output: {error}"))?;
|
||||
if selected.files.is_empty() || (name == "main") != (selected.name == "main") {
|
||||
return Err(format!(
|
||||
"Go package '{name}' has an invalid package declaration"
|
||||
));
|
||||
}
|
||||
if name == "main" && !selected.files.contains(&self.entry) {
|
||||
// The generated runner may be selected while the checked entry is excluded by tags.
|
||||
return Err("checked Go Worker entry is excluded by target build constraints".into());
|
||||
}
|
||||
for import in &selected.imports {
|
||||
self.compile(import)?;
|
||||
}
|
||||
let cfg = self.importcfg()?;
|
||||
let id = digest(name.as_bytes());
|
||||
let cfg_path = self.stage.write(
|
||||
&format!("input/config/{id}.importcfg"),
|
||||
cfg.as_bytes(),
|
||||
false,
|
||||
)?;
|
||||
let archive = self.stage.0.join(format!("build/{id}.a"));
|
||||
let mut args = vec![
|
||||
"-pack".to_owned(),
|
||||
"-complete".into(),
|
||||
"-nolocalimports".into(),
|
||||
// Staging is invocation-owned; recorded source paths must not make
|
||||
// identical locked Workers differ across run/build or projects.
|
||||
"-trimpath".into(),
|
||||
self.input.to_string_lossy().into_owned(),
|
||||
"-p".into(),
|
||||
name.into(),
|
||||
"-importcfg".into(),
|
||||
cfg_path.to_string_lossy().into_owned(),
|
||||
"-o".into(),
|
||||
archive.to_string_lossy().into_owned(),
|
||||
];
|
||||
if !selected.embed.is_empty() {
|
||||
let mut files = BTreeMap::new();
|
||||
for paths in selected.embed.values() {
|
||||
for path in paths {
|
||||
valid_path(path)?;
|
||||
let file = directory.join(path);
|
||||
if !file.is_file() {
|
||||
return Err("Go embed file is absent from locked source".into());
|
||||
}
|
||||
files.insert(path.clone(), self.visible_path(&file)?);
|
||||
}
|
||||
}
|
||||
let embed = json!({"Patterns":selected.embed,"Files":files});
|
||||
let path = self.stage.write(
|
||||
&format!("input/config/{id}.embedcfg"),
|
||||
&serde_json::to_vec(&embed).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
)?;
|
||||
args.extend(["-embedcfg".into(), path.to_string_lossy().into_owned()]);
|
||||
}
|
||||
for file in selected.files {
|
||||
valid_path(&file)?;
|
||||
args.push(directory.join(file).to_string_lossy().into_owned());
|
||||
}
|
||||
let compiler = self.input.join(&self.tools.compiler);
|
||||
let arguments = args.iter().map(String::as_str).collect::<Vec<_>>();
|
||||
self.run(&compiler, &arguments)?;
|
||||
self.archives.insert(name.into(), archive);
|
||||
self.visiting.remove(name);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn importcfg(&self) -> Result<String, String> {
|
||||
let mut cfg = String::new();
|
||||
for (name, path) in self.stdlib.iter().chain(&self.archives) {
|
||||
cfg.push_str(&format!(
|
||||
"packagefile {name}={}\n",
|
||||
self.visible_path(path)?
|
||||
));
|
||||
}
|
||||
Ok(cfg)
|
||||
}
|
||||
|
||||
fn link(&self, output: &str) -> Result<(), String> {
|
||||
let cfg = self.stage.write(
|
||||
"input/config/link.importcfg",
|
||||
self.importcfg()?.as_bytes(),
|
||||
false,
|
||||
)?;
|
||||
let main = self
|
||||
.archives
|
||||
.get("main")
|
||||
.ok_or("Go Worker main archive is missing")?;
|
||||
let linker = self.input.join(&self.tools.linker);
|
||||
let output_path = self.stage.0.join("build").join(output);
|
||||
let temporary_directory = self.stage.0.join("build");
|
||||
self.run(
|
||||
&linker,
|
||||
&[
|
||||
"-linkmode=internal",
|
||||
"-buildmode=exe",
|
||||
"-tmpdir",
|
||||
temporary_directory
|
||||
.to_str()
|
||||
.ok_or("Go build path is not UTF-8")?,
|
||||
"-importcfg",
|
||||
cfg.to_str().ok_or("Go importcfg path is not UTF-8")?,
|
||||
"-o",
|
||||
output_path.to_str().ok_or("Go output path is not UTF-8")?,
|
||||
main.to_str().ok_or("Go archive path is not UTF-8")?,
|
||||
],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn run(&self, executable: &Path, args: &[&str]) -> Result<Vec<u8>, String> {
|
||||
let arguments = args.iter().map(OsString::from).collect::<Vec<_>>();
|
||||
self.stage.run(
|
||||
self.sandbox(executable, &arguments).command()?,
|
||||
"locked Go build tool",
|
||||
&self.stage.0.join("build"),
|
||||
)
|
||||
}
|
||||
|
||||
fn sandbox<'a>(
|
||||
&'a self,
|
||||
executable: &'a Path,
|
||||
arguments: &'a [OsString],
|
||||
) -> dever_sandbox::Launch<'a> {
|
||||
dever_sandbox::Launch {
|
||||
assets: &self.assets,
|
||||
worker: &self.input,
|
||||
executable,
|
||||
arguments,
|
||||
working_directory: &self.input,
|
||||
capabilities: dever_sandbox::Capabilities::default(),
|
||||
grants: &self.grants,
|
||||
}
|
||||
}
|
||||
|
||||
fn visible_path(&self, path: &Path) -> Result<String, String> {
|
||||
self.sandbox(&self.input, &[])
|
||||
.argument_path(path)?
|
||||
.into_os_string()
|
||||
.into_string()
|
||||
.map_err(|_| "Go sandbox path is not UTF-8".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_import(name: &str) -> Result<(), String> {
|
||||
if name.is_empty()
|
||||
|| name.split('/').any(str::is_empty)
|
||||
|| !name.bytes().all(|byte| {
|
||||
byte.is_ascii_alphanumeric() || matches!(byte, b'/' | b'.' | b'_' | b'-' | b'~')
|
||||
})
|
||||
{
|
||||
return Err(format!("invalid locked Go import '{name}'"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn runner_source(contract: &ExternalWorkerContract) -> Result<String, String> {
|
||||
let mut handlers = String::new();
|
||||
let mut names = BTreeSet::new();
|
||||
for operation in &contract.operations {
|
||||
let mut function = String::new();
|
||||
for part in operation.split(['.', '_', '-']) {
|
||||
if part.is_empty() || !part.bytes().all(|byte| byte.is_ascii_alphanumeric()) {
|
||||
return Err(format!(
|
||||
"Go operation '{operation}' cannot bind to an exported function"
|
||||
));
|
||||
}
|
||||
let mut letters = part.chars();
|
||||
function.push(letters.next().expect("nonempty part").to_ascii_uppercase());
|
||||
function.extend(letters);
|
||||
}
|
||||
if !names.insert(function.clone())
|
||||
|| !function.starts_with(|character: char| character.is_ascii_uppercase())
|
||||
{
|
||||
return Err("Go operation names map to duplicate or invalid exported functions".into());
|
||||
}
|
||||
handlers.push_str(&format!("\t\t{operation:?}: {function},\n"));
|
||||
}
|
||||
Ok(format!(
|
||||
"package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\tcomponent \"dever-component\"\n)\n\nfunc main() {{\n\tmanifest, err := os.ReadFile(\"contract.json\")\n\tif err == nil {{\n\t\terr = component.ServeManifest(manifest, map[string]func(context.Context, any, any) (any, error){{\n{handlers}\t\t}}, os.Stdin, os.Stdout)\n\t}}\n\tif err != nil {{\n\t\tfmt.Fprintln(os.Stderr, err)\n\t\tos.Exit(1)\n\t}}\n}}\n"
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,588 @@
|
||||
//! One wheel admission and installation contract for registry downloads,
|
||||
//! offline Workers and isolated build dependencies.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use pep440_rs::Version;
|
||||
use pep508_rs::Requirement;
|
||||
use sha2::{Digest, Sha256, Sha384, Sha512};
|
||||
|
||||
use crate::libs::{LibSpec, archive_files};
|
||||
|
||||
pub(crate) mod native;
|
||||
mod scripts;
|
||||
pub use native::validate_native;
|
||||
|
||||
pub struct Wheel {
|
||||
files: Vec<(String, Vec<u8>)>,
|
||||
stem: String,
|
||||
pub requirements: Vec<String>,
|
||||
pub extras: Vec<String>,
|
||||
pub requires_python: Option<String>,
|
||||
}
|
||||
|
||||
pub struct InstalledFile {
|
||||
pub path: String,
|
||||
pub bytes: Vec<u8>,
|
||||
pub executable: bool,
|
||||
}
|
||||
|
||||
/// Filename tags can be compressed; runtime and WHEEL tags must be expanded.
|
||||
pub fn validate_tags(tags: &[String]) -> Result<(), String> {
|
||||
let mut unique = BTreeSet::new();
|
||||
if tags.is_empty() || tags.len() > 4096 {
|
||||
return Err("Python wheel compatibility tags are missing or exceed their bound".into());
|
||||
}
|
||||
for tag in tags {
|
||||
if tag.split('-').count() != 3
|
||||
|| tag.split('-').any(|part| {
|
||||
part.is_empty()
|
||||
|| !part
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
|
||||
})
|
||||
|| !unique.insert(tag)
|
||||
{
|
||||
return Err("Python wheel compatibility tags are invalid or repeated".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn validate_target(
|
||||
target: &str,
|
||||
tags: &[String],
|
||||
markers: &pep508_rs::MarkerEnvironment,
|
||||
) -> Result<(), String> {
|
||||
validate_tags(tags)?;
|
||||
let architecture = match target.parse::<crate::toolchain::BuildTarget>()? {
|
||||
crate::toolchain::BuildTarget::LinuxX86_64 => "x86_64",
|
||||
crate::toolchain::BuildTarget::LinuxAarch64 => "aarch64",
|
||||
};
|
||||
if markers.sys_platform() != "linux" || markers.platform_machine() != architecture {
|
||||
return Err("Python marker environment differs from the selected target".into());
|
||||
}
|
||||
for tag in tags {
|
||||
let platform = tag.rsplit('-').next().expect("validated wheel tag");
|
||||
if platform != "any"
|
||||
&& (!(platform.starts_with("linux_")
|
||||
|| platform.starts_with("manylinux")
|
||||
|| platform.starts_with("musllinux"))
|
||||
|| !platform.ends_with(&format!("_{architecture}")))
|
||||
{
|
||||
return Err("Python wheel platform tag differs from the selected target".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn expanded_tags(python: &str, abi: &str, platform: &str) -> Result<BTreeSet<String>, String> {
|
||||
let mut tags = BTreeSet::new();
|
||||
for python in python.split('.') {
|
||||
for abi in abi.split('.') {
|
||||
for platform in platform.split('.') {
|
||||
if tags.len() >= 4096 || !tags.insert(format!("{python}-{abi}-{platform}")) {
|
||||
return Err("wheel filename tags are repeated or exceed their bound".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
validate_tags(&tags.iter().cloned().collect::<Vec<_>>())?;
|
||||
Ok(tags)
|
||||
}
|
||||
|
||||
fn filename(
|
||||
filename: &str,
|
||||
spec: Option<&LibSpec>,
|
||||
) -> Result<(BTreeSet<String>, Option<String>), String> {
|
||||
super::valid_path(filename)?;
|
||||
if filename.contains('/') {
|
||||
return Err("wheel filename must be a leaf name".into());
|
||||
}
|
||||
let fields = filename
|
||||
.strip_suffix(".whl")
|
||||
.ok_or("wheel filename lacks .whl")?
|
||||
.split('-')
|
||||
.collect::<Vec<_>>();
|
||||
if !matches!(fields.len(), 5 | 6) {
|
||||
return Err("invalid wheel filename fields".into());
|
||||
}
|
||||
if let Some(spec) = spec {
|
||||
validate_identity(fields[0], fields[1], spec)?;
|
||||
}
|
||||
let build = if fields.len() == 6 {
|
||||
validate_build(fields[2])?;
|
||||
Some(fields[2].to_owned())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let tail = &fields[fields.len() - 3..];
|
||||
Ok((expanded_tags(tail[0], tail[1], tail[2])?, build))
|
||||
}
|
||||
|
||||
pub fn matches(filename_value: &str, selected_tag: &str) -> bool {
|
||||
filename(filename_value, None).is_ok_and(|(tags, _)| tags.contains(selected_tag))
|
||||
}
|
||||
|
||||
fn validate_build(value: &str) -> Result<(), String> {
|
||||
if !value.as_bytes().first().is_some_and(u8::is_ascii_digit)
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
|
||||
{
|
||||
return Err("invalid wheel build tag".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_identity(name: &str, version: &str, spec: &LibSpec) -> Result<(), String> {
|
||||
name.parse::<pep508_rs::PackageName>()
|
||||
.map_err(|error| format!("invalid wheel distribution name: {error}"))?;
|
||||
let identity: LibSpec = format!("pip:{name}@{version}").parse()?;
|
||||
if identity.distribution_name() != spec.distribution_name()
|
||||
|| version
|
||||
.parse::<Version>()
|
||||
.map_err(|error| error.to_string())?
|
||||
!= spec
|
||||
.version
|
||||
.parse::<Version>()
|
||||
.map_err(|error| error.to_string())?
|
||||
{
|
||||
return Err("wheel distribution identity differs from its lock".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn headers(bytes: &[u8]) -> Result<BTreeMap<String, Vec<String>>, String> {
|
||||
if bytes.len() > 16 * 1024 * 1024 {
|
||||
return Err("wheel metadata exceeds 16 MiB".into());
|
||||
}
|
||||
let text = std::str::from_utf8(bytes).map_err(|_| "wheel metadata is not UTF-8")?;
|
||||
let mut result = BTreeMap::<String, Vec<String>>::new();
|
||||
let mut previous: Option<String> = None;
|
||||
for line in text.lines() {
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
if line.starts_with([' ', '\t']) {
|
||||
let key = previous
|
||||
.as_ref()
|
||||
.ok_or("wheel metadata starts with a folded line")?;
|
||||
let value = result.get_mut(key).unwrap().last_mut().unwrap();
|
||||
value.push(' ');
|
||||
value.push_str(line.trim());
|
||||
continue;
|
||||
}
|
||||
let (key, value) = line
|
||||
.split_once(':')
|
||||
.ok_or("invalid wheel metadata header")?;
|
||||
if key.is_empty()
|
||||
|| !key
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||
|| value.contains('\0')
|
||||
{
|
||||
return Err("invalid wheel metadata header".into());
|
||||
}
|
||||
let key = key.to_ascii_lowercase();
|
||||
result
|
||||
.entry(key.clone())
|
||||
.or_default()
|
||||
.push(value.trim().into());
|
||||
previous = Some(key);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn optional_header<'a>(
|
||||
headers: &'a BTreeMap<String, Vec<String>>,
|
||||
key: &str,
|
||||
) -> Result<Option<&'a str>, String> {
|
||||
match headers.get(key) {
|
||||
None => Ok(None),
|
||||
Some(values) if values.len() == 1 => Ok(Some(values[0].as_str())),
|
||||
_ => Err(format!("wheel metadata repeats {key}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn required_header<'a>(
|
||||
headers: &'a BTreeMap<String, Vec<String>>,
|
||||
key: &str,
|
||||
) -> Result<&'a str, String> {
|
||||
optional_header(headers, key)?.ok_or_else(|| format!("wheel metadata lacks {key}"))
|
||||
}
|
||||
|
||||
impl Wheel {
|
||||
pub(crate) fn validate_dependencies(
|
||||
&self,
|
||||
lib: &crate::libs::LockedLib,
|
||||
markers: &pep508_rs::MarkerEnvironment,
|
||||
) -> Result<(), String> {
|
||||
let extras = lib
|
||||
.spec
|
||||
.name
|
||||
.split_once('[')
|
||||
.map(|(_, extras)| extras.trim_end_matches(']'))
|
||||
.unwrap_or("")
|
||||
.split(',')
|
||||
.filter(|extra| !extra.is_empty())
|
||||
.map(str::parse::<pep508_rs::ExtraName>)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let supplied_extras = self
|
||||
.extras
|
||||
.iter()
|
||||
.map(|extra| extra.parse::<pep508_rs::ExtraName>())
|
||||
.collect::<Result<BTreeSet<_>, _>>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
if extras.iter().any(|extra| !supplied_extras.contains(extra)) {
|
||||
return Err("wheel does not provide a requested extra".into());
|
||||
}
|
||||
let document = serde_json::json!({"info":{"requires_dist":self.requirements}});
|
||||
let expected = crate::libs::python_dependencies(&document, markers, &extras)?;
|
||||
let mut names = BTreeSet::new();
|
||||
for (name, rule) in expected {
|
||||
if name == lib.spec.name
|
||||
&& rule.contains(
|
||||
&lib.spec
|
||||
.version
|
||||
.parse::<Version>()
|
||||
.map_err(|error| error.to_string())?,
|
||||
)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let dependency = lib
|
||||
.dependencies
|
||||
.iter()
|
||||
.find(|dependency| dependency.spec.name == name)
|
||||
.ok_or("wheel dependency is missing from the exact lock")?;
|
||||
if !rule.contains(
|
||||
&dependency
|
||||
.spec
|
||||
.version
|
||||
.parse::<Version>()
|
||||
.map_err(|error| error.to_string())?,
|
||||
) {
|
||||
return Err("locked wheel dependency does not satisfy METADATA".into());
|
||||
}
|
||||
names.insert(name);
|
||||
}
|
||||
if lib
|
||||
.dependencies
|
||||
.iter()
|
||||
.any(|dependency| !names.contains(&dependency.spec.name))
|
||||
{
|
||||
return Err("wheel lock has an undeclared dependency".into());
|
||||
}
|
||||
if let Some(rule) = &self.requires_python
|
||||
&& !rule
|
||||
.parse::<pep440_rs::VersionSpecifiers>()
|
||||
.map_err(|error| error.to_string())?
|
||||
.contains(markers.python_full_version())
|
||||
{
|
||||
return Err("wheel Requires-Python excludes the signed runtime".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn parse(
|
||||
spec: &LibSpec,
|
||||
filename_value: Option<&str>,
|
||||
runtime_tags: &[String],
|
||||
bytes: &[u8],
|
||||
) -> Result<Self, String> {
|
||||
validate_tags(runtime_tags)?;
|
||||
let files = archive_files("whl", bytes)?;
|
||||
let mut info = BTreeSet::new();
|
||||
let mut data = BTreeSet::new();
|
||||
for (path, _) in &files {
|
||||
super::valid_path(path)?;
|
||||
if let Some((root, _)) = path.split_once('/') {
|
||||
if root.ends_with(".dist-info") {
|
||||
info.insert(root);
|
||||
}
|
||||
if root.ends_with(".data") {
|
||||
data.insert(root);
|
||||
}
|
||||
}
|
||||
}
|
||||
if info.len() != 1 {
|
||||
return Err("wheel must contain exactly one dist-info directory".into());
|
||||
}
|
||||
let info = info.into_iter().next().unwrap();
|
||||
let stem = info.strip_suffix(".dist-info").unwrap();
|
||||
let (name, version) = stem
|
||||
.rsplit_once('-')
|
||||
.ok_or("invalid wheel dist-info directory")?;
|
||||
validate_identity(name, version, spec)?;
|
||||
if data.iter().any(|root| *root != format!("{stem}.data")) {
|
||||
return Err("wheel data directory differs from its distribution identity".into());
|
||||
}
|
||||
let read = |name: &str| {
|
||||
files
|
||||
.iter()
|
||||
.find(|(path, _)| path == &format!("{info}/{name}"))
|
||||
.map(|(_, bytes)| bytes.as_slice())
|
||||
.ok_or_else(|| format!("wheel lacks {name}"))
|
||||
};
|
||||
let metadata = headers(read("METADATA")?)?;
|
||||
let metadata_version = required_header(&metadata, "metadata-version")?;
|
||||
if metadata_version
|
||||
.split_once('.')
|
||||
.is_none_or(|(major, minor)| {
|
||||
!matches!(major, "1" | "2") || minor.parse::<u32>().is_err()
|
||||
})
|
||||
{
|
||||
return Err("unsupported wheel core metadata version".into());
|
||||
}
|
||||
validate_identity(
|
||||
required_header(&metadata, "name")?,
|
||||
required_header(&metadata, "version")?,
|
||||
spec,
|
||||
)?;
|
||||
let wheel = headers(read("WHEEL")?)?;
|
||||
let wheel_version = required_header(&wheel, "wheel-version")?;
|
||||
if wheel_version
|
||||
.split_once('.')
|
||||
.is_none_or(|(major, minor)| major != "1" || minor.parse::<u32>().is_err())
|
||||
{
|
||||
return Err("unsupported Wheel-Version".into());
|
||||
}
|
||||
if !matches!(
|
||||
required_header(&wheel, "root-is-purelib")?,
|
||||
"true" | "false"
|
||||
) {
|
||||
return Err("invalid wheel Root-Is-Purelib".into());
|
||||
}
|
||||
let tags = wheel.get("tag").ok_or("wheel has no compatibility Tag")?;
|
||||
validate_tags(tags)?;
|
||||
if !tags.iter().any(|tag| runtime_tags.contains(tag)) {
|
||||
return Err("wheel tags do not match the signed Python runtime".into());
|
||||
}
|
||||
if tags.iter().all(|tag| tag.ends_with("-any"))
|
||||
&& files.iter().any(|(path, bytes)| {
|
||||
bytes.starts_with(b"\x7fELF")
|
||||
|| path.ends_with(".pyd")
|
||||
|| path.ends_with(".dll")
|
||||
|| path.ends_with(".dylib")
|
||||
})
|
||||
{
|
||||
return Err("platform-independent wheel contains a native binary".into());
|
||||
}
|
||||
let build = optional_header(&wheel, "build")?;
|
||||
if let Some(build) = build {
|
||||
validate_build(build)?;
|
||||
}
|
||||
if let Some(value) = filename_value {
|
||||
let (file_tags, file_build) = filename(value, Some(spec))?;
|
||||
if file_tags != tags.iter().cloned().collect() || file_build.as_deref() != build {
|
||||
return Err("wheel filename tags/build differ from WHEEL metadata".into());
|
||||
}
|
||||
}
|
||||
validate_record(&files, info, read("RECORD")?)?;
|
||||
let requirements = metadata.get("requires-dist").cloned().unwrap_or_default();
|
||||
for requirement in &requirements {
|
||||
requirement
|
||||
.parse::<Requirement>()
|
||||
.map_err(|error| format!("invalid wheel Requires-Dist: {error}"))?;
|
||||
}
|
||||
let extras = metadata.get("provides-extra").cloned().unwrap_or_default();
|
||||
for extra in &extras {
|
||||
extra
|
||||
.parse::<pep508_rs::ExtraName>()
|
||||
.map_err(|error| format!("invalid wheel Provides-Extra: {error}"))?;
|
||||
}
|
||||
let requires_python = optional_header(&metadata, "requires-python")?.map(str::to_owned);
|
||||
if let Some(rule) = &requires_python {
|
||||
rule.parse::<pep440_rs::VersionSpecifiers>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
let stem = stem.to_owned();
|
||||
Ok(Self {
|
||||
files,
|
||||
stem,
|
||||
requirements,
|
||||
extras,
|
||||
requires_python,
|
||||
})
|
||||
}
|
||||
|
||||
/// Install into the interpreter's real prefix so sys.prefix, sysconfig,
|
||||
/// standalone scripts and the isolated runner agree on resource locations.
|
||||
pub fn install(
|
||||
self,
|
||||
interpreter: &str,
|
||||
python_version: &str,
|
||||
) -> Result<Vec<InstalledFile>, String> {
|
||||
super::valid_path(interpreter)?;
|
||||
let prefix = interpreter
|
||||
.rsplit_once("/bin/")
|
||||
.map(|(prefix, _)| prefix)
|
||||
.ok_or("Python installation executable must live in its prefix/bin")?;
|
||||
super::valid_path(prefix)?;
|
||||
if python_version.split('.').count() != 2
|
||||
|| python_version
|
||||
.split('.')
|
||||
.any(|part| part.parse::<u32>().is_err())
|
||||
{
|
||||
return Err("Python installation requires a major.minor version".into());
|
||||
}
|
||||
let library = format!("{prefix}/lib/python{python_version}/site-packages");
|
||||
let name = self.stem.rsplit_once('-').unwrap().0;
|
||||
let data_prefix = format!("{}.data/", self.stem);
|
||||
let mut installed = BTreeSet::new();
|
||||
let mut output = Vec::with_capacity(self.files.len());
|
||||
let entry_points = self
|
||||
.files
|
||||
.iter()
|
||||
.find(|(path, _)| path == &format!("{}.dist-info/entry_points.txt", self.stem))
|
||||
.map(|(_, bytes)| scripts::entry_points(bytes, prefix, interpreter))
|
||||
.transpose()?
|
||||
.unwrap_or_default();
|
||||
for (path, mut bytes) in self.files {
|
||||
let (destination, executable) = if let Some(relative) = path.strip_prefix(&data_prefix)
|
||||
{
|
||||
let (scheme, relative) = relative
|
||||
.split_once('/')
|
||||
.ok_or("wheel data scheme has no file")?;
|
||||
let destination = match scheme {
|
||||
"purelib" | "platlib" => format!("{library}/{relative}"),
|
||||
"headers" => {
|
||||
format!("{prefix}/include/python{python_version}/{name}/{relative}")
|
||||
}
|
||||
"scripts" if !relative.contains('/') => format!("{prefix}/bin/{relative}"),
|
||||
"data" => format!("{prefix}/{relative}"),
|
||||
_ => return Err("invalid wheel data installation scheme".into()),
|
||||
};
|
||||
if scheme == "scripts"
|
||||
&& (bytes.starts_with(b"#!python\n")
|
||||
|| bytes.starts_with(b"#!pythonw\n")
|
||||
|| bytes.starts_with(b"#!python\r\n")
|
||||
|| bytes.starts_with(b"#!pythonw\r\n"))
|
||||
{
|
||||
let newline = bytes.iter().position(|byte| *byte == b'\n').unwrap();
|
||||
bytes = [
|
||||
format!("#!/worker/{interpreter} -I\n").as_bytes(),
|
||||
&bytes[newline + 1..],
|
||||
]
|
||||
.concat();
|
||||
}
|
||||
(destination, scheme == "scripts")
|
||||
} else {
|
||||
(format!("{library}/{path}"), false)
|
||||
};
|
||||
super::valid_path(&destination)?;
|
||||
if !installed.insert(destination.clone()) {
|
||||
return Err("wheel installation schemes produce conflicting files".into());
|
||||
}
|
||||
output.push(InstalledFile {
|
||||
path: destination,
|
||||
bytes,
|
||||
executable,
|
||||
});
|
||||
}
|
||||
for file in entry_points {
|
||||
if !installed.insert(file.path.clone()) {
|
||||
return Err("wheel entry point conflicts with another installed script".into());
|
||||
}
|
||||
output.push(file);
|
||||
}
|
||||
let record_path = format!("{library}/{}.dist-info/RECORD", self.stem);
|
||||
let mut record = csv::WriterBuilder::new()
|
||||
.has_headers(false)
|
||||
.from_writer(Vec::new());
|
||||
for file in &output {
|
||||
let base = library.split('/').collect::<Vec<_>>();
|
||||
let target = file.path.split('/').collect::<Vec<_>>();
|
||||
let common = base
|
||||
.iter()
|
||||
.zip(&target)
|
||||
.take_while(|(left, right)| left == right)
|
||||
.count();
|
||||
let relative = format!(
|
||||
"{}{}",
|
||||
"../".repeat(base.len() - common),
|
||||
target[common..].join("/")
|
||||
);
|
||||
let (hash, size) = if file.path == record_path {
|
||||
(String::new(), String::new())
|
||||
} else {
|
||||
(
|
||||
format!(
|
||||
"sha256={}",
|
||||
URL_SAFE_NO_PAD.encode(Sha256::digest(&file.bytes))
|
||||
),
|
||||
file.bytes.len().to_string(),
|
||||
)
|
||||
};
|
||||
record
|
||||
.write_record([relative, hash, size])
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
output
|
||||
.iter_mut()
|
||||
.find(|file| file.path == record_path)
|
||||
.expect("validated RECORD")
|
||||
.bytes = record.into_inner().map_err(|error| error.to_string())?;
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_record(files: &[(String, Vec<u8>)], info: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
let entries = files
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.as_str(), bytes))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let record_path = format!("{info}/RECORD");
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut reader = csv::ReaderBuilder::new()
|
||||
.has_headers(false)
|
||||
.flexible(false)
|
||||
.from_reader(bytes);
|
||||
for row in reader.records() {
|
||||
let row = row.map_err(|error| format!("invalid wheel RECORD CSV: {error}"))?;
|
||||
if row.len() != 3 {
|
||||
return Err("wheel RECORD requires three fields".into());
|
||||
}
|
||||
let path = &row[0];
|
||||
super::valid_path(path)?;
|
||||
if !seen.insert(path.to_owned()) {
|
||||
return Err("wheel RECORD repeats a file".into());
|
||||
}
|
||||
let content = entries
|
||||
.get(path)
|
||||
.ok_or("wheel RECORD references an absent file")?;
|
||||
if path == record_path {
|
||||
if !row[1].is_empty() || !row[2].is_empty() {
|
||||
return Err("wheel RECORD cannot hash itself".into());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if path == format!("{info}/RECORD.jws") || path == format!("{info}/RECORD.p7s") {
|
||||
return Err("wheel signatures must not be listed in RECORD".into());
|
||||
}
|
||||
let (algorithm, expected) = row[1]
|
||||
.split_once('=')
|
||||
.ok_or("wheel RECORD lacks a secure hash")?;
|
||||
let actual = match algorithm {
|
||||
"sha256" => URL_SAFE_NO_PAD.encode(Sha256::digest(content)),
|
||||
"sha384" => URL_SAFE_NO_PAD.encode(Sha384::digest(content)),
|
||||
"sha512" => URL_SAFE_NO_PAD.encode(Sha512::digest(content)),
|
||||
_ => return Err("wheel RECORD hash must use SHA-256 or stronger SHA-2".into()),
|
||||
};
|
||||
if actual != expected || row[2].parse::<usize>().ok() != Some(content.len()) {
|
||||
return Err("wheel RECORD hash or size does not match its file".into());
|
||||
}
|
||||
}
|
||||
if !seen.contains(&record_path)
|
||||
|| entries.keys().any(|path| {
|
||||
!seen.contains(*path)
|
||||
&& **path != format!("{info}/RECORD.jws")
|
||||
&& **path != format!("{info}/RECORD.p7s")
|
||||
})
|
||||
{
|
||||
return Err("wheel RECORD does not cover every file".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
//! Validate extension ABI and the dynamic loader's closed, packaged search tree.
|
||||
|
||||
use super::super::elf::{Closure, object, origin_directories};
|
||||
use goblin::elf::header;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
/// `files` contains Worker-relative installation files and separately verified
|
||||
/// OS ABI files at sandbox/lib/*. No host directory is searched or executed.
|
||||
pub fn validate_native(
|
||||
target: &str,
|
||||
suffixes: &[String],
|
||||
interpreter: &str,
|
||||
files: &BTreeMap<String, &[u8]>,
|
||||
wheel_files: &BTreeSet<String>,
|
||||
) -> Result<(), String> {
|
||||
let native = files
|
||||
.iter()
|
||||
.filter(|(path, _)| wheel_files.contains(*path))
|
||||
.filter(|(path, bytes)| {
|
||||
bytes.starts_with(b"\x7fELF")
|
||||
|| path.ends_with(".so")
|
||||
|| path.contains(".so.")
|
||||
|| path.ends_with(".pyd")
|
||||
|| path.ends_with(".dll")
|
||||
|| path.ends_with(".dylib")
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if native.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let machine = match target {
|
||||
"linux-x86_64" => header::EM_X86_64,
|
||||
"linux-aarch64" => header::EM_AARCH64,
|
||||
_ => return Err("native Python wheel target is unsupported".into()),
|
||||
};
|
||||
let closure = Closure {
|
||||
files,
|
||||
machine,
|
||||
directories: vec!["sandbox/lib".into()],
|
||||
};
|
||||
let interpreter_elf = object(
|
||||
interpreter,
|
||||
files
|
||||
.get(interpreter)
|
||||
.ok_or("native wheel requires its packaged Python executable")?,
|
||||
machine,
|
||||
)?;
|
||||
let inherited = if interpreter_elf.runpaths.is_empty() {
|
||||
origin_directories(interpreter, &interpreter_elf.rpaths)?
|
||||
} else {
|
||||
vec![]
|
||||
};
|
||||
let (globals, _) = closure.follow(interpreter, vec![], &BTreeMap::new())?;
|
||||
let mut entries = Vec::new();
|
||||
for (path, bytes) in native {
|
||||
if path.ends_with(".pyd") || path.ends_with(".dll") || path.ends_with(".dylib") {
|
||||
return Err("native Python wheel contains a binary for a different OS".into());
|
||||
}
|
||||
let elf = object(path, bytes, machine)?;
|
||||
if elf.header.e_type != header::ET_DYN || elf.interpreter.is_some() {
|
||||
return Err("native wheel content must be a shared library, not an executable".into());
|
||||
}
|
||||
let module = elf.dynsyms.iter().any(|symbol| {
|
||||
symbol.st_shndx != 0
|
||||
&& elf
|
||||
.dynstrtab
|
||||
.get_at(symbol.st_name)
|
||||
.is_some_and(|name| name.starts_with("PyInit_"))
|
||||
});
|
||||
let module = module || path.contains(".cpython-") || path.ends_with(".abi3.so");
|
||||
if module {
|
||||
let compatible = suffixes.iter().any(|suffix| {
|
||||
path.ends_with(suffix)
|
||||
&& (suffix != ".so"
|
||||
|| !path.contains(".cpython-") && !path.ends_with(".abi3.so"))
|
||||
});
|
||||
if !compatible {
|
||||
return Err(
|
||||
"native Python extension suffix differs from the signed interpreter ABI".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
entries.push((path, module));
|
||||
}
|
||||
entries.sort_by_key(|(_, module)| !*module);
|
||||
let mut reached = BTreeSet::new();
|
||||
for (path, module) in entries {
|
||||
// Vendored helpers inherit the importing extension's loader context;
|
||||
// reaching one does not imply an independent direct dlopen entry.
|
||||
if !module && reached.contains(path) {
|
||||
continue;
|
||||
}
|
||||
let (_, closure_paths) = closure.follow(path, inherited.clone(), &globals)?;
|
||||
reached.extend(closure_paths);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Compiler processes use the signed sysroot's standard library locations.
|
||||
/// Reuse the ELF version/SONAME closure checks used by installed extensions.
|
||||
pub(crate) fn validate_build_tools(
|
||||
target: &str,
|
||||
executables: &[String],
|
||||
files: &BTreeMap<String, &[u8]>,
|
||||
) -> Result<(), String> {
|
||||
let (machine, multiarch, interpreter) = match target {
|
||||
"linux-x86_64" => (
|
||||
header::EM_X86_64,
|
||||
"x86_64-linux-gnu",
|
||||
"/lib64/ld-linux-x86-64.so.2",
|
||||
),
|
||||
"linux-aarch64" => (
|
||||
header::EM_AARCH64,
|
||||
"aarch64-linux-gnu",
|
||||
"/lib/ld-linux-aarch64.so.1",
|
||||
),
|
||||
_ => return Err("build tools target is unsupported".into()),
|
||||
};
|
||||
let closure = Closure {
|
||||
files,
|
||||
machine,
|
||||
directories: vec![
|
||||
format!("rootfs/usr/lib/{multiarch}"),
|
||||
"sandbox/lib".into(),
|
||||
"rootfs/usr/lib".into(),
|
||||
],
|
||||
};
|
||||
for path in executables {
|
||||
let bytes = files.get(path).ok_or("build executable is missing")?;
|
||||
if bytes.starts_with(b"#!") {
|
||||
let first = bytes
|
||||
.split(|byte| *byte == b'\n')
|
||||
.next()
|
||||
.unwrap_or_default();
|
||||
if !matches!(
|
||||
first,
|
||||
b"#!/bin/sh"
|
||||
| b"#!/usr/bin/sh"
|
||||
| b"#!/usr/bin/env sh"
|
||||
| b"#!/usr/bin/env node"
|
||||
| b"#!/usr/bin/env python3"
|
||||
) {
|
||||
return Err("build tool script must name the signed shell directly".into());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let elf = object(path, bytes, machine)?;
|
||||
if elf.interpreter.is_some_and(|value| value != interpreter) {
|
||||
return Err("build tool interpreter differs from signed sandbox loader".into());
|
||||
}
|
||||
closure.follow(path, vec![], &BTreeMap::new())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Wheel console/gui entry points share the installed interpreter prefix.
|
||||
|
||||
use super::InstalledFile;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn identifier(path: &str) -> bool {
|
||||
!path.is_empty()
|
||||
&& path.split('.').all(|part| {
|
||||
let mut bytes = part.bytes();
|
||||
bytes
|
||||
.next()
|
||||
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
|
||||
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn entry_points(
|
||||
bytes: &[u8],
|
||||
prefix: &str,
|
||||
interpreter: &str,
|
||||
) -> Result<Vec<InstalledFile>, String> {
|
||||
let text = std::str::from_utf8(bytes).map_err(|_| "wheel entry points must be UTF-8")?;
|
||||
let mut active = false;
|
||||
let mut sections = BTreeSet::new();
|
||||
let mut entries = BTreeMap::<String, String>::new();
|
||||
let mut previous: Option<String> = None;
|
||||
for line in text.lines() {
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() || trimmed.starts_with(['#', ';']) {
|
||||
continue;
|
||||
}
|
||||
if let Some(section) = trimmed
|
||||
.strip_prefix('[')
|
||||
.and_then(|value| value.strip_suffix(']'))
|
||||
{
|
||||
active = matches!(section, "console_scripts" | "gui_scripts");
|
||||
if active && !sections.insert(section.to_owned()) {
|
||||
return Err("wheel repeats an entry-point group".into());
|
||||
}
|
||||
previous = None;
|
||||
continue;
|
||||
}
|
||||
if !active {
|
||||
continue;
|
||||
}
|
||||
if line.starts_with(char::is_whitespace) && !trimmed.contains('=') {
|
||||
let name = previous
|
||||
.as_ref()
|
||||
.ok_or("wheel entry point has an orphan continuation")?;
|
||||
let value = entries.get_mut(name).expect("previous entry");
|
||||
value.push(' ');
|
||||
value.push_str(trimmed);
|
||||
continue;
|
||||
}
|
||||
let (name, value) = trimmed
|
||||
.split_once('=')
|
||||
.ok_or("invalid wheel script entry point")?;
|
||||
let name = name.trim();
|
||||
if name.is_empty()
|
||||
|| !name
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte))
|
||||
|| matches!(name, "." | "..")
|
||||
{
|
||||
return Err("wheel script name must be a portable file name".into());
|
||||
}
|
||||
if entries
|
||||
.insert(name.to_owned(), value.trim().to_owned())
|
||||
.is_some()
|
||||
{
|
||||
return Err("wheel repeats a script name".into());
|
||||
}
|
||||
previous = Some(name.to_owned());
|
||||
}
|
||||
entries.into_iter().map(|(name,value)| {
|
||||
let value = if let Some((target,extras))=value.split_once('[') {
|
||||
let extras=extras.strip_suffix(']').ok_or("invalid entry-point extras")?;
|
||||
for extra in extras.split(',') {
|
||||
extra.trim().parse::<pep508_rs::ExtraName>().map_err(|error|format!("invalid entry-point extra: {error}"))?;
|
||||
}
|
||||
target.trim()
|
||||
}else{value.trim()};
|
||||
let (module,function)=value.split_once(':').ok_or("wheel console script must name a callable")?;
|
||||
if !identifier(module) || !identifier(function) { return Err("wheel script has an invalid Python entry point".into()); }
|
||||
let source=format!("#!/worker/{interpreter} -I\nimport importlib\nimport re\nimport sys\nentry = importlib.import_module({module:?})\nfor part in {function:?}.split('.'):\n entry = getattr(entry, part)\nsys.argv[0] = re.sub(r'(-script\\.pyw|\\.exe)?$', '', sys.argv[0])\nsys.exit(entry())\n");
|
||||
Ok(InstalledFile {path:format!("{prefix}/bin/{name}"),bytes:source.into_bytes(),executable:true})
|
||||
}).collect()
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//! One runtime descriptor and file-closure contract for authoring and Worker preparation.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::Path;
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use super::{MAX_TREE_BYTES, MAX_TREE_FILES, go_build, valid_path};
|
||||
use crate::libs::Ecosystem;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub(crate) struct Manifest {
|
||||
format: String,
|
||||
ecosystem: String,
|
||||
pub(crate) target: String,
|
||||
pub(crate) executable: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(super) arguments: Vec<String>,
|
||||
pub(super) build: Option<go_build::GoBuildManifest>,
|
||||
#[serde(default)]
|
||||
pub(crate) python_wheel_tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(crate) python_extension_suffixes: Vec<String>,
|
||||
pub(crate) python_markers: Option<pep508_rs::MarkerEnvironment>,
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub(crate) fn executable(&self, path: &str) -> bool {
|
||||
self.executable.as_deref() == Some(path)
|
||||
|| self
|
||||
.build
|
||||
.as_ref()
|
||||
.is_some_and(|build| build.executable(path))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate(
|
||||
files: &[(String, Vec<u8>)],
|
||||
ecosystem: &Ecosystem,
|
||||
target: &str,
|
||||
) -> Result<Manifest, String> {
|
||||
if files.len() > MAX_TREE_FILES {
|
||||
return Err("managed runtime pack has too many files".into());
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
let mut total = 0usize;
|
||||
for (path, bytes) in files {
|
||||
valid_path(path)?;
|
||||
if !paths.insert(path.as_str()) {
|
||||
return Err("managed runtime pack repeats a file".into());
|
||||
}
|
||||
total = total
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("managed runtime pack size overflow")?;
|
||||
if total > MAX_TREE_BYTES {
|
||||
return Err("managed runtime pack exceeds 256 MiB".into());
|
||||
}
|
||||
}
|
||||
for path in &paths {
|
||||
for parent in Path::new(path).ancestors().skip(1) {
|
||||
if parent.to_str().is_some_and(|name| paths.contains(name)) {
|
||||
return Err("managed runtime pack has a file/directory conflict".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
let metadata = files
|
||||
.iter()
|
||||
.find(|(path, _)| path == "dever-runtime.json")
|
||||
.ok_or("managed runtime pack has no dever-runtime.json")?;
|
||||
let manifest: Manifest = serde_json::from_slice(&metadata.1)
|
||||
.map_err(|error| format!("invalid managed runtime manifest: {error}"))?;
|
||||
if manifest.format != "dever-worker-runtime-v1"
|
||||
|| manifest.ecosystem != ecosystem.as_str()
|
||||
|| manifest.target != target
|
||||
{
|
||||
return Err("managed runtime pack identity differs from its lock and target".into());
|
||||
}
|
||||
if ecosystem != &Ecosystem::Pip
|
||||
&& (!manifest.python_wheel_tags.is_empty()
|
||||
|| !manifest.python_extension_suffixes.is_empty()
|
||||
|| manifest.python_markers.is_some())
|
||||
{
|
||||
return Err("non-Python runtime must not declare Python ABI metadata".into());
|
||||
}
|
||||
if ecosystem == &Ecosystem::Go {
|
||||
if manifest.executable.is_some() || !manifest.arguments.is_empty() {
|
||||
return Err("managed Go build pack must not declare a runtime interpreter".into());
|
||||
}
|
||||
manifest
|
||||
.build
|
||||
.as_ref()
|
||||
.ok_or("managed Go Worker requires a locked compile/link build pack")?
|
||||
.validate_files(files, target)?;
|
||||
} else {
|
||||
if manifest.build.is_some() {
|
||||
return Err("managed interpreter pack must not declare Go build tools".into());
|
||||
}
|
||||
let executable = manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.ok_or("managed runtime executable is missing")?;
|
||||
valid_path(executable)?;
|
||||
if !executable.starts_with("bin/")
|
||||
|| manifest.arguments.len() > 64
|
||||
|| manifest
|
||||
.arguments
|
||||
.iter()
|
||||
.any(|argument| !argument.starts_with('-') || argument.contains('\0'))
|
||||
{
|
||||
return Err("managed runtime executable or fixed flags are invalid".into());
|
||||
}
|
||||
if ecosystem == &Ecosystem::Pip && manifest.arguments != ["-I", "-S", "-B"] {
|
||||
return Err(
|
||||
"managed Python runtime requires the fixed -I -S -B isolation flags".into(),
|
||||
);
|
||||
}
|
||||
if ecosystem == &Ecosystem::Pip {
|
||||
if manifest.python_markers.is_none() {
|
||||
return Err("managed Python runtime lacks its marker environment".into());
|
||||
}
|
||||
super::python_wheel::validate_target(
|
||||
target,
|
||||
&manifest.python_wheel_tags,
|
||||
manifest
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.expect("validated Python markers"),
|
||||
)?;
|
||||
let mut suffixes = BTreeSet::new();
|
||||
if manifest.python_extension_suffixes.is_empty()
|
||||
|| manifest.python_extension_suffixes.len() > 16
|
||||
|| manifest.python_extension_suffixes.iter().any(|suffix| {
|
||||
!suffix.starts_with('.')
|
||||
|| !suffix.ends_with(".so")
|
||||
|| !suffix
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte))
|
||||
|| !suffixes.insert(suffix)
|
||||
})
|
||||
{
|
||||
return Err(
|
||||
"managed Python extension suffixes are missing, invalid or repeated".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
if !files
|
||||
.iter()
|
||||
.any(|(path, bytes)| path == executable && !bytes.is_empty())
|
||||
{
|
||||
return Err("managed runtime executable is missing from its pack".into());
|
||||
}
|
||||
for (path, bytes) in files {
|
||||
if path == executable || bytes.starts_with(b"\x7fELF") {
|
||||
validate_binary(path, bytes, target)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
pub(crate) fn validate_binary(path: &str, bytes: &[u8], target: &str) -> Result<(), String> {
|
||||
use goblin::elf::{Elf, header};
|
||||
let (machine, interpreter) = match target.parse::<crate::toolchain::BuildTarget>()? {
|
||||
crate::toolchain::BuildTarget::LinuxX86_64 => {
|
||||
(header::EM_X86_64, "/lib64/ld-linux-x86-64.so.2")
|
||||
}
|
||||
crate::toolchain::BuildTarget::LinuxAarch64 => {
|
||||
(header::EM_AARCH64, "/lib/ld-linux-aarch64.so.1")
|
||||
}
|
||||
};
|
||||
let elf =
|
||||
Elf::parse(bytes).map_err(|error| format!("invalid runtime ELF '{path}': {error}"))?;
|
||||
if !elf.is_64
|
||||
|| !elf.little_endian
|
||||
|| elf.header.e_machine != machine
|
||||
|| !matches!(elf.header.e_type, header::ET_DYN | header::ET_EXEC)
|
||||
|| elf.interpreter.is_some_and(|actual| actual != interpreter)
|
||||
{
|
||||
return Err(format!("runtime ELF '{path}' differs from target {target}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user