dever/core
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: release Dever 0.1.5 with direct workers and long transactions

Browse Source
This commit is contained in:
Codex committed 2026-10-11 11:17:33 +00:00
1 parent 89629c2197
commit a900d7e8db
153 files changed
+3676 -3708
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/dever-cli/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ workspace = true
[dependencies]
dever-backend-bridge = { path = "../dever-backend-bridge", features = ["embedded"] }
dever-core.workspace = true
dever-sandbox = { path = "../dever-sandbox" }
dever-process = { path = "../dever-process" }
dever-runtime = { workspace = true, features = ["database"] }
serde_json.workspace = true
serde.workspace = true
crates/dever-cli/src/compile.rs
+3 -2
View File
@@ -117,11 +117,11 @@ fn test_profile(program: &Program) -> RuntimeProfile {
pub(super) fn worker_compile(
program: &Program,
sources: &SourceMap,
request: &CompileRequest,
request: &mut CompileRequest,
build: &Path,
output: &Path,
) -> Result<(), String> {
let resources = request.embedded_resources();
let resources = request.take_embedded_resources();
let (profile, ir) = match request.kind {
CompileKind::Application => {
let profile = program.validate_database_bindings(
@@ -209,6 +209,7 @@ fn link_ir(
.open(&path)
.and_then(|mut file| file.write_all(&object))
.map_err(|error| format!("cannot write application object: {error}"))?;
drop(object);
let snapshot = pack.snapshot(build)?;
let inputs = snapshot
.start
crates/dever-cli/src/libs.rs
+15 -14
View File
@@ -779,13 +779,13 @@ impl build::Inputs for InstalledRegistry {
}
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> {
self.sandbox_resources(crate::toolchain::BuildTarget::host()?)?
self.process_runtime_resources(crate::toolchain::BuildTarget::host()?)?
.into_iter()
.map(|resource| {
let path = resource
.path
.strip_prefix("sandbox/")
.ok_or("invalid sandbox resource path")?
.strip_prefix("process-runtime/")
.ok_or("invalid process_runtime resource path")?
.to_owned();
Ok((path, resource.bytes))
})
@@ -861,11 +861,11 @@ impl InstalledRegistry {
self.resources.read(name)
}
fn sandbox_resources(
fn process_runtime_resources(
&self,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<dever_core::native::EmbeddedResource>, String> {
let prefix = format!("sandbox/{}/", target.platform());
let prefix = format!("process-runtime/{}/", target.platform());
let mut resources = Vec::new();
let mut files = Vec::new();
for artifact in self.resources.artifacts(&prefix)? {
@@ -875,13 +875,13 @@ impl InstalledRegistry {
let bytes = self.signed_artifact(Path::new(&artifact.path))?;
files.push((relative.to_owned(), bytes.clone()));
resources.push(dever_core::native::EmbeddedResource {
path: format!("sandbox/{relative}"),
path: format!("process-runtime/{relative}"),
sha256: artifact.sha256.clone(),
bytes,
executable: dever_sandbox::asset_executable(relative),
executable: dever_process::asset_executable(relative),
});
}
dever_sandbox::validate_assets_for_target(&files, target.platform())?;
dever_process::validate_assets_for_target(&files, target.platform())?;
Ok(resources)
}
}
@@ -1488,13 +1488,14 @@ pub fn prepare_resources(
let mut host_assets = Vec::new();
if program.has_external_adapters() {
let installed = InstalledRegistry::current(target)?;
resources.extend(installed.sandbox_resources(target)?);
resources.extend(installed.process_runtime_resources(target)?);
if program
.external_worker_contracts()
.iter()
.any(|worker| worker.ecosystem == "go")
{
host_assets = installed.sandbox_resources(crate::toolchain::BuildTarget::host()?)?;
host_assets =
installed.process_runtime_resources(crate::toolchain::BuildTarget::host()?)?;
}
}
let packaged = crate::workers::prepare_for_target(
@@ -1725,9 +1726,9 @@ pub fn doctor_with_go_verifier(lock: &LockFile, verifier: &sumdb::Verifier) -> R
.iter()
.any(|node| node.source.archive == lib.spec)
}))
&& !fixtures
&& fixtures
.package(&lib.spec)
.is_some_and(|package| package.locked() == *lib)
.is_none_or(|package| package.locked() != *lib)
{
return Err(
"npm archive requires an exact instance graph, not flat dependencies".into(),
@@ -1735,9 +1736,9 @@ pub fn doctor_with_go_verifier(lock: &LockFile, verifier: &sumdb::Verifier) -> R
}
if lib.spec.ecosystem == Ecosystem::Go
&& !go.contains(&lib.spec.name, &lib.spec.version)
&& !fixtures
&& fixtures
.package(&lib.spec)
.is_some_and(|package| package.locked() == *lib)
.is_none_or(|package| package.locked() != *lib)
{
return Err(format!(
"{} lacks authenticated Go sumdb evidence",
crates/dever-cli/src/libs/build/npm.js
+24 -18
View File
@@ -1,27 +1,33 @@
// Fixed registry-package lifecycle frontend. Runs only in the build sandbox.
const fs = require('node:fs');
const path = require('node:path');
const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script');
const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links');
const runScript = require(path.join(__dirname, 'npm/frontend/node_modules/npm/node_modules/@npmcli/run-script'));
const binLinks = require(path.join(__dirname, 'npm/frontend/node_modules/npm/node_modules/bin-links'));
const shellQuote = value => "'" + value.replace(/'/g, "'\\''") + "'";
async function main() {
const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8'));
const root = '/data/build/output/install';
fs.mkdirSync('/data/build/output');
fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true });
fs.mkdirSync('/data/build/tmp');
fs.mkdirSync('/data/build/home');
const work = fs.realpathSync(process.argv[2]);
const toolRoot = path.join(__dirname, 'rootfs');
const tools = path.join(toolRoot, 'usr/bin');
const request = JSON.parse(fs.readFileSync(path.join(__dirname, 'request.json'), 'utf8'));
const root = path.join(work, 'output/install');
fs.mkdirSync(path.join(work, 'output'));
fs.cpSync(path.join(__dirname, 'install'), root, { recursive: true, dereference: false, errorOnExist: true });
fs.mkdirSync(path.join(work, 'tmp'));
fs.mkdirSync(path.join(work, 'home'));
Object.assign(process.env, {
PATH: '/worker/runtime/bin:/usr/bin',
CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar',
PATH: [path.join(__dirname, 'runtime/bin'), tools].join(path.delimiter),
CC: `${shellQuote(path.join(tools, 'cc'))} ${shellQuote('--sysroot=' + toolRoot)}`,
CXX: `${shellQuote(path.join(tools, 'c++'))} ${shellQuote('--sysroot=' + toolRoot)}`,
AR: path.join(tools, 'ar'),
PYTHON: request.python,
npm_config_python: request.python,
npm_config_nodedir: '/worker/npm',
npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js',
npm_config_nodedir: path.join(__dirname, 'npm'),
npm_config_node_gyp: path.join(__dirname, 'npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js'),
npm_config_build_from_source: 'true',
npm_config_cache: '/data/build/npm-cache',
TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0',
HOME: '/data/build/home',
npm_config_cache: path.join(work, 'npm-cache'),
TMPDIR: path.join(work, 'tmp'), SOURCE_DATE_EPOCH: '0',
HOME: path.join(work, 'home'),
});
const nodes = new Map(request.instances.map(node => [node.path, node]));
for (const node of request.instances) {
@@ -48,7 +54,7 @@ async function main() {
}
const directory = path.join(root, name);
for (const event of ['preinstall', 'install', 'postinstall']) {
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' });
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: path.join(tools, 'sh') });
}
const native = path.join(directory, 'build', 'Release');
if (fs.existsSync(native)) {
@@ -97,7 +103,7 @@ async function main() {
fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true });
}
fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true });
fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
fs.writeFileSync(path.join(work, 'output/result.json'), JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
}
main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; });
crates/dever-cli/src/libs/build/npm/environment.rs
+18 -31
View File
@@ -70,7 +70,7 @@ impl Environment {
descriptor.validate(runtime, Some(&python))?;
let (manifest, tools) = pack::unpack(&bytes, &descriptor)?;
let assets = inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
dever_process::validate_assets(&assets)?;
pack::validate_tools(&manifest, &tools, &assets)?;
let mut files = BTreeMap::new();
for (path, bytes) in node_files {
@@ -122,9 +122,9 @@ impl Environment {
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
&format!("process-runtime/{path}"),
bytes,
dever_sandbox::asset_executable(path),
dever_process::asset_executable(path),
)?;
}
for (path, bytes) in files {
@@ -137,7 +137,7 @@ impl Environment {
let request = serde_json::json!({
"instances":graph.instances, "required":super::required_nodes(graph)?,
"roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(),
"python":format!("/worker/{}", self.python),
"python":input.join(&self.python).to_string_lossy(),
});
stage.write(
"input/request.json",
@@ -147,31 +147,20 @@ impl Environment {
stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [input.join("frontend.js").into_os_string()];
let arguments = [
input.join("frontend.js").into_os_string(),
work.clone().into_os_string(),
];
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
let assets = stage.0.join("process-runtime");
let launch = dever_process::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated npm lifecycle",
&work,
)?;
stage.run(launch.command()?, "managed npm lifecycle", &work)?;
inputs.diagnostic(&stage.diagnostics()?);
let mut outputs = super::super::python::output_files(&work.join("output"))?;
let response: ResultMetadata = serde_json::from_slice(
@@ -210,7 +199,7 @@ impl Environment {
rejections: BTreeMap::new(),
};
// Release the large compiler tree before staging the much smaller
// runtime-only native admission sandbox.
// runtime-only native admission process_runtime.
drop(stage);
self.admit_native(inputs, &mut output)?;
Ok(output)
@@ -244,7 +233,7 @@ impl Environment {
files.extend(
self.assets
.iter()
.map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())),
.map(|(path, bytes)| (format!("process-runtime/{path}"), bytes.as_slice())),
);
files.extend(
output
@@ -254,21 +243,21 @@ impl Environment {
);
let stage = Stage::new()?;
for (path, bytes) in &files {
let relative = if path.starts_with("sandbox/") {
let relative = if path.starts_with("process-runtime/") {
path.clone()
} else {
format!("input/{path}")
};
let executable = path == &self.node
|| path
.strip_prefix("sandbox/")
.is_some_and(dever_sandbox::asset_executable);
.strip_prefix("process-runtime/")
.is_some_and(dever_process::asset_executable);
stage.write(&relative, bytes, executable)?;
}
stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?;
let input = stage.0.join("input");
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
let assets = stage.0.join("process-runtime");
for (path, bytes) in candidates {
let rejection = match goblin::elf::Elf::parse(bytes) {
Err(_) => Some(NativeRejection::Format),
@@ -293,14 +282,12 @@ impl Environment {
input.join("probe.cjs").into_os_string(),
input.join(path).into_os_string(),
];
let launch = dever_sandbox::Launch {
let launch = dever_process::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities::default(),
grants: &[],
};
let admitted = stage.probe(launch.command()?)?;
inputs.diagnostic(&stage.diagnostics()?);
crates/dever-cli/src/libs/build/pack.rs
+1 -1
View File
@@ -156,7 +156,7 @@ pub fn validate_tools(
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in assets {
supplied.insert(format!("sandbox/{path}"), bytes);
supplied.insert(format!("process-runtime/{path}"), bytes);
}
crate::workers::python_wheel::native::validate_build_tools(
&manifest.target,
crates/dever-cli/src/libs/build/process.rs
+48 -10
View File
@@ -51,8 +51,6 @@ impl Stage {
let path = self.0.join(relative);
fs::create_dir_all(path.parent().expect("validated relative path"))
.map_err(|error| format!("cannot create build input directory: {error}"))?;
// This tree is private and never a writable child mount. A second write
// must not silently replace a previously verified input.
use std::io::Write;
let mut file = fs::OpenOptions::new()
.write(true)
@@ -89,7 +87,7 @@ impl Stage {
pub(crate) fn probe(&self, command: Command) -> Result<bool, String> {
match self
.execute(command, "isolated Node addon probe", None)?
.execute(command, "managed Node addon probe", None)?
.code()
{
Some(0) => Ok(true),
@@ -107,8 +105,6 @@ impl Stage {
operation: &str,
writable: Option<&Path>,
) -> Result<std::process::ExitStatus, String> {
// Captures are outside every child-writable grant. Namespace teardown
// reaps descendants when the owned bootstrap is killed on any exit path.
let stdout = self.0.join("tool-stdout");
let stderr = self.0.join("tool-stderr");
let child = command
@@ -121,7 +117,7 @@ impl Stage {
))
.spawn()
.map_err(|error| format!("cannot execute {operation}: {error}"))?;
let mut child = OwnedChild(child);
let mut child = OwnedChild::new(child)?;
let deadline = Instant::now() + DEADLINE;
let mut next_scratch_check = Instant::now();
let within_budget = || -> Result<(), String> {
@@ -141,7 +137,6 @@ impl Stage {
next_scratch_check = Instant::now() + Duration::from_millis(250);
}
if let Some(status) = child
.0
.try_wait()
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
{
@@ -176,11 +171,54 @@ impl Drop for Stage {
}
}
struct OwnedChild(Child);
struct OwnedChild {
child: Child,
#[cfg(target_os = "linux")]
group: Option<dever_process::ProcessGroup>,
}
impl OwnedChild {
fn new(mut child: Child) -> Result<Self, String> {
#[cfg(target_os = "linux")]
match dever_process::ProcessGroup::pin(child.id()) {
Ok(group) => Ok(Self {
child,
group: Some(group),
}),
Err(error) => {
let _ = child.kill();
let _ = child.wait();
Err(error)
}
}
#[cfg(not(target_os = "linux"))]
{
let _ = child.kill();
let _ = child.wait();
Err("managed builds require Linux".into())
}
}
fn try_wait(&mut self) -> Result<Option<std::process::ExitStatus>, String> {
#[cfg(target_os = "linux")]
if let Some(group) = &self.group {
if !group.exited()? {
return Ok(None);
}
group.terminate()?;
self.group.take();
}
self.child.try_wait().map_err(|error| error.to_string())
}
}
impl Drop for OwnedChild {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
#[cfg(target_os = "linux")]
if let Some(group) = self.group.take() {
let _ = group.terminate();
}
let _ = self.child.kill();
let _ = self.child.wait();
}
}
crates/dever-cli/src/libs/build/process/budget.rs
+1 -1
View File
@@ -64,5 +64,5 @@ fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> {
#[cfg(not(unix))]
fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> {
Err("managed build scratch accounting requires its Linux sandbox".into())
Err("managed build scratch accounting requires its Linux process_runtime".into())
}
crates/dever-cli/src/libs/build/python.py
+17 -12
View File
@@ -1,4 +1,4 @@
"""Fixed PEP 517 frontend, executed only inside the managed build sandbox."""
"""Fixed PEP 517 frontend executed with explicit build inputs."""
import importlib
import json
@@ -7,8 +7,11 @@ from pathlib import Path
import re
import shutil
import sys
import shlex
import tomllib
INPUT = Path(__file__).resolve().parent
def build_system(source):
project = source / "pyproject.toml"
@@ -82,7 +85,7 @@ def execute(request, source, output):
return {"metadata": str(prepared.relative_to(output)) if prepared else None}
prepared = None
if request.get("metadata") is not None:
shutil.copytree("/worker/prepared", output / "prepared")
shutil.copytree(INPUT / "prepared", output / "prepared")
prepared = output / "prepared" / request["metadata"]
if not prepared.resolve().is_relative_to((output / "prepared").resolve()):
raise ValueError("prepared metadata path escapes its snapshot")
@@ -93,22 +96,24 @@ def execute(request, source, output):
def main():
request = json.loads(Path("/worker/request.json").read_text("utf-8"))
work = Path("/data/build")
request = json.loads((INPUT / "request.json").read_text("utf-8"))
work = Path(sys.argv[1]).resolve(strict=True)
tool_root = INPUT / "rootfs"
tools = tool_root / "usr/bin"
source = work / "source"
shutil.copytree("/worker/source", source)
shutil.copytree(INPUT / "source", source)
output = work / "output"
output.mkdir()
(work / "tmp").mkdir()
# These values are part of the fixed frontend, never project/host settings.
os.environ.update({
"PATH": "/worker/runtime/bin:/usr/bin",
"CC": "/usr/bin/cc", "CXX": "/usr/bin/c++",
"AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib",
"LDSHARED": "/usr/bin/cc -shared",
"CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.",
"CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"],
"TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0",
"PATH": os.pathsep.join([str(INPUT / "runtime/bin"), str(tools)]),
"CC": shlex.join([str(tools / "cc"), "--sysroot=" + str(tool_root)]), "CXX": shlex.join([str(tools / "c++"), "--sysroot=" + str(tool_root)]),
"AR": str(tools / "ar"), "RANLIB": str(tools / "ranlib"),
"LDSHARED": shlex.join([str(tools / "cc"), "--sysroot=" + str(tool_root), "-shared"]),
"CFLAGS": shlex.join(["-fPIC", "-ffile-prefix-map=" + str(source) + "=."]),
"CPPFLAGS": shlex.join(["-I" + str(INPUT / "runtime/include" / ("python" + request["python_version"]))]),
"TMPDIR": str(work / "tmp"), "SOURCE_DATE_EPOCH": "0",
})
os.chdir(source)
result = execute(request, source, output)
crates/dever-cli/src/libs/build/python.rs
+8 -21
View File
@@ -358,7 +358,7 @@ impl Environment {
descriptor.validate(requested, None)?;
let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?;
let assets = session.inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
dever_process::validate_assets(&assets)?;
pack::validate_tools(&tools, &tool_files, &assets)?;
for (path, bytes) in tool_files {
let executable = tools.executables.contains(&path);
@@ -402,9 +402,9 @@ impl Environment {
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
&format!("process-runtime/{path}"),
bytes,
dever_sandbox::asset_executable(path),
dever_process::asset_executable(path),
)?;
}
for (path, bytes) in &source.files {
@@ -472,7 +472,7 @@ impl Environment {
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in &self.assets {
supplied.insert(format!("sandbox/{path}"), bytes);
supplied.insert(format!("process-runtime/{path}"), bytes);
}
for (path, bytes) in &installed {
supplied.insert(path.clone(), bytes);
@@ -494,35 +494,22 @@ impl Environment {
stage.write("input/frontend.py", include_bytes!("python.py"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [
OsString::from("-I"),
OsString::from("-B"),
input.join("frontend.py").into_os_string(),
work.clone().into_os_string(),
];
let executable = input.join(&self.executable);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
let assets = stage.0.join("process-runtime");
let launch = dever_process::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated Python build hook",
&work,
)?;
stage.run(launch.command()?, "isolated Python build hook", &work)?;
inputs.diagnostic(&stage.diagnostics()?);
let outputs = output_files(&work.join("output"))?;
let response = serde_json::from_slice(
crates/dever-cli/src/libs/registry.rs
+1 -1
View File
@@ -56,7 +56,7 @@ impl InstalledRegistryPack {
if self.format != "dever-registry-runtime-v1"
|| self.runtime.target != target
|| (ecosystem == &Ecosystem::Pip) != self.runtime.python_markers.is_some()
|| (ecosystem == &Ecosystem::Pip) != !self.runtime.python_wheel_tags.is_empty()
|| (ecosystem == &Ecosystem::Pip) == self.runtime.python_wheel_tags.is_empty()
{
return Err(format!(
"signed {} runtime manifest does not match this target",
crates/dever-cli/src/toolchain/bootstrap.rs
+20 -25
View File
@@ -11,7 +11,6 @@ use sha2::{Digest, Sha256};
use super::release::{self, InstallLock};
use super::{Artifact, Layout, MachineManager, ReleaseManifest, Version};
mod sandbox;
mod service;
mod transaction;
pub use service::{RecoveryAction, recovery_actions};
@@ -45,11 +44,6 @@ pub fn install(installer_root: &Path) -> Result<Version, String> {
})?;
absolute_root(&settings.system_root)?;
trusted_directory(&settings.system_root)?;
if settings.system_root == Path::new("/") {
// Publishing an AppArmor file can trigger a later system-wide reload,
// even when this installer does not load profiles or activate services.
dever_sandbox::validate_host_policy()?;
}
if settings.activate_service && settings.system_root != Path::new("/") {
return Err(
"service activation requires the actual system root; image installs are offline".into(),
@@ -96,7 +90,7 @@ pub fn install(installer_root: &Path) -> Result<Version, String> {
// The private copy is checked again by the version owner before any health
// command executes. A writable download cannot race signed bytes into it.
let manifest = MachineManager::verify_package(&package, None, &key)?;
validate_payload(&package, &manifest.artifacts, true)?;
validate_payload(&package, &manifest.artifacts, true, false)?;
pin_key(&layout, &key)?;
let manager = MachineManager::new(layout.clone());
if manager
@@ -140,6 +134,7 @@ pub(crate) fn validate_payload(
root: &Path,
artifacts: &[Artifact],
current_template: bool,
previous_layout: bool,
) -> Result<(), String> {
let scoped = artifacts
.iter()
@@ -154,10 +149,20 @@ pub(crate) fn validate_payload(
})
})
.collect::<Vec<_>>();
validate_files(&root.join("bootstrap"), &scoped, current_template)
validate_files(
&root.join("bootstrap"),
&scoped,
current_template,
previous_layout,
)
}
fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Result<(), String> {
fn validate_files(
root: &Path,
artifacts: &[Artifact],
new_payload: bool,
previous_layout: bool,
) -> Result<(), String> {
let bytes = artifacts
.iter()
.try_fold(0u64, |total, artifact| total.checked_add(artifact.bytes))
@@ -166,11 +171,12 @@ fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Res
return Err("bootstrap closure exceeds its 64-file/64-MiB budget".into());
}
for artifact in artifacts {
if !matches!(
let supported = matches!(
artifact.path.as_str(),
"dever" | "deverd" | "deverd.service" | "dever-sandbox"
) && !artifact.path.starts_with("lib/")
{
"dever" | "deverd" | "deverd.service"
) || artifact.path.starts_with("lib/")
|| previous_layout && !new_payload && artifact.path == "dever-sandbox";
if !supported {
return Err("bootstrap has an unsupported artifact".into());
}
let path = super::runtime_pack::checked_path(root, &artifact.path, true)?;
@@ -191,17 +197,6 @@ fn validate_files(root: &Path, artifacts: &[Artifact], new_payload: bool) -> Res
"bootstrap service definition is absent or differs from the supported contract".into(),
);
}
if new_payload
&& (!artifacts
.iter()
.any(|artifact| artifact.path == "dever-sandbox")
|| read_bounded(&root.join("dever-sandbox"), 8192)?
!= dever_sandbox::APPARMOR_PROFILE.as_bytes())
{
return Err(
"bootstrap AppArmor profile is absent or differs from the supported contract".into(),
);
}
super::core_libraries::validate_entries(root, artifacts, &["dever", "deverd"])
}
@@ -245,7 +240,7 @@ fn validate_bin(root: &Path, key: &[u8]) -> Result<Vec<u8>, String> {
})
})
.collect::<Vec<_>>();
validate_files(root, &artifacts, false)?;
validate_files(root, &artifacts, false, manifest.uses_sandbox_layout())?;
let expected = artifacts
.iter()
.map(|artifact| artifact.path.as_str())
crates/dever-cli/src/toolchain/bootstrap/sandbox.rs
-138
View File
@@ -1,138 +0,0 @@
//! Preserve signed content-addressed helpers across compiler upgrades/removal.
use std::fs;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use super::{
MachineManager, ReleaseManifest, digest, read_bounded, release, trusted_directory, trusted_file,
};
const MAX_HELPERS: usize = 24;
const MAX_HELPER_BYTES: u64 = 32 * 1024 * 1024;
fn artifact_path() -> String {
format!(
"sandbox/{}/bin/bwrap",
crate::toolchain::platform_identity()
)
}
pub(super) fn stage(
installed: &Path,
next: &Path,
source: &Path,
manifest: &ReleaseManifest,
key: &[u8],
) -> Result<(), String> {
fs::create_dir(next).map_err(|error| error.to_string())?;
fs::set_permissions(next, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
let mut hashes = std::collections::BTreeSet::new();
if installed.exists() {
trusted_directory(installed)?;
for entry in fs::read_dir(installed).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let hash = entry
.file_name()
.into_string()
.map_err(|_| "sandbox helper identity is not UTF-8")?;
if hashes.len() >= MAX_HELPERS || !valid_hash(&hash) {
return Err("installed sandbox helper store has unsupported entries or exceeds its version bound".into());
}
validate_installed(&entry.path(), &hash, key)?;
let destination = next.join(&hash);
copy_helper(&entry.path().join("bwrap"), &entry.path(), &destination)?;
hashes.insert(hash);
}
}
if let Some(artifact) = manifest
.artifacts
.iter()
.find(|artifact| artifact.path == artifact_path())
&& !hashes.contains(&artifact.sha256)
{
if hashes.len() >= MAX_HELPERS {
return Err("sandbox helper version store is full; retained applications must be retired before removing old helpers".into());
}
let bytes = read_bounded(&source.join(&artifact.path), MAX_HELPER_BYTES)?;
if bytes.len() as u64 != artifact.bytes
|| dever_sandbox::validate_bwrap(&bytes)? != artifact.sha256
{
return Err("sandbox helper differs from its signed release".into());
}
let destination = next.join(&artifact.sha256);
copy_helper(&source.join(&artifact.path), source, &destination)?;
validate_installed(&destination, &artifact.sha256, key)?;
}
release::sync_tree(next)
}
fn copy_helper(helper: &Path, signed_release: &Path, destination: &Path) -> Result<(), String> {
fs::create_dir(destination).map_err(|error| error.to_string())?;
fs::set_permissions(destination, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
release::copy_file(helper, &destination.join("bwrap"), true)?;
for name in ["manifest.json", "manifest.sig"] {
release::copy_file(&signed_release.join(name), &destination.join(name), false)?;
}
Ok(())
}
fn valid_hash(hash: &str) -> bool {
hash.len() == 64
&& hash
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
}
fn validate_installed(root: &Path, hash: &str, key: &[u8]) -> Result<(), String> {
use std::os::unix::fs::MetadataExt;
trusted_directory(root)?;
let names = fs::read_dir(root)
.map_err(|error| error.to_string())?
.map(|entry| entry.map(|entry| entry.file_name()))
.collect::<Result<std::collections::BTreeSet<_>, _>>()
.map_err(|error| error.to_string())?;
let expected = ["bwrap", "manifest.json", "manifest.sig"]
.map(std::ffi::OsString::from)
.into_iter()
.collect();
if names != expected {
return Err("installed sandbox helper contains unsigned entries".into());
}
for name in ["bwrap", "manifest.json", "manifest.sig"] {
trusted_file(&root.join(name))?;
}
let helper = root.join("bwrap");
if fs::metadata(&helper)
.map_err(|error| error.to_string())?
.mode()
& 0o6111
!= 0o111
{
return Err("installed sandbox helper has unsupported executable permissions".into());
}
match rustix::fs::lgetxattr(&helper, "security.capability", &mut [0u8; 64]) {
Err(rustix::io::Errno::NODATA | rustix::io::Errno::OPNOTSUPP) => {}
Ok(_) => return Err("installed sandbox helper must not have file capabilities".into()),
Err(error) => {
return Err(format!(
"cannot inspect installed sandbox helper capabilities: {error}"
));
}
}
let manifest = MachineManager::signed_manifest(root, key)?;
let artifact = manifest
.artifacts
.iter()
.find(|artifact| artifact.path == artifact_path())
.ok_or("installed sandbox helper has no signed artifact")?;
let bytes = read_bounded(&helper, MAX_HELPER_BYTES)?;
if artifact.sha256 != hash || artifact.bytes != bytes.len() as u64 || digest(&bytes) != hash {
return Err(
"installed sandbox helper differs from its signature or directory identity".into(),
);
}
dever_sandbox::validate_bwrap(&bytes)?;
Ok(())
}
crates/dever-cli/src/toolchain/bootstrap/service.rs
+2 -2
View File
@@ -95,8 +95,8 @@ fn property(name: &str) -> Result<String, String> {
.map_err(|_| "service state output is not UTF-8")?
.trim()
.to_owned();
if !status.success()
&& !(status.code() == Some(1) && name == "LoadState" && value == "not-found")
if !(status.success()
|| status.code() == Some(1) && name == "LoadState" && value == "not-found")
{
return Err(format!("cannot inspect Dever service {name}: {status}"));
}
crates/dever-cli/src/toolchain/bootstrap/transaction.rs
+1 -32
View File
@@ -22,17 +22,13 @@ const MAX_TREE_BYTES: u64 = 80 * 1024 * 1024;
#[serde(rename_all = "snake_case")]
enum Target {
Bin,
Sandbox,
Apparmor,
Service,
Entry,
Enable,
Active,
}
const TARGETS: [Target; 7] = [
Target::Sandbox,
Target::Apparmor,
const TARGETS: [Target; 5] = [
Target::Bin,
Target::Service,
Target::Entry,
@@ -44,8 +40,6 @@ impl Target {
fn relative(self) -> &'static str {
match self {
Self::Bin => "opt/dever/bin",
Self::Sandbox => "opt/dever/sandbox",
Self::Apparmor => "etc/apparmor.d/dever-sandbox",
Self::Service => "etc/systemd/system/deverd.service",
Self::Entry => "usr/local/bin/dever",
Self::Enable => "etc/systemd/system/multi-user.target.wants/deverd.service",
@@ -136,16 +130,6 @@ pub(super) fn install(
"bootstrap cannot replace an unrelated public entry or service link".into(),
);
}
} else if target == Target::Apparmor && target.path(root).exists() {
trusted_file(&target.path(root))?;
let installed_profile = layout.bin().join("dever-sandbox");
if previous_service.is_none()
|| !installed_profile.exists()
|| read_bounded(&target.path(root), 8192)?
!= read_bounded(&installed_profile, 8192)?
{
return Err("bootstrap cannot replace an unrelated AppArmor profile".into());
}
} else if target == Target::Service && target.path(root).exists() {
trusted_file(&target.path(root))?;
if previous_service.as_deref()
@@ -220,18 +204,6 @@ pub(super) fn install(
}
fn stage(root: &Path, source: &Path, manifest: &ReleaseManifest, key: &[u8]) -> Result<(), String> {
super::sandbox::stage(
&Target::Sandbox.path(root),
&Target::Sandbox.next(root),
source,
manifest,
key,
)?;
write_new(
&Target::Apparmor.next(root),
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
0o644,
)?;
let bin = Target::Bin.next(root);
fs::create_dir(&bin).map_err(|error| error.to_string())?;
for artifact in &manifest.artifacts {
@@ -276,9 +248,6 @@ fn stage(root: &Path, source: &Path, manifest: &ReleaseManifest, key: &[u8]) ->
}
fn publish(root: &Path, change: &Change) -> Result<(), String> {
if change.target == Target::Apparmor && root == Path::new("/") {
dever_sandbox::validate_host_policy()?;
}
let path = change.target.path(root);
let next = change.target.next(root);
if identity(&path, change.target.link())? != change.before
crates/dever-cli/src/toolchain/compilation.rs
+5 -5
View File
@@ -169,13 +169,13 @@ impl CompileRequest {
)
}
pub fn embedded_resources(&self) -> Vec<EmbeddedResource> {
pub fn take_embedded_resources(&mut self) -> Vec<EmbeddedResource> {
self.resources
.iter()
.drain(..)
.map(|resource| EmbeddedResource {
path: resource.path.clone(),
bytes: resource.bytes.clone(),
sha256: resource.sha256.clone(),
path: resource.path,
bytes: resource.bytes,
sha256: resource.sha256,
executable: resource.executable,
})
.collect()
crates/dever-cli/src/toolchain/core_libraries.rs
+1 -1
View File
@@ -52,7 +52,7 @@ pub(super) fn validate_entries(
entries: &[&str],
) -> Result<(), String> {
runtime_pack::host_target()?;
dever_sandbox::require_no_loader_preload()?;
dever_process::require_no_loader_preload()?;
let mut files = BTreeMap::new();
let mut total = 0u64;
for artifact in artifacts {
crates/dever-cli/src/toolchain/extensions.rs
+8 -3
View File
@@ -55,7 +55,7 @@ impl ExtensionId {
format!("ext-{}.tar.zst", self.name())
}
fn permits(&self, path: &str) -> bool {
fn permits(&self, path: &str, process_root: &str) -> bool {
let target = self.target.platform();
match &self.kind {
ExtensionKind::Runtime(ecosystem) => {
@@ -66,7 +66,7 @@ impl ExtensionId {
}
ExtensionKind::Target => {
path.starts_with(&format!("runtime/{target}/"))
|| path.starts_with(&format!("sandbox/{target}/"))
|| path.starts_with(&format!("{process_root}/{target}/"))
}
}
}
@@ -113,6 +113,11 @@ pub fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
}
let mut paths = BTreeSet::new();
validate_artifacts(&manifest.artifacts, &mut paths)?;
let process_root = if manifest.uses_sandbox_layout() {
"sandbox"
} else {
"process-runtime"
};
let mut ids = BTreeSet::new();
for extension in &manifest.extensions {
let id = extension.id();
@@ -126,7 +131,7 @@ pub fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
if extension
.artifacts
.iter()
.any(|artifact| !id.permits(&artifact.path))
.any(|artifact| !id.permits(&artifact.path, process_root))
{
return Err(format!(
"extension '{}' contains a file outside its resource namespace",
crates/dever-cli/src/toolchain/packaging.rs
+8 -8
View File
@@ -22,7 +22,7 @@ static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
mod bootstrap;
mod builds;
mod ecosystems;
mod sandbox;
mod process_runtime;
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
@@ -45,7 +45,7 @@ struct Settings {
#[serde(default)]
runtimes: ecosystems::RuntimeInputs,
#[serde(default)]
sandbox: Option<Vec<LinkInput>>,
process_runtime: Option<Vec<LinkInput>>,
#[serde(default)]
builds: builds::Inputs,
#[serde(default)]
@@ -53,7 +53,7 @@ struct Settings {
#[serde(default)]
build_targets: BTreeMap<BuildTarget, builds::Inputs>,
#[serde(default)]
sandbox_targets: BTreeMap<BuildTarget, Vec<LinkInput>>,
process_runtime_targets: BTreeMap<BuildTarget, Vec<LinkInput>>,
}
#[derive(Deserialize)]
@@ -232,14 +232,14 @@ fn assemble(
});
}
}
if let Some(inputs) = settings.sandbox {
artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?);
if let Some(inputs) = settings.process_runtime {
artifacts.extend(process_runtime::assemble(&mut writer, inputs, host)?);
}
for (target, inputs) in settings.sandbox_targets {
for (target, inputs) in settings.process_runtime_targets {
let selected = target_artifacts
.get_mut(&target)
.ok_or("target sandbox requires a matching native target")?;
selected.extend(sandbox::assemble(&mut writer, inputs, target)?);
.ok_or("target process_runtime requires a matching native target")?;
selected.extend(process_runtime::assemble(&mut writer, inputs, target)?);
}
for (target, artifacts) in target_artifacts {
extensions.push(Extension {
crates/dever-cli/src/toolchain/packaging/bootstrap.rs
+11 -20
View File
@@ -39,26 +39,17 @@ pub(super) fn assemble(
#[cfg(target_os = "linux")]
{
use sha2::{Digest, Sha256};
for (path, bytes) in [
(
"bootstrap/deverd.service",
crate::toolchain::bootstrap::SERVICE.as_bytes(),
),
(
"bootstrap/dever-sandbox",
dever_sandbox::APPARMOR_PROFILE.as_bytes(),
),
] {
let destination = writer.reserve(path)?;
std::fs::write(&destination, bytes).map_err(|error| error.to_string())?;
super::set_mode(&destination, false)?;
artifacts.push(Artifact {
path: path.into(),
bytes: bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(bytes)),
});
}
crate::toolchain::bootstrap::validate_payload(writer.staging, &artifacts, true)?;
let path = "bootstrap/deverd.service";
let bytes = crate::toolchain::bootstrap::SERVICE.as_bytes();
let destination = writer.reserve(path)?;
std::fs::write(&destination, bytes).map_err(|error| error.to_string())?;
super::set_mode(&destination, false)?;
artifacts.push(Artifact {
path: path.into(),
bytes: bytes.len() as u64,
sha256: format!("{:x}", Sha256::digest(bytes)),
});
crate::toolchain::bootstrap::validate_payload(writer.staging, &artifacts, true, false)?;
}
Ok(artifacts)
}
crates/dever-cli/src/toolchain/packaging/builds.rs
+14 -6
View File
@@ -110,22 +110,30 @@ pub(super) fn assemble(
files.sort_by(|left, right| left.0.cmp(&right.0));
manifest.validate(&files, &descriptor)?;
let mut assets = Vec::new();
for directory in ["bin", "lib"] {
for entry in fs::read_dir(writer.staging.join(format!("sandbox/{target}/{directory}")))
.map_err(|error| format!("build pack requires prepared sandbox assets: {error}"))?
for directory in ["lib"] {
for entry in fs::read_dir(
writer
.staging
.join(format!("process-runtime/{target}/{directory}")),
)
.map_err(|error| format!("build pack requires prepared process_runtime assets: {error}"))?
{
let entry = entry.map_err(|error| error.to_string())?;
let name = entry
.file_name()
.into_string()
.map_err(|_| "sandbox asset name is not UTF-8")?;
.map_err(|_| "process_runtime asset name is not UTF-8")?;
assets.push((
format!("{directory}/{name}"),
read_bounded(&entry.path(), 32 * 1024 * 1024, "sandbox build asset")?,
read_bounded(
&entry.path(),
32 * 1024 * 1024,
"process_runtime build asset",
)?,
));
}
}
dever_sandbox::validate_assets_for_target(&assets, &target)?;
dever_process::validate_assets_for_target(&assets, &target)?;
pack::validate_tools(&manifest, &files, &assets)?;
let prefix = format!("build/{}/{target}", ecosystem.as_str());
let archive_name = format!("{prefix}/build.pack");
crates/dever-cli/src/toolchain/packaging/sandbox.rs → crates/dever-cli/src/toolchain/packaging/process_runtime.rs
+7 -9
View File
@@ -1,5 +1,3 @@
//! Sandbox tools and OS libraries are authenticated by the existing release.
use super::{Artifact, LinkInput, PayloadWriter, read_bounded};
use sha2::{Digest, Sha256};
@@ -9,30 +7,30 @@ pub(super) fn assemble(
target: crate::toolchain::BuildTarget,
) -> Result<Vec<Artifact>, String> {
if inputs.len() > 128 {
return Err("sandbox pack contains too many files".into());
return Err("process_runtime pack contains too many files".into());
}
let mut files = Vec::new();
let mut remaining = 32 * 1024 * 1024;
for input in inputs {
super::runtime_pack::relative_path(&input.path)?;
let source = writer.source(&input.source, &input.path, &input.sha256)?;
let bytes = read_bounded(&source, remaining, "sandbox asset")?;
let bytes = read_bounded(&source, remaining, "process_runtime asset")?;
if format!("{:x}", Sha256::digest(&bytes)) != input.sha256 {
return Err("sandbox asset changed during packaging".into());
return Err("process_runtime asset changed during packaging".into());
}
remaining -= bytes.len() as u64;
files.push((input.path, bytes));
}
files.sort_by(|left, right| left.0.cmp(&right.0));
dever_sandbox::validate_assets_for_target(&files, target.platform())?;
dever_process::validate_assets_for_target(&files, target.platform())?;
// Leaf artifacts avoid a second manifest/signature or extraction protocol.
let mut artifacts = Vec::new();
for (path, bytes) in files {
let destination = format!("sandbox/{}/{path}", target.platform());
let destination = format!("process-runtime/{}/{path}", target.platform());
let output = writer.reserve(&destination)?;
std::fs::write(&output, &bytes)
.map_err(|error| format!("cannot write sandbox asset: {error}"))?;
super::set_mode(&output, dever_sandbox::asset_executable(&path))?;
.map_err(|error| format!("cannot write process_runtime asset: {error}"))?;
super::set_mode(&output, dever_process::asset_executable(&path))?;
artifacts.push(Artifact {
path: destination,
bytes: bytes.len() as u64,
crates/dever-cli/src/toolchain/release.rs
+33 -5
View File
@@ -78,6 +78,10 @@ pub struct ReleaseManifest {
}
impl ReleaseManifest {
pub(super) fn uses_sandbox_layout(&self) -> bool {
version_components(self.version.as_str()) < [0, 1, 5]
}
pub fn new(version: Version, artifacts: Vec<Artifact>) -> Self {
Self {
format: "dever-release-v2".into(),
@@ -703,7 +707,12 @@ impl MachineManager {
{
// Older installed versions retain their own signed service template.
// Only the explicit bootstrap installer applies the current template.
super::bootstrap::validate_payload(root, &manifest.artifacts, false)?;
super::bootstrap::validate_payload(
root,
&manifest.artifacts,
false,
manifest.uses_sandbox_layout(),
)?;
}
Ok(manifest)
}
@@ -947,16 +956,35 @@ pub(super) fn ensure_real_directory(path: &Path, label: &str) -> Result<(), Stri
}
fn create_private_directory(path: &Path) -> Result<(), String> {
match fs::symlink_metadata(path) {
Ok(_) => return ensure_real_directory(path, "machine directory"),
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(format!("cannot inspect '{}': {error}", path.display())),
}
if let Some(parent) = path.parent()
&& !parent.as_os_str().is_empty()
{
create_private_directory(parent)?;
}
let mut builder = fs::DirBuilder::new();
builder.recursive(true);
builder.recursive(false);
#[cfg(unix)]
{
use std::os::unix::fs::DirBuilderExt;
builder.mode(0o755);
}
builder
.create(path)
.map_err(|error| format!("cannot create '{}': {error}", path.display()))?;
match builder.create(path) {
Ok(()) => {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(path, fs::Permissions::from_mode(0o755))
.map_err(|error| format!("cannot protect '{}': {error}", path.display()))?;
}
}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {}
Err(error) => return Err(format!("cannot create '{}': {error}", path.display())),
}
ensure_real_directory(path, "machine directory")
}
crates/dever-cli/src/toolchain/service/compile.rs
+7 -10
View File
@@ -84,6 +84,7 @@ pub(super) fn serve(layout: &Layout, stream: &mut UnixStream, bytes: u64) -> Res
.map_err(|error| format!("cannot monitor compilation caller: {error}"))?;
require_connected(stream, started)?;
if let Some(artifact) = store.restore(&identity)? {
drop(compiler);
return send_output(stream, artifact.as_bytes(), started);
}
let directory = store.compilation_directory()?;
@@ -104,6 +105,7 @@ pub(super) fn serve(layout: &Layout, stream: &mut UnixStream, bytes: u64) -> Res
let artifact = store
.restore(&identity)?
.ok_or("compiled output was not published")?;
drop(compiler);
send_output(stream, artifact.as_bytes(), started)
}
@@ -131,16 +133,11 @@ fn verify_pack(
));
}
}
// This is the embedded compiler's fixed Linux LLVM dependency, not a PATH lookup.
if cfg!(target_os = "linux")
&& !compiler
.manifest
.artifacts
.iter()
.any(|artifact| artifact.path == "lib/libLLVM.so.18.1")
{
return Err("signed release is missing lib/libLLVM.so.18.1".into());
}
let core_root = compiler
.core
.parent()
.ok_or("installed compiler has no release directory")?;
super::super::core_libraries::validate(core_root, &compiler.manifest.artifacts)?;
Ok(())
}
crates/dever-cli/src/worker.rs
+2 -2
View File
@@ -28,7 +28,7 @@ pub(super) fn execute() -> Result<(), String> {
.read_to_end(&mut bytes)
})
.map_err(|error| format!("cannot read compile request: {error}"))?;
let request = CompileRequest::decode(&bytes)?;
let mut request = CompileRequest::decode(&bytes)?;
drop(bytes);
if request.version != version {
return Err("compile request version does not match worker".into());
@@ -63,7 +63,7 @@ pub(super) fn execute() -> Result<(), String> {
crate::compile::worker_compile(
&program,
&sources,
&request,
&mut request,
&directory,
&directory.join(WORKER_OUTPUT_FILE),
)
crates/dever-cli/src/workers.rs
+9 -33
View File
@@ -127,7 +127,13 @@ pub fn prepare_for_target(
}
for contract in workers {
if contract.ecosystem == "exec" {
output.extend(prepare_exec(&contract, &supplied, target)?);
output.extend(command::prepare(
project_root,
&contract,
&package_files,
&supplied,
target,
)?);
continue;
}
let lock = lock.as_ref().expect("managed Worker has a lock");
@@ -173,36 +179,6 @@ pub fn prepare_for_target(
Ok(output)
}
fn prepare_exec(
contract: &ExternalWorkerContract,
supplied: &BTreeMap<&str, &EmbeddedResource>,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<EmbeddedResource>, String> {
let entry = supplied
.get(contract.entry.as_str())
.ok_or("exec Worker is missing from verified resources")?;
if !entry.executable || entry.bytes.is_empty() {
return Err("exec Worker entry must be a nonempty executable resource".into());
}
runtime::validate_binary(&contract.entry, &entry.bytes, target.platform())?;
let identity =
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?;
let tree = format!("workers/{}", digest(&identity));
let executable = format!("{tree}/worker");
let manifest = json!({
"format": "dever-worker-launch-v1", "ecosystem": "exec", "entry": contract.entry,
"executable": executable, "arguments": [], "working_directory": tree,
});
Ok(vec![
resource(executable, entry.bytes.clone(), true),
resource(
format!("{}.dever-worker.json", contract.entry),
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
false,
),
])
}
fn prepare_worker(
project_root: &Path,
contract: &ExternalWorkerContract,
@@ -301,7 +277,7 @@ fn prepare_worker(
.build
.as_ref()
.expect("validated Go build descriptor"),
sandbox: build_assets,
process_runtime: build_assets,
},
&mut tree_files,
target,
@@ -696,7 +672,7 @@ fn validate_worker_native(
files.extend(
supplied
.iter()
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
.filter(|(path, _)| path.starts_with("process-runtime/lib/"))
.map(|(path, resource)| ((*path).to_owned(), resource.bytes.as_slice())),
);
python_wheel::validate_native(
crates/dever-cli/src/workers/command.rs
+44 -29
View File
@@ -20,12 +20,26 @@ pub(super) fn prepare(
supplied: &BTreeMap<&str, &EmbeddedResource>,
target: BuildTarget,
) -> Result<Vec<EmbeddedResource>, String> {
let worker = contract.ecosystem == "exec";
let tree_kind = if worker { "workers" } else { "commands" };
let binary_name = if worker { "worker" } else { "tool" };
let prefix = format!(
"commands/{}",
"{tree_kind}/{}",
digest(&serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?)
);
let mut tree = Tree::new(prefix.clone());
let source = command_files(root, &contract.entry, package_files)?;
let entry = if worker {
let entry = supplied
.get(contract.entry.as_str())
.ok_or("exec Worker is missing from verified resources")?;
if !entry.executable || entry.bytes.is_empty() {
return Err("exec Worker entry must be a nonempty executable resource".into());
}
Some(entry.bytes.as_slice())
} else {
None
};
let source = command_files(root, &contract.entry, package_files, entry)?;
let binary = &source["program/tool"];
runtime::validate_binary(&contract.entry, binary, target.platform())?;
let machine = match target {
@@ -44,19 +58,17 @@ pub(super) fn prepare(
files.extend(
supplied
.iter()
.filter(|(path, _)| path.starts_with("sandbox/lib/"))
.filter(|(path, _)| path.starts_with("process-runtime/lib/"))
.map(|(path, entry)| ((*path).to_owned(), entry.bytes.as_slice())),
);
let closure = elf::Closure {
files: &files,
machine,
directories: vec!["program/lib".into(), "sandbox/lib".into()],
directories: vec!["program/lib".into(), "process-runtime/lib".into()],
};
for (path, bytes) in &source {
let object = elf::object(path, bytes, machine)?;
reject_loader_hooks(&object)?;
// The logical program root becomes /worker in the sandbox. A path
// escaping it could otherwise resolve a library from a file grant.
for fields in [&object.rpaths, &object.runpaths] {
for directory in elf::origin_directories(path, fields)? {
if directory != "program" && !directory.starts_with("program/") {
@@ -65,9 +77,9 @@ pub(super) fn prepare(
}
}
if let Some(name) = path.strip_prefix("program/lib/") {
if supplied.contains_key(format!("sandbox/lib/{name}").as_str()) {
if supplied.contains_key(format!("process-runtime/lib/{name}").as_str()) {
return Err(format!(
"command library '{name}' overrides a signed sandbox library"
"command library '{name}' overrides a signed process_runtime library"
));
}
if object.header.e_type != header::ET_DYN
@@ -83,38 +95,38 @@ pub(super) fn prepare(
}
for (path, bytes) in source {
let name = path.strip_prefix("program/").expect("owned command tree");
tree.add(name, bytes, name == "tool")?;
tree.add(
if name == "tool" { binary_name } else { name },
bytes,
name == "tool",
)?;
}
let (executable, arguments) = if let Some(interpreter) = interpreter {
if worker {
tree.add(
"contract.json",
serde_json::to_vec(&contract.sdk_manifest()).map_err(|error| error.to_string())?,
false,
)?;
}
if let Some(interpreter) = interpreter {
let name = interpreter
.rsplit('/')
.next()
.expect("validated GNU loader");
let loader = supplied
.get(format!("sandbox/lib/{name}").as_str())
.get(format!("process-runtime/lib/{name}").as_str())
.ok_or("command requires the signed target GNU loader")?;
runtime::validate_binary(name, &loader.bytes, target.platform())?;
tree.add("loader", loader.bytes.clone(), true)?;
(
format!("{prefix}/loader"),
vec![
json!({"literal": "--inhibit-cache"}),
json!({"literal": "--library-path"}),
json!({"literal": "lib:/lib"}),
json!({"resource": format!("{prefix}/tool")}),
],
)
} else {
(format!("{prefix}/tool"), Vec::new())
};
}
let suffix = if worker { "worker" } else { "command" };
let manifest = json!({
"format": "dever-command-launch-v1", "ecosystem": "command",
"entry": contract.entry, "executable": executable,
"arguments": arguments, "working_directory": prefix,
"format": format!("dever-{suffix}-launch-v1"), "ecosystem": contract.ecosystem,
"entry": contract.entry, "executable": format!("{prefix}/{binary_name}"),
"arguments": [], "working_directory": prefix,
});
let mut output = tree.finish();
output.push(resource(
format!("{}.dever-command.json", contract.entry),
format!("{}.dever-{suffix}.json", contract.entry),
serde_json::to_vec(&manifest).map_err(|error| error.to_string())?,
false,
));
@@ -143,6 +155,7 @@ fn command_files(
root: &Path,
entry: &str,
package_files: &BTreeMap<String, Vec<u8>>,
verified: Option<&[u8]>,
) -> Result<BTreeMap<String, Vec<u8>>, String> {
valid_path(entry)?;
let parent = entry
@@ -151,7 +164,9 @@ fn command_files(
.ok_or("command entry must belong to a domain")?;
let library_prefix = format!("{parent}/lib/");
let packaged = crate::packages::owns_worker(root, entry)?;
let binary = if packaged {
let binary = if let Some(bytes) = verified {
bytes.to_vec()
} else if packaged {
package_files
.get(entry)
.cloned()
crates/dever-cli/src/workers/elf.rs
+1 -1
View File
@@ -46,7 +46,7 @@ pub(crate) fn origin_directories(path: &str, fields: &[&str]) -> Result<Vec<Stri
}
}
let normalized = normalized.join("/");
if normalized.is_empty() || normalized.starts_with("sandbox/") {
if normalized.is_empty() || normalized.starts_with("process-runtime/") {
return Err("native library path escapes its Worker installation".into());
}
if !result.contains(&normalized) {
crates/dever-cli/src/workers/go_build.rs
+14 -27
View File
@@ -43,7 +43,6 @@ struct Builder {
stage: Stage,
input: PathBuf,
assets: PathBuf,
grants: Vec<dever_sandbox::Grant>,
tools: GoBuildManifest,
entry: String,
stdlib: BTreeMap<String, PathBuf>,
@@ -55,7 +54,7 @@ struct Builder {
pub(super) struct Inputs<'a> {
pub files: &'a [(String, Vec<u8>)],
pub manifest: &'a GoBuildManifest,
pub sandbox: &'a [EmbeddedResource],
pub process_runtime: &'a [EmbeddedResource],
}
pub(super) fn compile_worker(
@@ -69,7 +68,7 @@ pub(super) fn compile_worker(
let Inputs {
files: pack_files,
manifest: build,
sandbox: build_assets,
process_runtime: build_assets,
} = inputs;
let references = build.validate_files(pack_files, target)?;
if build.host != crate::toolchain::BuildTarget::host()? {
@@ -77,24 +76,19 @@ pub(super) fn compile_worker(
}
let stage = Stage::new()?;
let input = stage.0.join("input");
let assets = stage.0.join("sandbox");
let assets = stage.0.join("process-runtime");
for resource in build_assets
.iter()
.filter(|resource| resource.path.starts_with("sandbox/"))
.filter(|resource| resource.path.starts_with("process-runtime/"))
{
if digest(&resource.bytes) != resource.sha256 {
return Err("Go build sandbox asset has a digest mismatch".into());
return Err("Go build process_runtime asset has a digest mismatch".into());
}
stage.write(&resource.path, &resource.bytes, resource.executable)?;
}
let build_directory = stage.0.join("build");
fs::create_dir(&build_directory)
.map_err(|error| format!("cannot create Go build output: {error}"))?;
let grants = vec![dever_sandbox::Grant {
source: build_directory,
destination: "/data/build".into(),
writable: true,
}];
for (path, bytes) in pack_files {
stage.write(
&format!("input/{path}"),
@@ -164,7 +158,6 @@ pub(super) fn compile_worker(
stage,
input,
assets,
grants,
tools: build.clone(),
entry: entry_name(&contract.entry)?.into(),
stdlib,
@@ -235,9 +228,7 @@ impl GoBuildManifest {
}
for tool in [&self.compiler, &self.linker, &self.analyzer] {
if !tool.starts_with("bin/")
|| !pack
.get(tool.as_str())
.is_some_and(|bytes| !bytes.is_empty())
|| pack.get(tool.as_str()).is_none_or(|bytes| bytes.is_empty())
{
return Err(format!("locked Go build tool '{tool}' is missing"));
}
@@ -259,7 +250,7 @@ impl GoBuildManifest {
.ok_or("invalid locked Go standard library importcfg")?;
valid_import(name)?;
if path != format!("stdlib/{name}.a")
|| !pack.get(path).is_some_and(|bytes| !bytes.is_empty())
|| pack.get(path).is_none_or(|bytes| bytes.is_empty())
|| stdlib.insert(name.into(), path.into()).is_some()
{
return Err(
@@ -503,34 +494,30 @@ impl Builder {
fn run(&self, executable: &Path, args: &[&str]) -> Result<Vec<u8>, String> {
let arguments = args.iter().map(OsString::from).collect::<Vec<_>>();
self.stage.run(
self.sandbox(executable, &arguments).command()?,
self.process_runtime(executable, &arguments).command()?,
"locked Go build tool",
&self.stage.0.join("build"),
)
}
fn sandbox<'a>(
fn process_runtime<'a>(
&'a self,
executable: &'a Path,
arguments: &'a [OsString],
) -> dever_sandbox::Launch<'a> {
dever_sandbox::Launch {
) -> dever_process::Launch<'a> {
dever_process::Launch {
assets: &self.assets,
worker: &self.input,
executable,
arguments,
working_directory: &self.input,
capabilities: dever_sandbox::Capabilities::default(),
grants: &self.grants,
}
}
fn visible_path(&self, path: &Path) -> Result<String, String> {
self.sandbox(&self.input, &[])
.argument_path(path)?
.into_os_string()
.into_string()
.map_err(|_| "Go sandbox path is not UTF-8".into())
path.to_str()
.map(str::to_owned)
.ok_or_else(|| "Go build path is not UTF-8".into())
}
}
crates/dever-cli/src/workers/python_wheel.rs
+1 -1
View File
@@ -463,7 +463,7 @@ impl Wheel {
{
let newline = bytes.iter().position(|byte| *byte == b'\n').unwrap();
bytes = [
format!("#!/worker/{interpreter} -I\n").as_bytes(),
scripts::launcher(interpreter)?.as_bytes(),
&bytes[newline + 1..],
]
.concat();
crates/dever-cli/src/workers/python_wheel/native.rs
+4 -4
View File
@@ -5,7 +5,7 @@ use goblin::elf::header;
use std::collections::{BTreeMap, BTreeSet};
/// `files` contains Worker-relative installation files and separately verified
/// OS ABI files at sandbox/lib/*. No host directory is searched or executed.
/// OS ABI files at process-runtime/lib/*. No host directory is searched or executed.
pub fn validate_native(
target: &str,
suffixes: &[String],
@@ -36,7 +36,7 @@ pub fn validate_native(
let closure = Closure {
files,
machine,
directories: vec!["sandbox/lib".into()],
directories: vec!["process-runtime/lib".into()],
};
let interpreter_elf = object(
interpreter,
@@ -121,7 +121,7 @@ pub(crate) fn validate_build_tools(
machine,
directories: vec![
format!("rootfs/usr/lib/{multiarch}"),
"sandbox/lib".into(),
"process-runtime/lib".into(),
"rootfs/usr/lib".into(),
],
};
@@ -146,7 +146,7 @@ pub(crate) fn validate_build_tools(
}
let elf = object(path, bytes, machine)?;
if elf.interpreter.is_some_and(|value| value != interpreter) {
return Err("build tool interpreter differs from signed sandbox loader".into());
return Err("build tool interpreter differs from signed process_runtime loader".into());
}
closure.follow(path, vec![], &BTreeMap::new())?;
}
crates/dever-cli/src/workers/python_wheel/scripts.rs
+19 -1
View File
@@ -14,6 +14,23 @@ fn identifier(path: &str) -> bool {
})
}
pub(super) fn launcher(interpreter: &str) -> Result<String, String> {
let name = interpreter
.rsplit('/')
.next()
.ok_or("missing Python interpreter name")?;
if name.is_empty()
|| !name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-'))
{
return Err("invalid Python interpreter file name".into());
}
Ok(format!(
"#!/bin/sh\n'''exec' \"${{0%/*}}/{name}\" -I \"$0\" \"$@\"\n' '''\n"
))
}
pub(super) fn entry_points(
bytes: &[u8],
prefix: &str,
@@ -82,7 +99,8 @@ pub(super) fn entry_points(
}else{value.trim()};
let (module,function)=value.split_once(':').ok_or("wheel console script must name a callable")?;
if !identifier(module) || !identifier(function) { return Err("wheel script has an invalid Python entry point".into()); }
let source=format!("#!/worker/{interpreter} -I\nimport importlib\nimport re\nimport sys\nentry = importlib.import_module({module:?})\nfor part in {function:?}.split('.'):\n entry = getattr(entry, part)\nsys.argv[0] = re.sub(r'(-script\\.pyw|\\.exe)?$', '', sys.argv[0])\nsys.exit(entry())\n");
let launcher = launcher(interpreter)?;
let source=format!("{launcher}import importlib\nimport re\nimport sys\nentry = importlib.import_module({module:?})\nfor part in {function:?}.split('.'):\n entry = getattr(entry, part)\nsys.argv[0] = re.sub(r'(-script\\.pyw|\\.exe)?$', '', sys.argv[0])\nsys.exit(entry())\n");
Ok(InstalledFile {path:format!("{prefix}/bin/{name}"),bytes:source.into_bytes(),executable:true})
}).collect()
}