feat: ship slim releases with shared on-demand extensions and locked restore

This commit is contained in:
Antigravity committed 2026-10-05 00:19:31 +00:00
1 parent 83b78a0db6
commit 4fda7c8c1a
58 files changed
+3774 -425

No files matched your search

@@ -10,6 +10,14 @@ fn run() -> i32 {
if arguments.as_slice() == ["--dever-bootstrap-health"] {
return 0;
}
if let [flag, config] = arguments.as_slice()
&& flag == "--dever-extract"
{
return match dever_cli::toolchain::extract_config(std::path::Path::new(config)) {
Ok(()) => 0,
Err(error) => fail(error),
};
}
#[cfg(target_os = "linux")]
if let [flag, root] = arguments.as_slice()
&& flag == "--dever-install"
+9 -1
View File
@@ -142,7 +142,15 @@ pub(super) fn worker_compile(
},
),
};
let pack = RuntimePack::load(&compiler_directory()?, profile, request.target)?;
let (layout, version) =
managed_compiler()?.ok_or("compile worker requires a managed compiler")?;
let resources =
dever_cli::toolchain::SignedResources::load(&layout, &Version::parse(&version)?)?;
let root = resources.root(&format!(
"runtime/{}/manifest.json",
request.target.platform()
))?;
let pack = RuntimePack::load(&root, profile, request.target)?;
link_ir(&ir, &pack, build, output)
}
+121 -36
View File
@@ -15,6 +15,7 @@ use sha2::{Digest, Sha256};
pub mod build;
mod mutation;
mod registry;
pub mod restore;
pub(crate) use mutation::ProjectMutation;
pub use registry::npm;
pub(crate) use registry::python_dependencies;
@@ -22,7 +23,7 @@ pub use registry::sumdb;
pub use registry::{HttpRegistry, RegistryResolver, RegistryRuntime, RegistryTransport};
pub(crate) use registry::{InstalledRegistryPack, archive_files};
const LOCK_FORMAT: &str = "dever-lock-v5";
const LOCK_FORMAT: &str = "dever-lock-v6";
const TOOLCHAIN: &str = env!("CARGO_PKG_VERSION");
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
@@ -240,6 +241,15 @@ pub struct LockedArtifact {
pub path: String,
pub bytes: u64,
pub sha256: String,
pub source: Option<RegistrySource>,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RegistrySource {
pub ecosystem: Ecosystem,
pub locator: String,
pub filename: String,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
@@ -421,6 +431,7 @@ impl FixturePackage {
path: artifact.path.clone(),
bytes: artifact.bytes.len() as u64,
sha256: sha256(&artifact.bytes),
source: None,
})
.collect::<Vec<_>>();
artifacts.sort();
@@ -635,7 +646,7 @@ pub(crate) fn resolve_project(
lock.encode()?;
return Ok(lock);
}
let managed = InstalledRegistry::current(target)?;
let managed = PreparingRegistry(InstalledRegistry::current(target)?);
let mut runtimes = BTreeMap::new();
let mut ecosystems = all
.iter()
@@ -646,7 +657,10 @@ pub(crate) fn resolve_project(
ecosystems.insert(worker.ecosystem.parse()?);
}
for ecosystem in ecosystems {
runtimes.insert(ecosystem.clone(), managed.runtime(ecosystem)?.0);
runtimes.insert(
ecosystem.clone(),
build::Inputs::runtime(&managed, ecosystem)?.0,
);
}
for worker in workers.iter().filter(|worker| worker.ecosystem != "exec") {
let ecosystem: Ecosystem = worker.ecosystem.parse()?;
@@ -658,7 +672,7 @@ pub(crate) fn resolve_project(
}
}
let store = ManagedArtifactStore {
layout: managed.layout.clone(),
layout: managed.0.layout.clone(),
};
let transport = HttpRegistry::official();
let verified = sumdb::verify_chain(&prior_evidence, &sumdb::Verifier::official())?;
@@ -705,13 +719,40 @@ impl LibResolver for ProjectResolver<'_> {
}
}
#[derive(Clone)]
struct InstalledRegistry {
layout: crate::toolchain::Layout,
version_root: PathBuf,
version: crate::toolchain::Version,
resources: crate::toolchain::SignedResources,
target: crate::toolchain::BuildTarget,
}
/// Only explicit dependency preparation constructs this capability. Offline
/// readers retain InstalledRegistry and cannot request an installation.
struct PreparingRegistry(InstalledRegistry);
impl PreparingRegistry {
fn ensure(&self, kind: crate::toolchain::ExtensionKind) -> Result<(), String> {
crate::toolchain::ensure_extension(&self.0.layout, &self.0.version, kind, self.0.target)
}
}
impl build::Inputs for PreparingRegistry {
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> {
self.ensure(crate::toolchain::ExtensionKind::Runtime(ecosystem.clone()))?;
self.0.runtime(ecosystem)
}
fn tools(&self, ecosystem: Ecosystem) -> Result<(build::pack::Descriptor, Vec<u8>), String> {
self.ensure(crate::toolchain::ExtensionKind::Build(ecosystem.clone()))?;
build::Inputs::tools(&self.0, ecosystem)
}
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> {
build::Inputs::assets(&self.0)
}
fn diagnostic(&self, output: &[u8]) {
build::Inputs::diagnostic(&self.0, output);
}
}
impl build::Inputs for InstalledRegistry {
fn diagnostic(&self, output: &[u8]) {
if !output.is_empty() {
@@ -782,8 +823,9 @@ impl InstalledRegistry {
return Err("running Dever core is not the verified managed release".into());
}
Ok(Self {
resources: crate::toolchain::SignedResources::load(&layout, &version)?,
layout,
version_root: version_root.to_path_buf(),
version,
target,
})
}
@@ -815,40 +857,18 @@ impl InstalledRegistry {
}
fn signed_artifact(&self, relative: &Path) -> Result<Vec<u8>, String> {
let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice(
&fs::read(self.version_root.join("manifest.json"))
.map_err(|error| error.to_string())?,
)
.map_err(|error| format!("invalid signed release manifest: {error}"))?;
let name = relative.to_str().ok_or("runtime path is not UTF-8")?;
let identity = manifest
.artifacts
.iter()
.find(|artifact| artifact.path == name)
.ok_or_else(|| format!("signed release does not include runtime artifact '{name}'"))?;
let bytes = fs::read(self.version_root.join(relative))
.map_err(|error| format!("cannot read signed runtime artifact '{name}': {error}"))?;
if bytes.len() as u64 != identity.bytes || sha256(&bytes) != identity.sha256 {
return Err(format!(
"signed runtime artifact '{name}' failed identity verification"
));
}
Ok(bytes)
self.resources.read(name)
}
fn sandbox_resources(
&self,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<dever_core::native::EmbeddedResource>, String> {
let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice(
&fs::read(self.version_root.join("manifest.json"))
.map_err(|error| error.to_string())?,
)
.map_err(|error| format!("invalid signed release manifest: {error}"))?;
let prefix = format!("sandbox/{}/", target.platform());
let mut resources = Vec::new();
let mut files = Vec::new();
for artifact in &manifest.artifacts {
for artifact in self.resources.artifacts(&prefix)? {
let Some(relative) = artifact.path.strip_prefix(&prefix) else {
continue;
};
@@ -896,7 +916,7 @@ impl ArtifactStore for ManagedArtifactStore {
bytes: u64,
_target: &str,
) -> Result<Option<Vec<u8>>, String> {
crate::toolchain::artifact_get(&self.layout, sha256, bytes).map(Some)
crate::toolchain::artifact_get_optional(&self.layout, sha256, bytes)
}
}
@@ -1237,13 +1257,28 @@ pub(crate) fn bound_program_workers(
program: &dever_core::hir::Program,
target: crate::toolchain::BuildTarget,
) -> Result<Vec<LockedWorker>, String> {
let mut workers = program_workers(program)?;
bind_workers(program_workers(program)?, target, false)
}
fn bind_workers(
mut workers: Vec<LockedWorker>,
target: crate::toolchain::BuildTarget,
prepare: bool,
) -> Result<Vec<LockedWorker>, String> {
if workers.iter().all(|worker| worker.ecosystem == "exec") {
return Ok(workers);
}
let managed = InstalledRegistry::current(target)?;
for worker in &mut workers {
if worker.ecosystem != "exec" {
if prepare {
crate::toolchain::ensure_extension(
&managed.layout,
&managed.version,
crate::toolchain::ExtensionKind::Runtime(worker.ecosystem.parse()?),
target,
)?;
}
worker.runtime = Some(managed.runtime(worker.ecosystem.parse()?)?.0.pack);
}
}
@@ -1876,9 +1911,22 @@ pub fn execute_for_target(
let _mutation = ProjectMutation::for_command(command, project_root)?;
let path = project_root.join("dever.lock");
match command {
"install" => {
if !specs.is_empty() {
return Err(
"dever lib install accepts no Lib specs; it restores the existing dever.lock"
.into(),
);
}
restore::install(project_root, target).map(|report| report.summary())
}
"add" => {
let mut requested = read_declarations(project_root)?;
let workers = source_workers(project_root, target)?;
let workers = prepare_source_workers_with_packages(
project_root,
&crate::packages::sources(project_root)?,
target,
)?;
let package_libs = crate::packages::declared_libs(project_root)?;
if specs.is_empty()
&& requested.is_empty()
@@ -1920,7 +1968,11 @@ pub fn execute_for_target(
.map(|value| value.parse())
.collect::<Result<Vec<LibSpec>, _>>()?
};
let workers = source_workers(project_root, target)?;
let workers = prepare_source_workers_with_packages(
project_root,
&crate::packages::sources(project_root)?,
target,
)?;
let package_libs = crate::packages::declared_libs(project_root)?;
if requested.is_empty() && workers.is_empty() && package_libs.is_empty() {
return Err("dever lib update requires declared libs or exact lib specs".into());
@@ -2174,6 +2226,29 @@ pub(crate) fn source_workers_with_packages(
project_root: &Path,
packages: &[dever_core::source::PackageSource],
target: crate::toolchain::BuildTarget,
) -> Result<Vec<LockedWorker>, String> {
bind_workers(
unbound_source_workers(project_root, packages)?,
target,
false,
)
}
pub(crate) fn prepare_source_workers_with_packages(
project_root: &Path,
packages: &[dever_core::source::PackageSource],
target: crate::toolchain::BuildTarget,
) -> Result<Vec<LockedWorker>, String> {
bind_workers(
unbound_source_workers(project_root, packages)?,
target,
true,
)
}
fn unbound_source_workers(
project_root: &Path,
packages: &[dever_core::source::PackageSource],
) -> Result<Vec<LockedWorker>, String> {
let source_root = project_root.join("module");
if !source_root.exists() && packages.is_empty() {
@@ -2193,7 +2268,7 @@ pub(crate) fn source_workers_with_packages(
.map(|error| error.render(&sources))
.collect::<String>()
})?;
bound_program_workers(&program, target)
program_workers(&program)
}
fn declaration_path(project_root: &Path) -> PathBuf {
@@ -2481,6 +2556,16 @@ fn validate_runtime(runtime: &RuntimePack) -> Result<(), String> {
}
fn validate_artifact(artifact: &LockedArtifact) -> Result<(), String> {
if let Some(source) = &artifact.source {
crate::workers::valid_path(&source.filename)?;
if source.filename.contains('/')
|| source.locator.is_empty()
|| source.locator.len() > 8192
|| source.locator.chars().any(char::is_control)
{
return Err("invalid locked registry source locator or filename".into());
}
}
if artifact.target.is_empty() || artifact.bytes == 0 {
return Err("artifact target/size is invalid".into());
}
+65 -10
View File
@@ -63,7 +63,7 @@ pub struct Receipt {
pub frontend: String,
pub python: Option<PythonSystem>,
pub dynamic_requires: Vec<String>,
pub dependencies: Box<LockFile>,
pub inputs: Box<LockFile>,
pub config_settings: BTreeMap<String, serde_json::Value>,
pub output: LockedArtifact,
}
@@ -90,6 +90,29 @@ pub struct Session<'a> {
}
impl<'a> Session<'a> {
pub(crate) fn replay(
&self,
receipt: &Receipt,
store: &dyn super::ArtifactStore,
) -> Result<(), String> {
if receipt.frontend != frontend(&receipt.ecosystem)? {
return Err("locked build frontend differs from this toolchain".into());
}
let bytes = match receipt.ecosystem {
Ecosystem::Pip => self.replay_python(receipt, store)?,
Ecosystem::Npm => self.replay_npm(receipt, store)?,
Ecosystem::Go => return Err("Go source build receipts are unsupported".into()),
};
if bytes.len() as u64 != receipt.output.bytes
|| super::sha256(&bytes) != receipt.output.sha256
{
return Err("fixed build replay is not byte reproducible: output differs from dever.lock; lock was not changed".into());
}
if store.publish(&bytes, &receipt.output.target)? != receipt.output.sha256 {
return Err("restored build cache identity mismatch".into());
}
Ok(())
}
pub fn new(inputs: &'a dyn Inputs) -> Self {
Self {
inputs,
@@ -229,6 +252,9 @@ pub fn doctor(lock: &LockFile) -> Result<(), String> {
}
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
if lock.format != super::LOCK_FORMAT || lock.toolchain != super::TOOLCHAIN {
return Err("build input lock format or toolchain differs from this release".into());
}
if depth > 16 {
return Err("build dependency depth exceeds limit".into());
}
@@ -361,7 +387,6 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
&& (receipt.python.is_some()
|| receipt.python_markers.is_some()
|| !receipt.dynamic_requires.is_empty()
|| !receipt.dependencies.libs.is_empty()
|| receipt.graph.is_none())
{
return Err("npm build receipt contains Python backend state".into());
@@ -372,20 +397,50 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
if let Some(system) = &receipt.python {
validate_requirements(receipt, system)?;
}
if !receipt.dependencies.workers.is_empty()
|| !receipt.dependencies.packages.is_empty()
if !receipt.inputs.workers.is_empty()
|| !receipt.inputs.packages.is_empty()
|| receipt
.dependencies
.inputs
.libs
.iter()
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
.any(|lib| lib.spec.ecosystem != receipt.ecosystem)
{
return Err("build dependency closure is not a separate Python environment".into());
return Err("build input closure contains unrelated environments".into());
}
doctor_at(&receipt.dependencies, depth + 1)?;
if receipt.output.source.is_some()
|| receipt.sources.iter().any(|source| {
source
.source
.as_ref()
.is_none_or(|origin| origin.ecosystem != receipt.ecosystem)
})
{
return Err("build receipt lacks exact registry source identity".into());
}
if receipt.ecosystem == Ecosystem::Npm {
let mut sources = receipt
.inputs
.libs
.iter()
.flat_map(|lib| lib.artifacts.clone())
.collect::<Vec<_>>();
sources.sort();
sources.dedup();
if sources != receipt.sources
|| receipt.inputs.npm.len() != 1
|| receipt.inputs.npm[0].roots != receipt.roots
|| receipt.inputs.npm[0].build.is_some()
|| !receipt.inputs.builds.is_empty()
{
return Err("npm build input graph differs from its original sources".into());
}
} else if receipt.sources.len() != 1 || !receipt.inputs.npm.is_empty() {
return Err("Python build receipt has invalid source inputs".into());
}
doctor_at(&receipt.inputs, depth + 1)?;
// Full graph checks belong to the ordinary lock owner; recursion is
// bounded above before invoking that owner on a dependency lock.
let mut dependencies = (*receipt.dependencies).clone();
let mut dependencies = (*receipt.inputs).clone();
dependencies.builds.clear();
for lib in &mut dependencies.libs {
lib.build = None;
@@ -447,7 +502,7 @@ fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(),
let document = serde_json::json!({"info":{"requires_dist":requirements}});
let expected = super::python_dependencies(&document, markers, &[])?;
let libraries = receipt
.dependencies
.inputs
.libs
.iter()
.map(|lib| (lib.spec.name.as_str(), lib))
+67 -22
View File
@@ -12,6 +12,43 @@ mod environment;
use environment::{Environment, Output};
impl Session<'_> {
pub(super) fn replay_npm(
&self,
receipt: &Receipt,
store: &dyn crate::libs::ArtifactStore,
) -> Result<Vec<u8>, String> {
let (runtime, _) = self.inputs.runtime(Ecosystem::Npm)?;
if runtime.pack != receipt.runtime {
return Err("locked npm build runtime changed".into());
}
let environment = Environment::new(self.inputs, &runtime)?;
if environment.descriptor != receipt.tools
|| Some(&environment.python_runtime) != receipt.auxiliary_runtime.as_ref()
{
return Err("locked npm build tools or auxiliary runtime changed".into());
}
let mut graph = receipt
.inputs
.npm
.first()
.ok_or("npm build input graph missing")?
.clone();
let (files, executables) = installation_inputs(store, &receipt.inputs, &graph)?;
let output = environment.run(self.inputs, &graph, &files, &executables)?;
prune_failed(&mut graph, &output.failed)?;
npm::validate_built_files(&graph, &output.files)?;
if Some(&graph) != receipt.graph.as_ref()
|| output.native != receipt.native_entries
|| output.rejections != receipt.native_rejections
{
return Err(
"npm fixed build replay changed graph, optional outcomes or native admission"
.into(),
);
}
archive(&output)
}
pub(crate) fn npm(
&self,
resolver: &RegistryResolver<'_>,
@@ -21,26 +58,7 @@ impl Session<'_> {
) -> Result<(), String> {
let mut lock = LockFile::new(libraries.clone())?;
lock.npm.push(graph.clone());
let archives = libraries
.iter()
.map(|lib| {
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
Ok((
lib.spec.clone(),
resolver.store.verify_exact(
&artifact.sha256,
artifact.bytes,
&artifact.target,
)?,
))
})
.collect::<Result<BTreeMap<_, _>, String>>()?;
let files = npm::build_files(graph, &lock, |lib| {
archives
.get(&lib.spec)
.map(Vec::as_slice)
.ok_or("npm source archive missing".into())
})?;
let (files, executables) = installation_inputs(resolver.store, &lock, graph)?;
if !npm::requires_build(&files)? {
return Ok(());
}
@@ -66,7 +84,6 @@ impl Session<'_> {
retain_archives(libraries, graph);
return Ok(());
}
let executables = npm::build_executables(graph, &archives)?;
let environment = Environment::new(self.inputs, runtime)?;
let output = environment.run(self.inputs, graph, &files, &executables)?;
prune_failed(graph, &output.failed)?;
@@ -82,6 +99,7 @@ impl Session<'_> {
path: format!("build/npm/{digest}/installation.tgz"),
bytes: bytes.len() as u64,
sha256: digest,
source: None,
};
let receipt = Receipt {
ecosystem: Ecosystem::Npm,
@@ -97,7 +115,7 @@ impl Session<'_> {
frontend: super::frontend(&Ecosystem::Npm)?,
python: None,
dynamic_requires: vec![],
dependencies: Box::new(LockFile::new(vec![])?),
inputs: Box::new(lock),
config_settings: BTreeMap::new(),
output: output_artifact,
};
@@ -114,6 +132,33 @@ impl Session<'_> {
}
}
type InstallationInputs = (Vec<(String, Vec<u8>)>, BTreeSet<String>);
fn installation_inputs(
store: &dyn crate::libs::ArtifactStore,
lock: &LockFile,
graph: &npm::Environment,
) -> Result<InstallationInputs, String> {
let archives = lock
.libs
.iter()
.map(|lib| {
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
Ok((
lib.spec.clone(),
store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?,
))
})
.collect::<Result<BTreeMap<_, _>, String>>()?;
let files = npm::build_files(graph, lock, |lib| {
archives
.get(&lib.spec)
.map(Vec::as_slice)
.ok_or("npm source archive missing".into())
})?;
Ok((files, npm::build_executables(graph, &archives)?))
}
fn archive(output: &Output) -> Result<Vec<u8>, String> {
let encoder = flate2::GzBuilder::new()
.mtime(0)
+143 -54
View File
@@ -8,8 +8,8 @@ use std::path::{Path, PathBuf};
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
use crate::libs::{
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
sha256,
ArtifactStore, Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime,
RegistrySource, archive_files, sha256,
};
use crate::workers::{python_wheel::Wheel, runtime};
use serde_json::{Value, json};
@@ -23,6 +23,12 @@ struct Environment {
descriptor: pack::Descriptor,
}
struct Execution<'a> {
store: &'a dyn ArtifactStore,
runtime: &'a RegistryRuntime,
inputs: &'a dyn super::Inputs,
}
struct Active<'a> {
session: &'a Session<'a>,
key: String,
@@ -34,12 +40,69 @@ impl Drop for Active<'_> {
}
impl Session<'_> {
pub(super) fn replay_python(
&self,
receipt: &Receipt,
store: &dyn ArtifactStore,
) -> Result<Vec<u8>, String> {
let (runtime, _) = self.inputs.runtime(Ecosystem::Pip)?;
if runtime.pack != receipt.runtime || runtime.python_markers != receipt.python_markers {
return Err("locked Python build runtime or marker policy changed".into());
}
let environment = Environment::new(self, &runtime)?;
let execution = Execution {
store,
runtime: &runtime,
inputs: self.inputs,
};
if environment.descriptor != receipt.tools {
return Err("locked Python build tools changed".into());
}
let artifact = receipt
.sources
.first()
.ok_or("Python build source missing")?;
let origin = artifact
.source
.as_ref()
.ok_or("Python build source locator missing")?;
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
let source = source_files(&origin.filename, &bytes)?;
let system = receipt
.python
.as_ref()
.ok_or("Python build backend missing")?;
let empty = LockFile::new(vec![])?;
let inspected = environment.run(
&execution,
&empty,
&source,
json!({"phase":"inspect"}),
None,
)?;
let actual: PythonSystem =
serde_json::from_value(inspected.response).map_err(|error| error.to_string())?;
if &actual != system {
return Err("locked Python build backend configuration changed".into());
}
// The locked closure is final. Replaying never asks the resolver to
// discover or select static/dynamic build requirements again.
let (_, bytes) = environment.wheel(
&execution,
&receipt.inputs,
&source,
system,
receipt.roots.first().ok_or("Python build root missing")?,
)?;
Ok(bytes)
}
pub(crate) fn python(
&self,
resolver: &RegistryResolver<'_>,
spec: &LibSpec,
runtime: &RegistryRuntime,
filename: &str,
source_origin: RegistrySource,
source: &[u8],
) -> Result<Product, String> {
let key = format!(
@@ -62,11 +125,21 @@ impl Session<'_> {
session: self,
key: key.clone(),
};
let sources = source_files(filename, source)?;
let sources = source_files(&source_origin.filename, source)?;
let environment = Environment::new(self, runtime)?;
let execution = Execution {
store: resolver.store,
runtime,
inputs: self.inputs,
};
let empty = LockFile::new(vec![])?;
let inspected =
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
let inspected = environment.run(
&execution,
&empty,
&sources,
json!({"phase":"inspect"}),
None,
)?;
let system: PythonSystem = serde_json::from_value(inspected.response)
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
let mut requirements = system.requires.clone();
@@ -75,7 +148,7 @@ impl Session<'_> {
let mut discovered = BTreeSet::new();
for _ in 0..8 {
let result = environment.run(
resolver,
&execution,
&dependencies,
&sources,
json!({"phase":"requires","system":system}),
@@ -99,38 +172,8 @@ impl Session<'_> {
dependencies = next;
}
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
let metadata = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"metadata","system":system}),
None,
)?;
let result = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
Some(&metadata.outputs),
)?;
let filename = result
.response
.get("filename")
.and_then(Value::as_str)
.ok_or("build backend omitted wheel filename")?;
crate::workers::valid_path(filename)?;
if filename.contains('/') || !filename.ends_with(".whl") {
return Err("build backend returned invalid wheel filename".into());
}
let bytes = result
.outputs
.get(&format!("wheels/{filename}"))
.ok_or("build backend did not produce its declared wheel")?
.clone();
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
compare_metadata(prefix, &metadata.outputs, &bytes)?;
}
let (filename, bytes) =
environment.wheel(&execution, &dependencies, &sources, &system, spec)?;
let output = artifact(
runtime,
format!(
@@ -140,11 +183,12 @@ impl Session<'_> {
),
&bytes,
);
let source_artifact = artifact(
let mut source_artifact = artifact(
runtime,
format!("build/source/{}/source", sha256(source)),
source,
);
source_artifact.source = Some(source_origin);
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
return Err("build artifact cache identity mismatch".into());
@@ -165,13 +209,13 @@ impl Session<'_> {
frontend: super::frontend(&Ecosystem::Pip)?,
python: Some(system),
dynamic_requires: dynamic,
dependencies: Box::new(dependencies),
inputs: Box::new(dependencies),
config_settings: BTreeMap::new(),
output,
};
let product = Product {
bytes,
filename: filename.into(),
filename,
receipt,
};
self.remember(key, product.clone())?;
@@ -185,6 +229,7 @@ fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArti
path,
bytes: bytes.len() as u64,
sha256: sha256(bytes),
source: None,
}
}
@@ -236,6 +281,54 @@ struct ResultFiles {
}
impl Environment {
fn wheel(
&self,
execution: &Execution<'_>,
dependencies: &LockFile,
source: &Source,
system: &PythonSystem,
spec: &LibSpec,
) -> Result<(String, Vec<u8>), String> {
let metadata = self.run(
execution,
dependencies,
source,
json!({"phase":"metadata","system":system}),
None,
)?;
let result = self.run(
execution,
dependencies,
source,
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
Some(&metadata.outputs),
)?;
let filename = result
.response
.get("filename")
.and_then(Value::as_str)
.ok_or("build backend omitted wheel filename")?;
crate::workers::valid_path(filename)?;
if filename.contains('/') || !filename.ends_with(".whl") {
return Err("build backend returned invalid wheel filename".into());
}
let bytes = result
.outputs
.get(&format!("wheels/{filename}"))
.ok_or("build backend did not produce its declared wheel")?
.clone();
Wheel::parse(
spec,
Some(filename),
&execution.runtime.python_wheel_tags,
&bytes,
)?;
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
compare_metadata(prefix, &metadata.outputs, &bytes)?;
}
Ok((filename.into(), bytes))
}
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
super::require_execution_target(&requested.target, "Python source build")?;
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
@@ -291,12 +384,17 @@ impl Environment {
fn run(
&self,
resolver: &RegistryResolver<'_>,
execution: &Execution<'_>,
dependencies: &LockFile,
source: &Source,
mut request: Value,
prepared: Option<&BTreeMap<String, Vec<u8>>>,
) -> Result<ResultFiles, String> {
let Execution {
store,
runtime,
inputs,
} = execution;
let stage = Stage::new()?;
let input = stage.0.join("input");
for (path, (bytes, executable)) in &self.files {
@@ -322,10 +420,6 @@ impl Environment {
}
}
let mut installed = BTreeMap::<String, Vec<u8>>::new();
let runtime = resolver
.runtimes
.get(&Ecosystem::Pip)
.ok_or("build dependency runtime is missing")?;
let mut distributions = BTreeSet::new();
for lib in &dependencies.libs {
super::validate_python_policy(
@@ -342,10 +436,7 @@ impl Environment {
);
}
let artifact = &lib.artifacts[0];
let bytes =
resolver
.store
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
wheel.validate_dependencies(
lib,
@@ -432,9 +523,7 @@ impl Environment {
"isolated Python build hook",
&work,
)?;
if let Some(session) = resolver.build {
session.inputs.diagnostic(&stage.diagnostics()?);
}
inputs.diagnostic(&stage.diagnostics()?);
let outputs = output_files(&work.join("output"))?;
let response = serde_json::from_slice(
outputs
+1 -1
View File
@@ -10,7 +10,7 @@ pub(crate) struct ProjectMutation {
impl ProjectMutation {
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
if !matches!(command, "add" | "update" | "remove") {
if !matches!(command, "add" | "update" | "remove" | "install") {
return Ok(None);
}
let directory = fs::symlink_metadata(root)
+44 -6
View File
@@ -19,7 +19,7 @@ pub mod sumdb;
use super::{
ArtifactStore, Ecosystem, LibResolver, LibSpec, LockFile, LockedArtifact, LockedDependency,
LockedLib, ProviderManifest, RuntimePack, sha256, validate_artifact,
LockedLib, ProviderManifest, RegistrySource, RuntimePack, sha256, validate_artifact,
};
const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024;
@@ -553,7 +553,7 @@ impl RegistryResolver<'_> {
runtime,
"whl",
&candidate.bytes,
"pypi-wheel-v1",
candidate.source.clone(),
)?;
lib.build = candidate.build.clone();
distributions.insert(base.to_owned(), lib.clone());
@@ -925,6 +925,11 @@ impl RegistryResolver<'_> {
}
Ok(Some(PythonCandidate {
build: None,
source: Some(RegistrySource {
ecosystem: Ecosystem::Pip,
locator: url.into(),
filename: field(file, "filename")?.into(),
}),
bytes,
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
}))
@@ -965,7 +970,17 @@ impl RegistryResolver<'_> {
if sha256(&bytes) != expected {
return Err("PyPI source SHA-256 mismatch".into());
}
let product = session.python(self, spec, runtime, field(file, "filename")?, &bytes)?;
let product = session.python(
self,
spec,
runtime,
RegistrySource {
ecosystem: Ecosystem::Pip,
locator: field(file, "url")?.into(),
filename: field(file, "filename")?.into(),
},
&bytes,
)?;
let wheel = crate::workers::python_wheel::Wheel::parse(
spec,
Some(&product.filename),
@@ -980,6 +995,7 @@ impl RegistryResolver<'_> {
}
Ok(Some(PythonCandidate {
bytes: product.bytes,
source: None,
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
build: Some(product.receipt.identity()?),
}))
@@ -1012,7 +1028,7 @@ impl RegistryResolver<'_> {
runtime: &RegistryRuntime,
extension: &str,
bytes: &[u8],
schema: &str,
source: Option<RegistrySource>,
) -> Result<LockedLib, String> {
let files = archive_files(extension, bytes)?;
if spec.ecosystem == Ecosystem::Go {
@@ -1035,6 +1051,7 @@ impl RegistryResolver<'_> {
path,
bytes: bytes.len() as u64,
sha256: sha256(bytes),
source,
};
validate_artifact(&artifact)?;
let published = self.store.publish(bytes, &runtime.target)?;
@@ -1052,7 +1069,12 @@ impl RegistryResolver<'_> {
dependencies,
runtime: runtime.pack.clone(),
artifacts: vec![artifact],
schema: schema.into(),
schema: match spec.ecosystem {
Ecosystem::Pip => "pypi-wheel-v1",
Ecosystem::Npm => "npm-instances-v1",
Ecosystem::Go => "go-sumdb-v1",
}
.into(),
build: None,
})
}
@@ -1126,7 +1148,22 @@ impl RegistryResolver<'_> {
MAX_ARTIFACT_BYTES,
)?;
sumdb.verify_zip(name, version, &zip)?;
self.lock_archive(&spec, dependencies, runtime, "zip", &zip, "go-sumdb-v1")
self.lock_archive(
&spec,
dependencies,
runtime,
"zip",
&zip,
Some(RegistrySource {
ecosystem: Ecosystem::Go,
locator: format!(
"/{}/@v/v{}.zip",
go_path(name),
version.trim_start_matches('v')
),
filename: format!("v{}.zip", version.trim_start_matches('v')),
}),
)
})
.collect::<Result<Vec<_>, String>>()?;
return Ok((libs, sumdb.finish()));
@@ -1179,6 +1216,7 @@ struct PythonCandidate {
bytes: Vec<u8>,
info: Value,
build: Option<String>,
source: Option<RegistrySource>,
}
struct SearchState<'a> {
@@ -128,6 +128,18 @@ impl Catalog<'_, '_> {
"npm archive cache exceeds its 256 MiB/package budget".into(),
));
}
let locator = document
.get("dist")
.and_then(|dist| dist.get("tarball"))
.and_then(serde_json::Value::as_str)
.ok_or_else(|| SearchError::Fatal("npm tarball locator missing".into()))?;
let filename = locator
.split(['?', '#'])
.next()
.unwrap_or(locator)
.rsplit('/')
.next()
.ok_or_else(|| SearchError::Fatal("npm tarball filename missing".into()))?;
let lib = self
.resolver
.lock_archive(
@@ -136,7 +148,11 @@ impl Catalog<'_, '_> {
self.runtime,
"tgz",
&bytes,
"npm-instances-v1",
Some(crate::libs::RegistrySource {
ecosystem: Ecosystem::Npm,
locator: locator.into(),
filename: filename.into(),
}),
)
.map_err(SearchError::Fatal)?;
self.archives.insert(spec.clone(), (lib, archive));
+278
View File
@@ -0,0 +1,278 @@
//! Exact restoration is a preparation operation, never a dependency resolver.
use std::collections::BTreeSet;
use std::fs;
use std::path::Path;
use super::{
ArtifactStore, FixtureRegistry, LockFile, LockedArtifact, RegistryTransport, build, sumdb,
};
pub(super) fn install(
project_root: &Path,
target: crate::toolchain::BuildTarget,
) -> Result<super::PreparationReport, String> {
super::reject_declaration_symlinks(project_root)?;
let lock_bytes = fs::read(project_root.join("dever.lock"))
.map_err(|error| format!("dever lib install requires an existing dever.lock: {error}"))?;
let setting_bytes = setting_snapshot(project_root)?;
let lock = LockFile::decode(&lock_bytes)?;
super::doctor(&lock)?;
validate_target(&lock, target.platform())?;
let fixtures = FixtureRegistry::builtin();
if lock.packages.is_empty()
&& lock
.libs
.iter()
.all(|lib| fixtures.package(&lib.spec).is_some())
&& lock.workers.iter().all(|worker| worker.ecosystem == "exec")
{
let store = super::FixtureArtifactStore::default();
validate_project(project_root, &lock, &store, target, false)?;
restore_fixtures(&lock, &store)?;
} else {
let inputs = super::PreparingRegistry(super::InstalledRegistry::current(target)?);
let store = super::ManagedArtifactStore {
layout: inputs.0.layout.clone(),
};
crate::packages::restore_locked(project_root, &lock, &store)?;
validate_project(project_root, &lock, &store, target, true)?;
for worker in lock
.workers
.iter()
.filter(|worker| worker.ecosystem != "exec")
{
let (runtime, _) = build::Inputs::runtime(&inputs, worker.ecosystem.parse()?)?;
if worker.runtime.as_ref() != Some(&runtime.pack) {
return Err("locked Worker runtime differs from signed release".into());
}
}
if target != crate::toolchain::BuildTarget::host()? && !lock.workers.is_empty() {
inputs.ensure(crate::toolchain::ExtensionKind::Target)?;
}
restore_artifacts(
&lock,
&super::HttpRegistry::official(),
&store,
&inputs,
target.platform(),
&sumdb::Verifier::official(),
)?;
super::embedded_resources_for_target(project_root, target)?;
}
if fs::read(project_root.join("dever.lock")).map_err(|error| error.to_string())? != lock_bytes
|| setting_snapshot(project_root)? != setting_bytes
{
return Err("project setting or lock changed during exact restoration".into());
}
let artifacts = lock
.libs
.iter()
.flat_map(|lib| &lib.artifacts)
.chain(build::npm_outputs(&lock).map(|receipt| &receipt.output))
.collect::<BTreeSet<_>>();
Ok(super::PreparationReport {
libs: lock.libs.len(),
artifacts: artifacts.len(),
runtimes: lock
.libs
.iter()
.map(|lib| &lib.runtime)
.chain(
lock.workers
.iter()
.filter_map(|worker| worker.runtime.as_ref()),
)
.collect::<BTreeSet<_>>()
.len(),
artifact_bytes: artifacts.iter().map(|artifact| artifact.bytes).sum(),
})
}
fn setting_snapshot(project_root: &Path) -> Result<Option<Vec<u8>>, String> {
match fs::read(project_root.join("config/setting.json")) {
Ok(bytes) => Ok(Some(bytes)),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(error) => Err(format!("cannot read project setting snapshot: {error}")),
}
}
fn validate_project(
project_root: &Path,
lock: &LockFile,
store: &dyn ArtifactStore,
target: crate::toolchain::BuildTarget,
prepare: bool,
) -> Result<(), String> {
let sources = crate::packages::sources_with(project_root, store)?;
let workers = super::unbound_source_workers(project_root, &sources)?;
let mut unbound = lock.clone();
for worker in &mut unbound.workers {
worker.runtime = None;
}
super::validate_worker_bindings(&unbound, &workers)?;
let mut roots = super::read_declarations(project_root)?;
roots.extend(crate::packages::declared_libs_with(project_root, store)?);
if super::retain_locked_libs(lock, &roots, lock.workers.clone())? != *lock {
return Err("dever.lock contains Lib entries or environments outside the declared closure; run 'dever lib update <project-root>'".into());
}
let workers = super::bind_workers(workers, target, prepare)?;
super::validate_worker_bindings(lock, &workers)?;
Ok(())
}
fn validate_target(lock: &LockFile, target: &str) -> Result<(), String> {
for artifact in lock.libs.iter().flat_map(|lib| &lib.artifacts).chain(
lock.builds
.iter()
.flat_map(|receipt| receipt.sources.iter().chain([&receipt.output])),
) {
if artifact.target != target && artifact.target != "host" {
return Err(format!(
"locked artifact targets {}, not {target}; install never switches lock targets",
artifact.target
));
}
}
for receipt in &lock.builds {
validate_target(&receipt.inputs, target)?;
}
Ok(())
}
fn restore_fixtures(lock: &LockFile, store: &dyn ArtifactStore) -> Result<(), String> {
let fixtures = FixtureRegistry::builtin();
for lib in &lock.libs {
if let Some(package) = fixtures.package(&lib.spec) {
if package.locked() != *lib {
return Err("locked fixture metadata differs from builtin package".into());
}
for artifact in &package.artifacts {
let expected = super::sha256(&artifact.bytes);
if store.publish(&artifact.bytes, &artifact.target)? != expected {
return Err("fixture artifact cache identity mismatch".into());
}
}
}
}
Ok(())
}
/// Shared by production preparation and isolated registry fixtures. Every
/// network request names the recorded archive; no metadata/version query runs.
pub fn restore_artifacts(
lock: &LockFile,
transport: &dyn RegistryTransport,
store: &dyn ArtifactStore,
inputs: &dyn build::Inputs,
target: &str,
verifier: &sumdb::Verifier,
) -> Result<(), String> {
super::doctor_with_go_verifier(lock, verifier)?;
validate_target(lock, target)?;
let session = build::Session::new(inputs);
restore_at(lock, transport, store, inputs, &session)?;
super::verify_locked_artifacts_with_go_verifier(lock, store, target, verifier)?;
Ok(())
}
fn restore_at(
lock: &LockFile,
transport: &dyn RegistryTransport,
store: &dyn ArtifactStore,
inputs: &dyn build::Inputs,
session: &build::Session<'_>,
) -> Result<(), String> {
restore_fixtures(lock, store)?;
let fixtures = FixtureRegistry::builtin();
for lib in &lock.libs {
if fixtures.package(&lib.spec).is_some() {
continue;
}
let (runtime, _) = inputs.runtime(lib.spec.ecosystem.clone())?;
if runtime.pack != lib.runtime
|| lib
.artifacts
.iter()
.any(|artifact| artifact.target != runtime.target)
{
return Err(format!(
"{} locked runtime differs from signed release",
lib.spec.key()
));
}
if let Some(markers) = &runtime.python_markers {
build::validate_python_policy(lock, lib, markers)?;
}
for artifact in &lib.artifacts {
if artifact
.source
.as_ref()
.is_some_and(|source| source.ecosystem != lib.spec.ecosystem)
{
return Err("locked registry source ecosystem differs from its Lib".into());
}
if artifact.source.is_some() {
restore_archive(artifact, transport, store)?;
} else if lib.build.is_none() {
return Err(format!("{} lacks an exact registry source", lib.spec.key()));
}
}
}
for receipt in &lock.builds {
if cached(&receipt.output, store)? {
continue;
}
restore_at(&receipt.inputs, transport, store, inputs, session)?;
for source in &receipt.sources {
restore_archive(source, transport, store)?;
}
session.replay(receipt, store)?;
}
Ok(())
}
fn cached(artifact: &LockedArtifact, store: &dyn ArtifactStore) -> Result<bool, String> {
match store.get_exact(&artifact.sha256, artifact.bytes, &artifact.target)? {
Some(bytes) => {
validate_bytes(artifact, &bytes)?;
Ok(true)
}
None => Ok(false),
}
}
fn validate_bytes(artifact: &LockedArtifact, bytes: &[u8]) -> Result<(), String> {
if bytes.len() as u64 != artifact.bytes || super::sha256(bytes) != artifact.sha256 {
return Err(format!(
"locked archive '{}' failed exact byte/hash verification",
artifact.path
));
}
Ok(())
}
fn restore_archive(
artifact: &LockedArtifact,
transport: &dyn RegistryTransport,
store: &dyn ArtifactStore,
) -> Result<(), String> {
if cached(artifact, store)? {
return Ok(());
}
let source = artifact
.source
.as_ref()
.ok_or("locked archive source is absent")?;
let limit =
usize::try_from(artifact.bytes).map_err(|_| "locked archive exceeds address space")?;
if limit > 64 * 1024 * 1024 {
return Err("locked registry archive exceeds byte budget".into());
}
let bytes = transport.get(source.ecosystem.clone(), &source.locator, limit)?;
validate_bytes(artifact, &bytes)?;
if store.publish(&bytes, &artifact.target)? != artifact.sha256 {
return Err("restored archive cache identity mismatch".into());
}
Ok(())
}
+2 -2
View File
@@ -18,7 +18,7 @@ mod worker;
#[path = "../../../test/dever-cli-tests/build_arguments.rs"]
mod build_arguments;
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|install|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
enum Action {
Check,
@@ -435,7 +435,7 @@ fn arguments_for(arguments: &[OsString]) -> Result<(PathBuf, Action), String> {
if command == "lib"
&& matches!(
operation.to_str(),
Some("add" | "list" | "update" | "remove" | "doctor")
Some("add" | "install" | "list" | "update" | "remove" | "doctor")
) =>
{
let command = operation.to_str().unwrap().to_owned();
+65 -2
View File
@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use url::Url;
use crate::libs::{self, ArtifactStore, LibSpec};
use crate::libs::{self, ArtifactStore, LibSpec, LockFile};
const INDEX_FORMAT: &str = "dever-package-index-v1";
const MANIFEST_FORMAT: &str = "dever-package-v1";
@@ -406,6 +406,64 @@ fn settings(project_root: &Path) -> Result<(String, Vec<PackageRequest>), String
}
}
pub(crate) fn restore_locked(
project_root: &Path,
lock: &LockFile,
store: &dyn ArtifactStore,
) -> Result<(), String> {
let (origin, roots) = settings(project_root)?;
validate_locked_roots(&roots, &lock.packages)?;
if lock.packages.is_empty() {
return Ok(());
}
restore_locked_with(
project_root,
lock,
&HttpPackageTransport::new(&origin)?,
store,
)
}
/// Exact Package archives are restored before source/Worker checking. No index
/// is consulted: both dependency versions and archive identities are locked.
pub fn restore_locked_with(
project_root: &Path,
lock: &LockFile,
transport: &dyn PackageTransport,
store: &dyn ArtifactStore,
) -> Result<(), String> {
validate_locked_roots(&settings(project_root)?.1, &lock.packages)?;
for package in &lock.packages {
let cached = store.get_exact(&package.sha256, package.bytes, "package")?;
let missing = cached.is_none();
let bytes = match cached {
Some(bytes) => bytes,
None => transport.get(
&format!("/v1/packages/{}/{}.zip", package.name, package.version),
package.bytes as usize,
)?,
};
if bytes.len() as u64 != package.bytes || digest(&bytes) != package.sha256 {
return Err(format!(
"Package {}@{} differs from dever.lock",
package.name, package.version
));
}
let archive = manifest(&bytes, &package.name, &package.version)?;
if archive.manifest_sha256 != package.manifest_sha256 {
return Err(format!(
"Package {} manifest differs from dever.lock",
package.name
));
}
if missing && store.publish(&bytes, "package")? != package.sha256 {
return Err("restored Package cache identity mismatch".into());
}
}
owned_files_with(project_root, store)?;
Ok(())
}
#[derive(Clone)]
struct SelectedVersion {
index: IndexVersion,
@@ -1169,7 +1227,12 @@ fn execute_locked(
resolve_with(&roots, transport, store)?
};
check_project_collisions(project_root, &closure.packages, store)?;
let workers = libs::source_workers_with_packages(
let worker_loader = if command == "remove" {
libs::source_workers_with_packages
} else {
libs::prepare_source_workers_with_packages
};
let workers = worker_loader(
project_root,
&closure.sources,
crate::toolchain::BuildTarget::host()?,
+21 -2
View File
@@ -235,14 +235,33 @@ impl CacheStore {
}
pub fn artifact(&self, expected: &ArtifactReceipt) -> Result<StoredArtifact, String> {
self.artifact_optional(expected)?
.ok_or_else(|| "shared artifact unavailable: artifact is not cached".into())
}
pub fn artifact_optional(
&self,
expected: &ArtifactReceipt,
) -> Result<Option<StoredArtifact>, String> {
expected.validate()?;
let operation = CacheOperation::acquire(&self.root)?;
let directory = self.root.join("artifacts");
for path in [&directory, &directory.join(&expected.sha256)] {
match fs::symlink_metadata(path) {
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(error) => return Err(format!("cannot inspect shared artifact: {error}")),
}
if path == &directory {
ensure_real_directory(&directory)?;
}
}
let file = self.read_artifact_file(expected)?;
Ok(StoredArtifact {
Ok(Some(StoredArtifact {
receipt: expected.clone(),
file,
_operation: operation,
})
}))
}
fn read_artifact_file(&self, expected: &ArtifactReceipt) -> Result<File, String> {
@@ -0,0 +1,506 @@
//! One signed catalog owns both the base release and immutable optional resources.
use std::collections::BTreeSet;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use super::BuildTarget;
use super::release::{
self, Artifact, InstallLock, Layout, MachineManager, ReleaseManifest, Version,
};
use crate::libs::Ecosystem;
/// Explicit author/test transport. Product IPC always uses the compiled official source.
pub trait ExtensionSource {
fn open(&self, version: &Version, extension: &ExtensionId)
-> Result<Box<dyn io::Read>, String>;
}
pub(super) const PAYLOAD_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
pub(super) const ARTIFACT_LIMIT: usize = 4096;
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(
tag = "type",
content = "ecosystem",
rename_all = "snake_case",
deny_unknown_fields
)]
pub enum ExtensionKind {
Runtime(Ecosystem),
Build(Ecosystem),
Target,
}
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ExtensionId {
pub kind: ExtensionKind,
pub target: BuildTarget,
}
impl ExtensionId {
pub fn name(&self) -> String {
let kind = match &self.kind {
ExtensionKind::Runtime(ecosystem) => format!("runtime-{}", ecosystem.as_str()),
ExtensionKind::Build(ecosystem) => format!("build-{}", ecosystem.as_str()),
ExtensionKind::Target => "target".into(),
};
format!("{kind}-{}", self.target.platform())
}
pub fn asset_suffix(&self) -> String {
format!("ext-{}.tar.zst", self.name())
}
fn permits(&self, path: &str) -> bool {
let target = self.target.platform();
match &self.kind {
ExtensionKind::Runtime(ecosystem) => {
path.starts_with(&format!("runtime/{}/{target}/", ecosystem.as_str()))
}
ExtensionKind::Build(ecosystem) => {
path.starts_with(&format!("build/{}/{target}/", ecosystem.as_str()))
}
ExtensionKind::Target => {
path.starts_with(&format!("runtime/{target}/"))
|| path.starts_with(&format!("sandbox/{target}/"))
}
}
}
fn missing(&self) -> String {
match self.kind {
ExtensionKind::Target => format!(
"target resources are not installed; run 'dever target add {}'",
self.target.platform()
),
_ => format!(
"{} resources are not installed; run 'dever lib install <project-root> --target {}' (or 'dever lib add' for new dependencies)",
self.name(),
self.target.platform()
),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Extension {
pub kind: ExtensionKind,
pub target: BuildTarget,
pub artifacts: Vec<Artifact>,
}
impl Extension {
pub fn id(&self) -> ExtensionId {
ExtensionId {
kind: self.kind.clone(),
target: self.target,
}
}
}
pub fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
Version::parse(manifest.version.as_str())?;
if manifest.format != "dever-release-v2" || manifest.platform != release::platform_identity() {
return Err("unsupported release format or host platform".into());
}
if manifest.extensions.len() > 12 {
return Err("release extension count exceeds its limit".into());
}
let mut paths = BTreeSet::new();
validate_artifacts(&manifest.artifacts, &mut paths)?;
let mut ids = BTreeSet::new();
for extension in &manifest.extensions {
let id = extension.id();
if !ids.insert(id.clone())
|| matches!(id.kind, ExtensionKind::Build(Ecosystem::Go))
|| matches!(id.kind, ExtensionKind::Target) && id.target == BuildTarget::host()?
{
return Err("release has a duplicate or unsupported extension identity".into());
}
validate_artifacts(&extension.artifacts, &mut paths)?;
if extension
.artifacts
.iter()
.any(|artifact| !id.permits(&artifact.path))
{
return Err(format!(
"extension '{}' contains a file outside its resource namespace",
id.name()
));
}
}
Ok(())
}
pub(super) fn validate_artifacts<'a>(
artifacts: &'a [Artifact],
paths: &mut BTreeSet<&'a str>,
) -> Result<(), String> {
if artifacts.is_empty() || artifacts.len() > ARTIFACT_LIMIT {
return Err("release artifact count is outside 1..=4096".into());
}
let mut bytes = 0_u64;
for artifact in artifacts {
if artifact.path.is_empty()
|| artifact.path.len() > 1024
|| artifact
.path
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
|| artifact.path.contains(['\\', ':', '\0'])
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|| !paths.insert(&artifact.path)
{
return Err("release has an unsafe or duplicate artifact path".into());
}
if artifact.sha256.len() != 64
|| !artifact
.sha256
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err("release artifact has an invalid SHA-256".into());
}
bytes = bytes
.checked_add(artifact.bytes)
.ok_or("release size overflow")?;
if bytes > PAYLOAD_LIMIT {
return Err("release exceeds its two-GiB unpacked size limit".into());
}
}
// A signed file cannot also be the parent directory of another file.
for path in paths.iter() {
for (index, _) in path.match_indices('/') {
if paths.contains(&path[..index]) {
return Err("release artifact paths overlap as file and directory".into());
}
}
}
Ok(())
}
#[derive(Clone)]
pub struct SignedResources {
layout: Layout,
base: PathBuf,
manifest: ReleaseManifest,
catalog: String,
}
impl SignedResources {
pub fn load(layout: &Layout, version: &Version) -> Result<Self, String> {
Version::parse(version.as_str())?;
layout.validate_machine_permissions()?;
let base = layout.versions().join(version.as_str());
let manifest = MachineManager::new(layout.clone()).verify_release(&base, Some(version))?;
release::validate_installed_release(layout, &base, &manifest)?;
let catalog = release::sha256_file(&base.join("manifest.json"))?;
Ok(Self {
layout: layout.clone(),
base,
manifest,
catalog,
})
}
pub fn manifest(&self) -> &ReleaseManifest {
&self.manifest
}
pub fn root(&self, logical_path: &str) -> Result<PathBuf, String> {
if let Some(artifact) = self
.manifest
.artifacts
.iter()
.find(|artifact| artifact.path == logical_path)
{
verify_file(&self.base, artifact)?;
return Ok(self.base.clone());
}
let extension = self
.manifest
.extensions
.iter()
.find(|extension| {
extension
.artifacts
.iter()
.any(|artifact| artifact.path == logical_path)
})
.ok_or_else(|| format!("signed release does not declare resource '{logical_path}'"))?;
self.extension_root(extension)
}
pub fn path(&self, logical_path: &str) -> Result<PathBuf, String> {
Ok(self.root(logical_path)?.join(logical_path))
}
pub fn read(&self, logical_path: &str) -> Result<Vec<u8>, String> {
let path = self.path(logical_path)?;
let artifact = self
.manifest
.artifacts
.iter()
.chain(
self.manifest
.extensions
.iter()
.flat_map(|extension| &extension.artifacts),
)
.find(|artifact| artifact.path == logical_path)
.expect("root required signed artifact");
let bytes =
fs::read(path).map_err(|error| format!("cannot read signed resource: {error}"))?;
use sha2::{Digest, Sha256};
if bytes.len() as u64 != artifact.bytes
|| release::hex(&Sha256::digest(&bytes)) != artifact.sha256
{
return Err("signed resource changed while reading".into());
}
Ok(bytes)
}
pub fn artifacts(&self, prefix: &str) -> Result<Vec<Artifact>, String> {
let mut artifacts = Vec::new();
for artifact in self
.manifest
.artifacts
.iter()
.filter(|artifact| artifact.path.starts_with(prefix))
{
verify_file(&self.base, artifact)?;
artifacts.push(artifact.clone());
}
for extension in &self.manifest.extensions {
if extension
.artifacts
.iter()
.any(|artifact| artifact.path.starts_with(prefix))
{
self.extension_root(extension)?;
artifacts.extend(
extension
.artifacts
.iter()
.filter(|artifact| artifact.path.starts_with(prefix))
.cloned(),
);
}
}
Ok(artifacts)
}
fn store(&self) -> PathBuf {
self.layout
.extensions()
.join(self.manifest.version.as_str())
.join(&self.catalog)
}
fn destination(&self, id: &ExtensionId) -> PathBuf {
self.store().join(id.name())
}
fn extension(&self, id: &ExtensionId) -> Result<&Extension, String> {
self.manifest
.extensions
.iter()
.find(|extension| extension.id() == *id)
.ok_or_else(|| {
format!(
"Dever {} does not declare extension '{}'",
self.manifest.version,
id.name()
)
})
}
fn extension_root(&self, extension: &Extension) -> Result<PathBuf, String> {
let id = extension.id();
let root = self.destination(&id);
if !self.store_present()? || !exists(&root)? {
return Err(id.missing());
}
verify_extension(&root, &extension.artifacts)?;
Ok(root)
}
fn store_present(&self) -> Result<bool, String> {
let mut parent = self.layout.cache();
for directory in [
self.layout.extensions(),
self.layout
.extensions()
.join(self.manifest.version.as_str()),
self.store(),
] {
if !exists(&directory)? {
return Ok(false);
}
release::validate_shared_directory(&directory, &parent, "extension store")?;
parent = directory;
}
Ok(true)
}
pub(super) fn installed_extensions(&self) -> Result<Vec<ExtensionId>, String> {
if !self.store_present()? {
return Ok(Vec::new());
}
let mut installed = Vec::new();
for entry in fs::read_dir(self.store()).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let extension = self
.manifest
.extensions
.iter()
.find(|extension| entry.file_name() == extension.id().name().as_str())
.ok_or("extension store contains an undeclared entry")?;
self.extension_root(extension)?;
installed.push(extension.id());
}
Ok(installed)
}
}
pub(super) fn exists(path: &Path) -> Result<bool, String> {
match fs::symlink_metadata(path) {
Ok(_) => Ok(true),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
Err(error) => Err(format!(
"cannot inspect extension path '{}': {error}",
path.display()
)),
}
}
fn verify_file(root: &Path, artifact: &Artifact) -> Result<(), String> {
let mut parent = root.to_path_buf();
let relative = Path::new(&artifact.path);
for component in relative.parent().into_iter().flat_map(Path::components) {
let directory = parent.join(component);
release::validate_shared_directory(&directory, &parent, "signed resource directory")?;
parent = directory;
}
let path = root.join(relative);
release::validate_shared_file(&path, &parent, "signed resource", false)?;
if fs::metadata(&path)
.map_err(|error| error.to_string())?
.len()
!= artifact.bytes
|| release::sha256_file(&path)? != artifact.sha256
{
return Err(format!(
"signed resource '{}' failed integrity verification",
artifact.path
));
}
Ok(())
}
pub(super) fn verify_extension(root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
release::validate_shared_directory(
root,
root.parent().ok_or("extension root has no parent")?,
"installed extension",
)?;
for artifact in artifacts {
verify_file(root, artifact)?;
}
fn entries(root: &Path, directory: &Path, expected: &BTreeSet<&str>) -> Result<(), String> {
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let path = entry.path();
let kind = entry.file_type().map_err(|error| error.to_string())?;
let relative = path
.strip_prefix(root)
.map_err(|error| error.to_string())?
.to_str()
.ok_or("extension path is not UTF-8")?;
if kind.is_dir() {
release::validate_shared_directory(&path, directory, "extension directory")?;
if !expected
.iter()
.any(|artifact| artifact.starts_with(&format!("{relative}/")))
{
return Err("installed extension contains an unsigned directory".into());
}
entries(root, &path, expected)?;
} else if !kind.is_file() || !expected.contains(relative) {
return Err("installed extension contains an unsigned or non-regular file".into());
}
}
Ok(())
}
entries(
root,
root,
&artifacts
.iter()
.map(|artifact| artifact.path.as_str())
.collect(),
)
}
pub(super) fn ensure(layout: &Layout, version: &Version, id: &ExtensionId) -> Result<(), String> {
prepare_extension_with(
layout,
version,
id,
&super::release_source::ReleaseSource::official(),
)
}
pub fn prepare_extension_with(
layout: &Layout,
version: &Version,
id: &ExtensionId,
source: &dyn ExtensionSource,
) -> Result<(), String> {
Version::parse(version.as_str())?;
let _lock = InstallLock::acquire_wait(layout)?;
ensure_locked(layout, version, id, source)
}
pub(super) fn ensure_locked(
layout: &Layout,
version: &Version,
id: &ExtensionId,
source: &dyn ExtensionSource,
) -> Result<(), String> {
#[cfg(target_os = "linux")]
super::bootstrap::require_no_pending(layout)?;
let resources = SignedResources::load(layout, version)?;
let extension = resources.extension(id)?;
let destination = resources.destination(id);
if exists(&destination)? {
resources.extension_root(extension)?;
return Ok(());
}
let mut parent = layout.cache();
for directory in [
layout.extensions(),
layout.extensions().join(version.as_str()),
resources.store(),
] {
if !exists(&directory)? {
fs::create_dir(&directory)
.map_err(|error| format!("cannot create extension store: {error}"))?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&directory, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
}
release::sync_directory(&parent).map_err(|error| error.to_string())?;
}
release::validate_shared_directory(&directory, &parent, "extension store")?;
parent = directory;
}
super::release_source::prepare_extension(layout, version, extension, &destination, source)
}
+15 -1
View File
@@ -5,7 +5,7 @@ use std::process::Command;
use super::release::{Layout, MachineManager, Version, platform_contract};
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever target add linux-aarch64\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|install|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
#[derive(Debug)]
pub enum CommandResult {
@@ -30,6 +30,20 @@ pub fn execute_in(layout: &Layout, arguments: &[OsString]) -> Result<CommandResu
super::bootstrap::require_no_pending(layout)?;
let manager = MachineManager::new(layout.clone());
match arguments {
[command, operation, target] if command == "target" && operation == "add" => {
let target = utf8(target, "target")?.parse::<super::BuildTarget>()?;
if target != super::BuildTarget::LinuxAarch64 {
return Err("target add supports linux-aarch64".into());
}
let version = manager
.active_version()?
.ok_or("no active Dever version; run 'dever install latest'")?;
super::ensure_extension(layout, &version, super::ExtensionKind::Target, target)?;
Ok(CommandResult::Message(format!(
"prepared {} for Dever {version}",
target.platform()
)))
}
[command, requested] if command == "install" => {
let requested = utf8(requested, "version")?;
super::release_source::prepare(layout, requested)?;
+11 -1
View File
@@ -3,15 +3,22 @@ pub mod bootstrap;
mod cache;
pub mod compilation;
mod core_libraries;
mod extensions;
mod launcher;
pub mod packaging;
mod release;
mod release_archive;
mod release_source;
pub mod runtime_pack;
mod service;
mod skill;
mod target;
pub use extensions::{
Extension, ExtensionId, ExtensionKind, ExtensionSource, SignedResources,
prepare_extension_with, validate_catalog,
};
pub use release_source::extract_config;
pub use target::BuildTarget;
pub use cache::{
@@ -22,4 +29,7 @@ pub use release::{
Artifact, DownloadedReleases, Layout, MachineManager, PlatformContract, ReleaseManifest,
Version, platform_contract, platform_identity, sha256_file,
};
pub use service::{artifact_get, artifact_put, cache_clean, cache_status, compile, serve};
pub use service::{
artifact_get, artifact_get_optional, artifact_put, cache_clean, cache_status, compile,
ensure_extension, serve,
};
+49 -8
View File
@@ -12,7 +12,10 @@ use serde::Deserialize;
use sha2::{Digest, Sha256};
use super::runtime_pack::{self, Manifest, Profile, Profiles, RuntimeArtifact};
use super::{Artifact, BuildTarget, ReleaseManifest, Version, sha256_file};
use super::{
Artifact, BuildTarget, Extension, ExtensionKind, ReleaseManifest, Version, sha256_file,
validate_catalog,
};
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
@@ -205,33 +208,71 @@ fn assemble(
end: settings.end,
},
)?);
let mut extensions = Vec::new();
let mut target_artifacts = BTreeMap::new();
for (target, inputs) in settings.native_targets {
artifacts.extend(assemble_native(&mut writer, &version, target, inputs)?);
target_artifacts.insert(
target,
assemble_native(&mut writer, &version, target, inputs)?,
);
}
for (ecosystem, input) in settings.runtimes.entries() {
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, host)?);
extensions.push(Extension {
kind: ExtensionKind::Runtime(ecosystem.clone()),
target: host,
artifacts: ecosystems::assemble(&mut writer, ecosystem, input, host)?,
});
}
for (target, inputs) in settings.ecosystem_targets {
for (ecosystem, input) in inputs.entries() {
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, target)?);
extensions.push(Extension {
kind: ExtensionKind::Runtime(ecosystem.clone()),
target,
artifacts: ecosystems::assemble(&mut writer, ecosystem, input, target)?,
});
}
}
if let Some(inputs) = settings.sandbox {
artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?);
}
for (target, inputs) in settings.sandbox_targets {
artifacts.extend(sandbox::assemble(&mut writer, inputs, target)?);
let selected = target_artifacts
.get_mut(&target)
.ok_or("target sandbox requires a matching native target")?;
selected.extend(sandbox::assemble(&mut writer, inputs, target)?);
}
for (target, artifacts) in target_artifacts {
extensions.push(Extension {
kind: ExtensionKind::Target,
target,
artifacts,
});
}
for (ecosystem, input) in settings.builds.entries() {
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, host)?);
extensions.push(Extension {
kind: ExtensionKind::Build(ecosystem.clone()),
target: host,
artifacts: builds::assemble(&mut writer, ecosystem, input, host)?,
});
}
for (target, inputs) in settings.build_targets {
for (ecosystem, input) in inputs.entries() {
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, target)?);
extensions.push(Extension {
kind: ExtensionKind::Build(ecosystem.clone()),
target,
artifacts: builds::assemble(&mut writer, ecosystem, input, target)?,
});
}
}
artifacts.sort_by(|left, right| left.path.cmp(&right.path));
let manifest = ReleaseManifest::new(version, artifacts);
for extension in &mut extensions {
extension
.artifacts
.sort_by(|left, right| left.path.cmp(&right.path));
}
let mut manifest = ReleaseManifest::new(version, artifacts);
manifest.extensions = extensions;
validate_catalog(&manifest)?;
let bytes = write_metadata(&staging.join("manifest.json"), &manifest)?;
let signature = key.pair.sign(&bytes);
let hex: String = signature
+77 -17
View File
@@ -74,15 +74,17 @@ pub struct ReleaseManifest {
pub version: Version,
pub platform: String,
pub artifacts: Vec<Artifact>,
pub extensions: Vec<super::extensions::Extension>,
}
impl ReleaseManifest {
pub fn new(version: Version, artifacts: Vec<Artifact>) -> Self {
Self {
format: "dever-release-v1".into(),
format: "dever-release-v2".into(),
version,
platform: platform_identity(),
artifacts,
extensions: Vec::new(),
}
}
}
@@ -129,6 +131,9 @@ impl Layout {
pub fn cache(&self) -> PathBuf {
self.root.join("cache")
}
pub fn extensions(&self) -> PathBuf {
self.cache().join("extensions")
}
pub fn downloads(&self) -> PathBuf {
self.cache().join("downloads")
}
@@ -307,6 +312,11 @@ impl MachineManager {
}
pub fn update(&self) -> Result<Version, String> {
self.update_with(&super::release_source::ReleaseSource::official())
}
/// Uses the ordinary install transaction with an explicitly supplied author transport.
pub fn update_with(&self, source: &dyn super::ExtensionSource) -> Result<Version, String> {
let version = self.releases.latest()?;
let _lock = InstallLock::acquire(&self.layout)?;
#[cfg(target_os = "linux")]
@@ -321,6 +331,12 @@ impl MachineManager {
}
self.cleanup_staging()?;
self.install_locked(&version)?;
if let Some(active) = self.active_version()? {
let resources = super::extensions::SignedResources::load(&self.layout, &active)?;
for extension in resources.installed_extensions()? {
super::extensions::ensure_locked(&self.layout, &version, &extension, source)?;
}
}
self.activate_locked(&version)?;
Ok(version)
}
@@ -468,6 +484,7 @@ impl MachineManager {
Ok(InstalledCompiler {
core,
manifest,
resources: super::SignedResources::load(&self.layout, version)?,
identity: sha256_file(&root.join(MANIFEST_NAME))?,
_lease: lease,
})
@@ -486,6 +503,14 @@ impl MachineManager {
if destination.exists() {
let manifest = self.verify_release(&destination, Some(version))?;
validate_installed_release(&self.layout, &destination, &manifest)?;
if package.exists() {
self.verify_release(package, Some(version))?;
if read_regular_file(&destination.join(MANIFEST_NAME))?
!= read_regular_file(&package.join(MANIFEST_NAME))?
{
return Err("published Dever version changed its signed manifest".into());
}
}
return Ok(());
}
let manifest = self.verify_release(package, Some(version))?;
@@ -548,7 +573,7 @@ impl MachineManager {
.map_err(|error| format!("cannot activate Dever {version}: {error}"))
}
fn verify_release(
pub(super) fn verify_release(
&self,
root: &Path,
expected: Option<&Version>,
@@ -601,7 +626,7 @@ impl MachineManager {
manifest_path.display()
)
})?;
if manifest.format != "dever-release-v1" {
if manifest.format != "dever-release-v2" {
return Err("unsupported Dever release manifest format".into());
}
if manifest.platform != platform_identity() {
@@ -611,6 +636,7 @@ impl MachineManager {
platform_identity()
));
}
super::extensions::validate_catalog(&manifest)?;
Ok(manifest)
}
@@ -706,6 +732,7 @@ pub(crate) struct InstalledCompiler {
pub(crate) core: PathBuf,
pub(crate) manifest: ReleaseManifest,
pub(crate) identity: String,
pub(crate) resources: super::SignedResources,
_lease: InstallLock,
}
@@ -713,14 +740,18 @@ pub(super) struct InstallLock(File);
impl InstallLock {
pub(super) fn acquire(layout: &Layout) -> Result<Self, String> {
Self::open(layout, false)
Self::open(layout, false, false)
}
pub(super) fn acquire_wait(layout: &Layout) -> Result<Self, String> {
Self::open(layout, false, true)
}
fn shared(layout: &Layout) -> Result<Self, String> {
Self::open(layout, true)
Self::open(layout, true, false)
}
fn open(layout: &Layout, shared: bool) -> Result<Self, String> {
fn open(layout: &Layout, shared: bool, wait: bool) -> Result<Self, String> {
let path = layout.state().join("install.lock");
let mut options = OpenOptions::new();
options.read(true).write(true).create(true).truncate(false);
@@ -750,14 +781,29 @@ impl InstallLock {
}
#[cfg(not(unix))]
let _ = current;
let locked = if shared {
fs2::FileExt::try_lock_shared(&file)
} else {
fs2::FileExt::try_lock_exclusive(&file)
};
locked.map_err(|error| {
format!("another Dever installation operation or compilation is active: {error}")
})?;
let started = std::time::Instant::now();
loop {
let locked = if shared {
fs2::FileExt::try_lock_shared(&file)
} else {
fs2::FileExt::try_lock_exclusive(&file)
};
match locked {
Ok(()) => break,
Err(error)
if wait
&& error.kind() == io::ErrorKind::WouldBlock
&& started.elapsed() < std::time::Duration::from_secs(300) =>
{
std::thread::sleep(std::time::Duration::from_millis(50))
}
Err(error) => {
return Err(format!(
"another Dever installation operation or compilation is active: {error}"
));
}
}
}
Ok(Self(file))
}
}
@@ -912,7 +958,17 @@ fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> {
if !metadata.file_type().is_file() {
return Err(format!("'{}' must be a regular file", path.display()));
}
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
const LIMIT: u64 = 2 * 1024 * 1024;
let mut bytes = Vec::new();
File::open(path)
.map_err(|error| format!("cannot read '{}': {error}", path.display()))?
.take(LIMIT + 1)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
if bytes.len() as u64 > LIMIT {
return Err("release metadata exceeds its size limit".into());
}
Ok(bytes)
}
pub(super) fn copy_file(source: &Path, destination: &Path, executable: bool) -> Result<(), String> {
@@ -964,7 +1020,7 @@ pub(super) fn sync_tree(root: &Path) -> Result<(), String> {
sync_directory(root).map_err(|error| error.to_string())
}
fn validate_installed_release(
pub(super) fn validate_installed_release(
layout: &Layout,
root: &Path,
manifest: &ReleaseManifest,
@@ -995,7 +1051,11 @@ fn validate_installed_release(
Ok(())
}
fn validate_shared_directory(path: &Path, parent: &Path, label: &str) -> Result<(), String> {
pub(super) fn validate_shared_directory(
path: &Path,
parent: &Path,
label: &str,
) -> Result<(), String> {
let metadata = fs::symlink_metadata(path)
.map_err(|error| format!("cannot inspect {label} '{}': {error}", path.display()))?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
@@ -0,0 +1,206 @@
//! Bounded single-frame Zstandard and exact signed USTAR extraction.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, OpenOptions};
use std::io::{self, BufRead, BufReader, Read};
use std::path::Path;
use super::extensions::{ARTIFACT_LIMIT, PAYLOAD_LIMIT, validate_artifacts};
use super::release::{Artifact, set_shared_file_permissions, sha256_file};
pub(super) fn extract(
reader: impl Read,
root: &Path,
artifacts: &[Artifact],
) -> Result<(), String> {
validate_artifacts(artifacts, &mut BTreeSet::new())?;
let mut input = Bounded {
input: reader,
remaining: PAYLOAD_LIMIT,
};
let mut header = [0_u8; 5];
input
.read_exact(&mut header)
.map_err(|error| format!("cannot read Zstandard header: {error}"))?;
if header[..4] != [0x28, 0xb5, 0x2f, 0xfd] || header[4] & 4 == 0 {
return Err("release requires a Zstandard frame with a content checksum".into());
}
let mut decoder = zstd::stream::read::Decoder::with_buffer(BufReader::new(
io::Cursor::new(header).chain(input),
))
.map_err(|error| format!("cannot initialize Zstandard decoder: {error}"))?
.single_frame();
decoder
.window_log_max(27)
.map_err(|error| format!("cannot bound Zstandard window: {error}"))?;
unpack(&mut decoder, root, artifacts)?;
// unpack drains the decoder through its checksum, including errors after TAR EOF.
let mut input = decoder.finish();
if !input
.fill_buf()
.map_err(|error| format!("cannot finish release frame: {error}"))?
.is_empty()
{
return Err("release contains data after its single Zstandard frame".into());
}
Ok(())
}
pub(super) fn unpack(reader: impl Read, root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
validate_artifacts(artifacts, &mut BTreeSet::new())?;
let expected: BTreeMap<_, _> = artifacts
.iter()
.map(|artifact| (artifact.path.as_str(), artifact))
.collect();
let mut seen = BTreeSet::new();
let mut archive = Bounded {
input: reader,
remaining: PAYLOAD_LIMIT + ARTIFACT_LIMIT as u64 * 1024 + 10240,
};
while let Some(header) = read_header(&mut archive)? {
let name = std::str::from_utf8(&header.path_bytes())
.map_err(|_| "release archive path is not UTF-8")?
.to_owned();
let artifact = expected
.get(name.as_str())
.ok_or("release archive contains an unsigned file")?;
if !seen.insert(name) || header.size().map_err(|error| error.to_string())? != artifact.bytes
{
return Err("release archive contains a duplicate or wrong-sized file".into());
}
extract_file(&mut archive, root, artifact)?;
}
if seen.len() != expected.len() {
return Err("release archive is missing signed artifacts".into());
}
verify_trailer(&mut archive)
}
fn read_header(reader: &mut impl Read) -> Result<Option<tar::Header>, String> {
let mut header = tar::Header::new_ustar();
reader
.read_exact(header.as_mut_bytes())
.map_err(|error| format!("incomplete release TAR header: {error}"))?;
if header.as_bytes().iter().all(|byte| *byte == 0) {
return Ok(None);
}
let checksum = header.as_bytes()[..148]
.iter()
.chain(&header.as_bytes()[156..])
.map(|byte| u32::from(*byte))
.sum::<u32>()
+ 8 * u32::from(b' ');
if header.cksum().map_err(|error| error.to_string())? != checksum {
return Err("release TAR header checksum is invalid".into());
}
if !header.entry_type().is_file() || header.as_ustar().is_none() {
return Err("release archives may contain only USTAR regular files".into());
}
Ok(Some(header))
}
fn extract_file(reader: &mut impl Read, root: &Path, artifact: &Artifact) -> Result<(), String> {
let path = root.join(&artifact.path);
let mut parent = root.to_path_buf();
for part in Path::new(&artifact.path)
.parent()
.into_iter()
.flat_map(Path::components)
{
parent.push(part);
match fs::create_dir(&parent) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
let metadata = fs::symlink_metadata(&parent).map_err(|error| error.to_string())?;
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err("release artifact parent is not a real directory".into());
}
}
Err(error) => return Err(format!("cannot create release artifact directory: {error}")),
}
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&parent, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
}
}
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.map_err(|error| format!("cannot create release artifact: {error}"))?;
if io::copy(&mut reader.by_ref().take(artifact.bytes), &mut file)
.map_err(|error| format!("cannot extract release artifact: {error}"))?
!= artifact.bytes
{
return Err("release archive contains a truncated file".into());
}
let padding_length = ((512 - artifact.bytes % 512) % 512) as usize;
let mut padding = [0; 512];
reader
.read_exact(&mut padding[..padding_length])
.map_err(|error| format!("incomplete TAR entry padding: {error}"))?;
if padding[..padding_length].iter().any(|byte| *byte != 0) {
return Err("release TAR entry padding must be zero".into());
}
if sha256_file(&path)? != artifact.sha256 {
return Err("release artifact failed integrity verification".into());
}
// First-install invokes the extracted signed bootstrap before machine install.
// Archive mode bits are untrusted; executable roles come from fixed paths.
let executable = matches!(
artifact.path.as_str(),
"dever-core" | "bootstrap/dever" | "bootstrap/deverd"
);
set_shared_file_permissions(&path, executable)?;
file.sync_all().map_err(|error| error.to_string())
}
fn verify_trailer(reader: &mut impl Read) -> Result<(), String> {
let mut tail_bytes = 0_usize;
let mut tail = [0_u8; 4096];
loop {
let length = reader
.read(&mut tail)
.map_err(|error| format!("invalid release archive trailer: {error}"))?;
if length == 0 {
break;
}
tail_bytes += length;
if tail_bytes > 10240 || tail[..length].iter().any(|byte| *byte != 0) {
return Err("release archive contains an invalid TAR trailer".into());
}
}
// The first zero header was consumed by read_header; require the second.
if tail_bytes < 512 || !tail_bytes.is_multiple_of(512) {
return Err("release archive is missing its complete TAR terminator".into());
}
Ok(())
}
struct Bounded<R> {
input: R,
remaining: u64,
}
impl<R: Read> Read for Bounded<R> {
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
if buffer.is_empty() {
return Ok(0);
}
if self.remaining == 0 {
let mut probe = [0];
return match self.input.read(&mut probe)? {
0 => Ok(0),
_ => Err(io::Error::new(
io::ErrorKind::InvalidData,
"release byte limit exceeded",
)),
};
}
let limit = (self.remaining.min(buffer.len() as u64)) as usize;
let length = self.input.read(&mut buffer[..limit])?;
self.remaining -= length as u64;
Ok(length)
}
}
+134 -83
View File
@@ -1,31 +1,35 @@
//! Explicit installation/update downloads. Project commands never enter here.
use std::collections::{BTreeMap, BTreeSet};
use std::fs::{self, OpenOptions};
use std::fs::{self, File, OpenOptions};
use std::io::{self, Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use super::extensions::Extension;
#[cfg(test)]
use super::extensions::PAYLOAD_LIMIT as RELEASE_LIMIT;
#[cfg(test)]
use super::release::ReleaseManifest;
use super::release::{
InstallLock, Layout, MachineManager, ReleaseManifest, Version, decode_hex_file,
ensure_real_directory, platform_identity, reject_symlink_ancestors,
set_shared_file_permissions, sync_directory, sync_tree, validate_shared_file,
InstallLock, Layout, MachineManager, Version, decode_hex_file, ensure_real_directory,
platform_identity, reject_symlink_ancestors, set_shared_file_permissions, sync_directory,
sync_tree, validate_shared_file,
};
use super::validate_catalog;
const OFFICIAL_RELEASES: &str = "https://github.com/shemic/dever-main/releases";
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300);
const METADATA_LIMIT: u64 = 2 * 1024 * 1024;
const RELEASE_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
static NEXT_DOWNLOAD: AtomicU64 = AtomicU64::new(0);
struct ReleaseSource {
pub(super) struct ReleaseSource {
base: String,
agent: ureq::Agent,
}
impl ReleaseSource {
fn official() -> Self {
pub(super) fn official() -> Self {
Self::new(OFFICIAL_RELEASES.to_owned())
}
@@ -122,6 +126,23 @@ impl ReleaseSource {
}
}
impl super::ExtensionSource for ReleaseSource {
fn open(
&self,
version: &Version,
extension: &super::ExtensionId,
) -> Result<Box<dyn Read>, String> {
Ok(Box::new(
self.response(
self.asset(version.as_str(), &extension.asset_suffix()),
Instant::now(),
)?
.into_body()
.into_reader(),
))
}
}
fn redirect(current: &str, location: &str) -> Result<String, String> {
let next = url::Url::parse(current)
.and_then(|base| base.join(location))
@@ -218,12 +239,11 @@ fn prepare_from(
}
Err(error) if error.kind() == io::ErrorKind::NotFound => {
let mut response =
source.response(source.asset(version.as_str(), "tar.gz"), started)?;
let compressed = response.body_mut().as_reader().take(RELEASE_LIMIT + 1);
unpack(
flate2::read::GzDecoder::new(compressed),
source.response(source.asset(version.as_str(), "tar.zst"), started)?;
super::release_archive::extract(
response.body_mut().as_reader(),
&staging.0,
&manifest,
&manifest.artifacts,
)?;
MachineManager::verify_package(&staging.0, Some(&version), &key)?;
sync_tree(&staging.0)?;
@@ -250,80 +270,111 @@ fn prepare_from(
Ok(version)
}
fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
if manifest.artifacts.is_empty() || manifest.artifacts.len() > 4096 {
return Err("release artifact count is outside 1..=4096".into());
}
let mut paths = BTreeSet::new();
let mut size = 0_u64;
for artifact in &manifest.artifacts {
if artifact
.path
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
|| artifact.path.contains(['\\', ':', '\0'])
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|| !paths.insert(&artifact.path)
{
return Err("release has an unsafe or duplicate artifact path".into());
}
size = size
.checked_add(artifact.bytes)
.ok_or("release size overflow")?;
if size > RELEASE_LIMIT {
return Err("release exceeds its two-GiB unpacked size limit".into());
}
}
Ok(())
}
#[cfg(test)]
fn unpack(reader: impl Read, root: &Path, manifest: &ReleaseManifest) -> Result<(), String> {
validate_catalog(manifest)?;
let expected: BTreeMap<_, _> = manifest
.artifacts
.iter()
.map(|artifact| (artifact.path.as_str(), artifact))
.collect();
let mut seen = BTreeSet::new();
// Raw entries reject PAX/GNU extensions before tar can buffer arbitrary metadata.
let mut archive = tar::Archive::new(reader.take(RELEASE_LIMIT + 4096 * 1024));
for entry in archive
.entries()
.map_err(|error| format!("invalid release archive: {error}"))?
.raw(true)
super::release_archive::unpack(reader, root, &manifest.artifacts)
}
pub(super) fn prepare_extension(
layout: &Layout,
version: &Version,
extension: &Extension,
destination: &Path,
source: &dyn super::ExtensionSource,
) -> Result<(), String> {
let staging = Download::new(&layout.staging())?;
super::release_archive::extract(
source.open(version, &extension.id())?,
&staging.0,
&extension.artifacts,
)?;
#[cfg(unix)]
{
let mut entry = entry.map_err(|error| format!("invalid release archive entry: {error}"))?;
if !entry.header().entry_type().is_file() {
return Err("release archives may contain only regular files".into());
}
let bytes = entry.path_bytes();
let name = std::str::from_utf8(&bytes)
.map_err(|_| "release archive path is not UTF-8")?
.to_owned();
let artifact = expected
.get(name.as_str())
.ok_or("release archive contains an unsigned file")?;
if !seen.insert(name.clone()) || entry.size() != artifact.bytes {
return Err("release archive contains a duplicate or wrong-sized file".into());
}
let path = root.join(&name);
fs::create_dir_all(path.parent().expect("artifact has a parent"))
.map_err(|error| format!("cannot create release artifact directory: {error}"))?;
let mut file = OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.map_err(|error| format!("cannot create release artifact: {error}"))?;
let bytes = io::copy(&mut entry, &mut file)
.map_err(|error| format!("cannot extract release artifact: {error}"))?;
if bytes != artifact.bytes {
return Err("release archive contains a truncated file".into());
}
set_shared_file_permissions(&path, false)?;
file.sync_all().map_err(|error| error.to_string())?;
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&staging.0, fs::Permissions::from_mode(0o755))
.map_err(|error| error.to_string())?;
}
if seen.len() != expected.len() {
return Err("release archive is missing signed artifacts".into());
super::extensions::verify_extension(&staging.0, &extension.artifacts)?;
if matches!(extension.kind, super::ExtensionKind::Target) {
super::runtime_pack::validate(&staging.0, version.as_str(), extension.target)?;
}
sync_tree(&staging.0)?;
publish(&staging.0, destination)?;
sync_directory(destination.parent().ok_or("extension has no parent")?)
.map_err(|error| error.to_string())
}
/// Private, independently authenticated installer entry; never accepted over IPC.
pub fn extract_config(config: &Path) -> Result<(), String> {
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct Extraction {
release: PathBuf,
archive: PathBuf,
trusted_key: PathBuf,
}
trusted_input(config, false)?;
let mut bytes = Vec::new();
File::open(config)
.map_err(|error| error.to_string())?
.take(METADATA_LIMIT + 1)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
if bytes.len() as u64 > METADATA_LIMIT {
return Err("extraction config exceeds its size limit".into());
}
let settings: Extraction = serde_json::from_slice(&bytes)
.map_err(|error| format!("invalid extraction config: {error}"))?;
trusted_input(&settings.release, true)?;
trusted_input(&settings.archive, false)?;
trusted_input(&settings.trusted_key, false)?;
if settings.trusted_key.starts_with(&settings.release) {
return Err("trusted key must be supplied independently outside the release".into());
}
let key = decode_hex_file(&settings.trusted_key, 32)?;
let manifest = MachineManager::signed_manifest(&settings.release, &key)?;
let archive = File::open(&settings.archive).map_err(|error| error.to_string())?;
super::release_archive::extract(archive, &settings.release, &manifest.artifacts)?;
MachineManager::verify_package(&settings.release, Some(&manifest.version), &key)?;
sync_tree(&settings.release)
}
fn trusted_input(path: &Path, directory: bool) -> Result<(), String> {
if !path.is_absolute()
|| path.components().any(|part| {
matches!(
part,
std::path::Component::ParentDir | std::path::Component::CurDir
)
})
{
return Err("extraction paths must be absolute and normalized".into());
}
reject_symlink_ancestors(path)?;
let metadata = fs::symlink_metadata(path)
.map_err(|error| format!("cannot inspect extraction input: {error}"))?;
if directory && !metadata.is_dir() || !directory && !metadata.is_file() {
return Err("extraction input has the wrong file type".into());
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if metadata.uid() != 0
|| metadata.mode() & 0o022 != 0
|| !directory && metadata.nlink() != 1
{
return Err(
"extraction inputs must be root-owned and not writable by other users".into(),
);
}
for ancestor in path.ancestors().skip(1) {
let metadata = fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?;
if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0
{
return Err("extraction input ancestor is not protected by root".into());
}
}
}
Ok(())
}
+72 -5
View File
@@ -101,7 +101,13 @@ enum Operation {
Clean,
ArtifactPut(ArtifactReceipt),
ArtifactGet(ArtifactReceipt),
Compile { bytes: u64 },
EnsureExtension {
version: Version,
extension: super::ExtensionId,
},
Compile {
bytes: u64,
},
}
#[derive(Debug, Serialize, Deserialize)]
@@ -325,6 +331,15 @@ pub fn artifact_put(layout: &Layout, bytes: &[u8]) -> Result<ArtifactReceipt, St
}
pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>, String> {
artifact_get_optional(layout, sha256, bytes)?
.ok_or_else(|| "shared artifact unavailable: artifact is not cached".into())
}
pub fn artifact_get_optional(
layout: &Layout,
sha256: &str,
bytes: u64,
) -> Result<Option<Vec<u8>>, String> {
let expected = ArtifactReceipt {
sha256: sha256.to_owned(),
bytes,
@@ -341,7 +356,11 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
installed: Vec::new(),
},
)?;
require_artifact(read_frame(&mut stream)?, &expected)?;
let response: Response = read_frame(&mut stream)?;
if response.ok && response.artifact.is_none() {
return Ok(None);
}
require_artifact(response, &expected)?;
let mut bytes = vec![0; expected.bytes as usize];
stream
.read_exact(&mut bytes)
@@ -349,7 +368,7 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
if hex(&Sha256::digest(&bytes)) != expected.sha256 {
return Err("deverd artifact download failed SHA-256 verification".into());
}
Ok(bytes)
Ok(Some(bytes))
}
#[cfg(not(unix))]
{
@@ -358,6 +377,30 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
}
}
pub fn ensure_extension(
layout: &Layout,
version: &Version,
kind: super::ExtensionKind,
target: super::BuildTarget,
) -> Result<(), String> {
Version::parse(version.as_str())?;
let response = request(
layout,
Operation::EnsureExtension {
version: version.clone(),
extension: super::ExtensionId { kind, target },
},
&[],
)?;
if response.ok {
Ok(())
} else {
Err(response
.error
.unwrap_or_else(|| "extension preparation failed".into()))
}
}
fn require_artifact(response: Response, expected: &ArtifactReceipt) -> Result<(), String> {
if !response.ok {
return Err(response
@@ -418,6 +461,11 @@ fn request(
#[cfg(unix)]
{
let (mut stream, token) = connect(layout)?;
if matches!(operation, Operation::EnsureExtension { .. }) {
stream
.set_read_timeout(Some(std::time::Duration::from_secs(620)))
.map_err(|error| error.to_string())?;
}
let request = Request {
token,
operation,
@@ -513,8 +561,16 @@ fn handle(
};
}
Operation::ArtifactGet(expected) => {
let mut artifact = match store.artifact(&expected) {
Ok(artifact) => artifact,
let mut artifact = match store.artifact_optional(&expected) {
Ok(Some(artifact)) => artifact,
Ok(None) => {
return Response {
ok: true,
status: None,
artifact: None,
error: None,
};
}
Err(error) => return failure(error),
};
if let Err(error) = write_frame(stream, &artifact_response(artifact.receipt)) {
@@ -530,6 +586,17 @@ fn handle(
error: None,
};
}
Operation::EnsureExtension { version, extension } => {
return match super::extensions::ensure(layout, &version, &extension) {
Ok(()) => Response {
ok: true,
status: None,
artifact: None,
error: None,
},
Err(error) => failure(error),
};
}
Operation::Compile { bytes } => {
return match compile::serve(layout, stream, bytes) {
Ok(()) => Response {
@@ -112,13 +112,15 @@ fn verify_pack(
target: super::super::BuildTarget,
) -> Result<(), String> {
let root = compiler
.core
.parent()
.ok_or("installed compiler has no directory")?;
.resources
.root(&format!("runtime/{}/manifest.json", target.platform()))?;
let artifacts =
super::super::runtime_pack::validate(root, compiler.manifest.version.as_str(), target)?;
super::super::runtime_pack::validate(&root, compiler.manifest.version.as_str(), target)?;
let signed_artifacts = compiler
.resources
.artifacts(&format!("runtime/{}/", target.platform()))?;
for input in artifacts {
if !compiler.manifest.artifacts.iter().any(|signed| {
if !signed_artifacts.iter().any(|signed| {
signed.path == input.path
&& signed.bytes == input.bytes
&& signed.sha256 == input.sha256