feat: ship slim releases with shared on-demand extensions and locked restore
This commit is contained in:
1 parent
83b78a0db6
commit
4fda7c8c1a
58 files changed
+3774
-425
No files matched your search
@@ -28,6 +28,7 @@ base64 = "=0.22.1"
|
||||
zip = { version = "=2.4.2", default-features = false, features = ["deflate"] }
|
||||
tar = { version = "=0.4.44", default-features = false }
|
||||
flate2 = "=1.1.5"
|
||||
zstd = { version = "=0.13.3", default-features = false }
|
||||
csv = "=1.4.0"
|
||||
goblin = { version = "=0.10.7", default-features = false, features = ["elf32", "elf64", "endian_fd", "std"] }
|
||||
|
||||
@@ -70,6 +71,11 @@ name = "native-acceptance-init"
|
||||
path = "../../test/dever-cli-tests/support/native_acceptance_init.rs"
|
||||
test = false
|
||||
|
||||
[[example]]
|
||||
name = "prepare-extensions"
|
||||
path = "../../test/onboarding/prepare_extensions.rs"
|
||||
test = false
|
||||
|
||||
[[test]]
|
||||
name = "native_release"
|
||||
path = "../../test/dever-cli-tests/native_release.rs"
|
||||
|
||||
@@ -10,6 +10,14 @@ fn run() -> i32 {
|
||||
if arguments.as_slice() == ["--dever-bootstrap-health"] {
|
||||
return 0;
|
||||
}
|
||||
if let [flag, config] = arguments.as_slice()
|
||||
&& flag == "--dever-extract"
|
||||
{
|
||||
return match dever_cli::toolchain::extract_config(std::path::Path::new(config)) {
|
||||
Ok(()) => 0,
|
||||
Err(error) => fail(error),
|
||||
};
|
||||
}
|
||||
#[cfg(target_os = "linux")]
|
||||
if let [flag, root] = arguments.as_slice()
|
||||
&& flag == "--dever-install"
|
||||
|
||||
@@ -142,7 +142,15 @@ pub(super) fn worker_compile(
|
||||
},
|
||||
),
|
||||
};
|
||||
let pack = RuntimePack::load(&compiler_directory()?, profile, request.target)?;
|
||||
let (layout, version) =
|
||||
managed_compiler()?.ok_or("compile worker requires a managed compiler")?;
|
||||
let resources =
|
||||
dever_cli::toolchain::SignedResources::load(&layout, &Version::parse(&version)?)?;
|
||||
let root = resources.root(&format!(
|
||||
"runtime/{}/manifest.json",
|
||||
request.target.platform()
|
||||
))?;
|
||||
let pack = RuntimePack::load(&root, profile, request.target)?;
|
||||
link_ir(&ir, &pack, build, output)
|
||||
}
|
||||
|
||||
|
||||
+121
-36
@@ -15,6 +15,7 @@ use sha2::{Digest, Sha256};
|
||||
pub mod build;
|
||||
mod mutation;
|
||||
mod registry;
|
||||
pub mod restore;
|
||||
pub(crate) use mutation::ProjectMutation;
|
||||
pub use registry::npm;
|
||||
pub(crate) use registry::python_dependencies;
|
||||
@@ -22,7 +23,7 @@ pub use registry::sumdb;
|
||||
pub use registry::{HttpRegistry, RegistryResolver, RegistryRuntime, RegistryTransport};
|
||||
pub(crate) use registry::{InstalledRegistryPack, archive_files};
|
||||
|
||||
const LOCK_FORMAT: &str = "dever-lock-v5";
|
||||
const LOCK_FORMAT: &str = "dever-lock-v6";
|
||||
const TOOLCHAIN: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
@@ -240,6 +241,15 @@ pub struct LockedArtifact {
|
||||
pub path: String,
|
||||
pub bytes: u64,
|
||||
pub sha256: String,
|
||||
pub source: Option<RegistrySource>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct RegistrySource {
|
||||
pub ecosystem: Ecosystem,
|
||||
pub locator: String,
|
||||
pub filename: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
@@ -421,6 +431,7 @@ impl FixturePackage {
|
||||
path: artifact.path.clone(),
|
||||
bytes: artifact.bytes.len() as u64,
|
||||
sha256: sha256(&artifact.bytes),
|
||||
source: None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
artifacts.sort();
|
||||
@@ -635,7 +646,7 @@ pub(crate) fn resolve_project(
|
||||
lock.encode()?;
|
||||
return Ok(lock);
|
||||
}
|
||||
let managed = InstalledRegistry::current(target)?;
|
||||
let managed = PreparingRegistry(InstalledRegistry::current(target)?);
|
||||
let mut runtimes = BTreeMap::new();
|
||||
let mut ecosystems = all
|
||||
.iter()
|
||||
@@ -646,7 +657,10 @@ pub(crate) fn resolve_project(
|
||||
ecosystems.insert(worker.ecosystem.parse()?);
|
||||
}
|
||||
for ecosystem in ecosystems {
|
||||
runtimes.insert(ecosystem.clone(), managed.runtime(ecosystem)?.0);
|
||||
runtimes.insert(
|
||||
ecosystem.clone(),
|
||||
build::Inputs::runtime(&managed, ecosystem)?.0,
|
||||
);
|
||||
}
|
||||
for worker in workers.iter().filter(|worker| worker.ecosystem != "exec") {
|
||||
let ecosystem: Ecosystem = worker.ecosystem.parse()?;
|
||||
@@ -658,7 +672,7 @@ pub(crate) fn resolve_project(
|
||||
}
|
||||
}
|
||||
let store = ManagedArtifactStore {
|
||||
layout: managed.layout.clone(),
|
||||
layout: managed.0.layout.clone(),
|
||||
};
|
||||
let transport = HttpRegistry::official();
|
||||
let verified = sumdb::verify_chain(&prior_evidence, &sumdb::Verifier::official())?;
|
||||
@@ -705,13 +719,40 @@ impl LibResolver for ProjectResolver<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct InstalledRegistry {
|
||||
layout: crate::toolchain::Layout,
|
||||
version_root: PathBuf,
|
||||
version: crate::toolchain::Version,
|
||||
resources: crate::toolchain::SignedResources,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
}
|
||||
|
||||
/// Only explicit dependency preparation constructs this capability. Offline
|
||||
/// readers retain InstalledRegistry and cannot request an installation.
|
||||
struct PreparingRegistry(InstalledRegistry);
|
||||
|
||||
impl PreparingRegistry {
|
||||
fn ensure(&self, kind: crate::toolchain::ExtensionKind) -> Result<(), String> {
|
||||
crate::toolchain::ensure_extension(&self.0.layout, &self.0.version, kind, self.0.target)
|
||||
}
|
||||
}
|
||||
|
||||
impl build::Inputs for PreparingRegistry {
|
||||
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> {
|
||||
self.ensure(crate::toolchain::ExtensionKind::Runtime(ecosystem.clone()))?;
|
||||
self.0.runtime(ecosystem)
|
||||
}
|
||||
fn tools(&self, ecosystem: Ecosystem) -> Result<(build::pack::Descriptor, Vec<u8>), String> {
|
||||
self.ensure(crate::toolchain::ExtensionKind::Build(ecosystem.clone()))?;
|
||||
build::Inputs::tools(&self.0, ecosystem)
|
||||
}
|
||||
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
build::Inputs::assets(&self.0)
|
||||
}
|
||||
fn diagnostic(&self, output: &[u8]) {
|
||||
build::Inputs::diagnostic(&self.0, output);
|
||||
}
|
||||
}
|
||||
|
||||
impl build::Inputs for InstalledRegistry {
|
||||
fn diagnostic(&self, output: &[u8]) {
|
||||
if !output.is_empty() {
|
||||
@@ -782,8 +823,9 @@ impl InstalledRegistry {
|
||||
return Err("running Dever core is not the verified managed release".into());
|
||||
}
|
||||
Ok(Self {
|
||||
resources: crate::toolchain::SignedResources::load(&layout, &version)?,
|
||||
layout,
|
||||
version_root: version_root.to_path_buf(),
|
||||
version,
|
||||
target,
|
||||
})
|
||||
}
|
||||
@@ -815,40 +857,18 @@ impl InstalledRegistry {
|
||||
}
|
||||
|
||||
fn signed_artifact(&self, relative: &Path) -> Result<Vec<u8>, String> {
|
||||
let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice(
|
||||
&fs::read(self.version_root.join("manifest.json"))
|
||||
.map_err(|error| error.to_string())?,
|
||||
)
|
||||
.map_err(|error| format!("invalid signed release manifest: {error}"))?;
|
||||
let name = relative.to_str().ok_or("runtime path is not UTF-8")?;
|
||||
let identity = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.path == name)
|
||||
.ok_or_else(|| format!("signed release does not include runtime artifact '{name}'"))?;
|
||||
let bytes = fs::read(self.version_root.join(relative))
|
||||
.map_err(|error| format!("cannot read signed runtime artifact '{name}': {error}"))?;
|
||||
if bytes.len() as u64 != identity.bytes || sha256(&bytes) != identity.sha256 {
|
||||
return Err(format!(
|
||||
"signed runtime artifact '{name}' failed identity verification"
|
||||
));
|
||||
}
|
||||
Ok(bytes)
|
||||
self.resources.read(name)
|
||||
}
|
||||
|
||||
fn sandbox_resources(
|
||||
&self,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<dever_core::native::EmbeddedResource>, String> {
|
||||
let manifest: crate::toolchain::ReleaseManifest = serde_json::from_slice(
|
||||
&fs::read(self.version_root.join("manifest.json"))
|
||||
.map_err(|error| error.to_string())?,
|
||||
)
|
||||
.map_err(|error| format!("invalid signed release manifest: {error}"))?;
|
||||
let prefix = format!("sandbox/{}/", target.platform());
|
||||
let mut resources = Vec::new();
|
||||
let mut files = Vec::new();
|
||||
for artifact in &manifest.artifacts {
|
||||
for artifact in self.resources.artifacts(&prefix)? {
|
||||
let Some(relative) = artifact.path.strip_prefix(&prefix) else {
|
||||
continue;
|
||||
};
|
||||
@@ -896,7 +916,7 @@ impl ArtifactStore for ManagedArtifactStore {
|
||||
bytes: u64,
|
||||
_target: &str,
|
||||
) -> Result<Option<Vec<u8>>, String> {
|
||||
crate::toolchain::artifact_get(&self.layout, sha256, bytes).map(Some)
|
||||
crate::toolchain::artifact_get_optional(&self.layout, sha256, bytes)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1237,13 +1257,28 @@ pub(crate) fn bound_program_workers(
|
||||
program: &dever_core::hir::Program,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<LockedWorker>, String> {
|
||||
let mut workers = program_workers(program)?;
|
||||
bind_workers(program_workers(program)?, target, false)
|
||||
}
|
||||
|
||||
fn bind_workers(
|
||||
mut workers: Vec<LockedWorker>,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
prepare: bool,
|
||||
) -> Result<Vec<LockedWorker>, String> {
|
||||
if workers.iter().all(|worker| worker.ecosystem == "exec") {
|
||||
return Ok(workers);
|
||||
}
|
||||
let managed = InstalledRegistry::current(target)?;
|
||||
for worker in &mut workers {
|
||||
if worker.ecosystem != "exec" {
|
||||
if prepare {
|
||||
crate::toolchain::ensure_extension(
|
||||
&managed.layout,
|
||||
&managed.version,
|
||||
crate::toolchain::ExtensionKind::Runtime(worker.ecosystem.parse()?),
|
||||
target,
|
||||
)?;
|
||||
}
|
||||
worker.runtime = Some(managed.runtime(worker.ecosystem.parse()?)?.0.pack);
|
||||
}
|
||||
}
|
||||
@@ -1876,9 +1911,22 @@ pub fn execute_for_target(
|
||||
let _mutation = ProjectMutation::for_command(command, project_root)?;
|
||||
let path = project_root.join("dever.lock");
|
||||
match command {
|
||||
"install" => {
|
||||
if !specs.is_empty() {
|
||||
return Err(
|
||||
"dever lib install accepts no Lib specs; it restores the existing dever.lock"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
restore::install(project_root, target).map(|report| report.summary())
|
||||
}
|
||||
"add" => {
|
||||
let mut requested = read_declarations(project_root)?;
|
||||
let workers = source_workers(project_root, target)?;
|
||||
let workers = prepare_source_workers_with_packages(
|
||||
project_root,
|
||||
&crate::packages::sources(project_root)?,
|
||||
target,
|
||||
)?;
|
||||
let package_libs = crate::packages::declared_libs(project_root)?;
|
||||
if specs.is_empty()
|
||||
&& requested.is_empty()
|
||||
@@ -1920,7 +1968,11 @@ pub fn execute_for_target(
|
||||
.map(|value| value.parse())
|
||||
.collect::<Result<Vec<LibSpec>, _>>()?
|
||||
};
|
||||
let workers = source_workers(project_root, target)?;
|
||||
let workers = prepare_source_workers_with_packages(
|
||||
project_root,
|
||||
&crate::packages::sources(project_root)?,
|
||||
target,
|
||||
)?;
|
||||
let package_libs = crate::packages::declared_libs(project_root)?;
|
||||
if requested.is_empty() && workers.is_empty() && package_libs.is_empty() {
|
||||
return Err("dever lib update requires declared libs or exact lib specs".into());
|
||||
@@ -2174,6 +2226,29 @@ pub(crate) fn source_workers_with_packages(
|
||||
project_root: &Path,
|
||||
packages: &[dever_core::source::PackageSource],
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<LockedWorker>, String> {
|
||||
bind_workers(
|
||||
unbound_source_workers(project_root, packages)?,
|
||||
target,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn prepare_source_workers_with_packages(
|
||||
project_root: &Path,
|
||||
packages: &[dever_core::source::PackageSource],
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<Vec<LockedWorker>, String> {
|
||||
bind_workers(
|
||||
unbound_source_workers(project_root, packages)?,
|
||||
target,
|
||||
true,
|
||||
)
|
||||
}
|
||||
|
||||
fn unbound_source_workers(
|
||||
project_root: &Path,
|
||||
packages: &[dever_core::source::PackageSource],
|
||||
) -> Result<Vec<LockedWorker>, String> {
|
||||
let source_root = project_root.join("module");
|
||||
if !source_root.exists() && packages.is_empty() {
|
||||
@@ -2193,7 +2268,7 @@ pub(crate) fn source_workers_with_packages(
|
||||
.map(|error| error.render(&sources))
|
||||
.collect::<String>()
|
||||
})?;
|
||||
bound_program_workers(&program, target)
|
||||
program_workers(&program)
|
||||
}
|
||||
|
||||
fn declaration_path(project_root: &Path) -> PathBuf {
|
||||
@@ -2481,6 +2556,16 @@ fn validate_runtime(runtime: &RuntimePack) -> Result<(), String> {
|
||||
}
|
||||
|
||||
fn validate_artifact(artifact: &LockedArtifact) -> Result<(), String> {
|
||||
if let Some(source) = &artifact.source {
|
||||
crate::workers::valid_path(&source.filename)?;
|
||||
if source.filename.contains('/')
|
||||
|| source.locator.is_empty()
|
||||
|| source.locator.len() > 8192
|
||||
|| source.locator.chars().any(char::is_control)
|
||||
{
|
||||
return Err("invalid locked registry source locator or filename".into());
|
||||
}
|
||||
}
|
||||
if artifact.target.is_empty() || artifact.bytes == 0 {
|
||||
return Err("artifact target/size is invalid".into());
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ pub struct Receipt {
|
||||
pub frontend: String,
|
||||
pub python: Option<PythonSystem>,
|
||||
pub dynamic_requires: Vec<String>,
|
||||
pub dependencies: Box<LockFile>,
|
||||
pub inputs: Box<LockFile>,
|
||||
pub config_settings: BTreeMap<String, serde_json::Value>,
|
||||
pub output: LockedArtifact,
|
||||
}
|
||||
@@ -90,6 +90,29 @@ pub struct Session<'a> {
|
||||
}
|
||||
|
||||
impl<'a> Session<'a> {
|
||||
pub(crate) fn replay(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn super::ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
if receipt.frontend != frontend(&receipt.ecosystem)? {
|
||||
return Err("locked build frontend differs from this toolchain".into());
|
||||
}
|
||||
let bytes = match receipt.ecosystem {
|
||||
Ecosystem::Pip => self.replay_python(receipt, store)?,
|
||||
Ecosystem::Npm => self.replay_npm(receipt, store)?,
|
||||
Ecosystem::Go => return Err("Go source build receipts are unsupported".into()),
|
||||
};
|
||||
if bytes.len() as u64 != receipt.output.bytes
|
||||
|| super::sha256(&bytes) != receipt.output.sha256
|
||||
{
|
||||
return Err("fixed build replay is not byte reproducible: output differs from dever.lock; lock was not changed".into());
|
||||
}
|
||||
if store.publish(&bytes, &receipt.output.target)? != receipt.output.sha256 {
|
||||
return Err("restored build cache identity mismatch".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn new(inputs: &'a dyn Inputs) -> Self {
|
||||
Self {
|
||||
inputs,
|
||||
@@ -229,6 +252,9 @@ pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
}
|
||||
|
||||
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if lock.format != super::LOCK_FORMAT || lock.toolchain != super::TOOLCHAIN {
|
||||
return Err("build input lock format or toolchain differs from this release".into());
|
||||
}
|
||||
if depth > 16 {
|
||||
return Err("build dependency depth exceeds limit".into());
|
||||
}
|
||||
@@ -361,7 +387,6 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
&& (receipt.python.is_some()
|
||||
|| receipt.python_markers.is_some()
|
||||
|| !receipt.dynamic_requires.is_empty()
|
||||
|| !receipt.dependencies.libs.is_empty()
|
||||
|| receipt.graph.is_none())
|
||||
{
|
||||
return Err("npm build receipt contains Python backend state".into());
|
||||
@@ -372,20 +397,50 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if let Some(system) = &receipt.python {
|
||||
validate_requirements(receipt, system)?;
|
||||
}
|
||||
if !receipt.dependencies.workers.is_empty()
|
||||
|| !receipt.dependencies.packages.is_empty()
|
||||
if !receipt.inputs.workers.is_empty()
|
||||
|| !receipt.inputs.packages.is_empty()
|
||||
|| receipt
|
||||
.dependencies
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
|
||||
.any(|lib| lib.spec.ecosystem != receipt.ecosystem)
|
||||
{
|
||||
return Err("build dependency closure is not a separate Python environment".into());
|
||||
return Err("build input closure contains unrelated environments".into());
|
||||
}
|
||||
doctor_at(&receipt.dependencies, depth + 1)?;
|
||||
if receipt.output.source.is_some()
|
||||
|| receipt.sources.iter().any(|source| {
|
||||
source
|
||||
.source
|
||||
.as_ref()
|
||||
.is_none_or(|origin| origin.ecosystem != receipt.ecosystem)
|
||||
})
|
||||
{
|
||||
return Err("build receipt lacks exact registry source identity".into());
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Npm {
|
||||
let mut sources = receipt
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| lib.artifacts.clone())
|
||||
.collect::<Vec<_>>();
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
if sources != receipt.sources
|
||||
|| receipt.inputs.npm.len() != 1
|
||||
|| receipt.inputs.npm[0].roots != receipt.roots
|
||||
|| receipt.inputs.npm[0].build.is_some()
|
||||
|| !receipt.inputs.builds.is_empty()
|
||||
{
|
||||
return Err("npm build input graph differs from its original sources".into());
|
||||
}
|
||||
} else if receipt.sources.len() != 1 || !receipt.inputs.npm.is_empty() {
|
||||
return Err("Python build receipt has invalid source inputs".into());
|
||||
}
|
||||
doctor_at(&receipt.inputs, depth + 1)?;
|
||||
// Full graph checks belong to the ordinary lock owner; recursion is
|
||||
// bounded above before invoking that owner on a dependency lock.
|
||||
let mut dependencies = (*receipt.dependencies).clone();
|
||||
let mut dependencies = (*receipt.inputs).clone();
|
||||
dependencies.builds.clear();
|
||||
for lib in &mut dependencies.libs {
|
||||
lib.build = None;
|
||||
@@ -447,7 +502,7 @@ fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(),
|
||||
let document = serde_json::json!({"info":{"requires_dist":requirements}});
|
||||
let expected = super::python_dependencies(&document, markers, &[])?;
|
||||
let libraries = receipt
|
||||
.dependencies
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| (lib.spec.name.as_str(), lib))
|
||||
|
||||
@@ -12,6 +12,43 @@ mod environment;
|
||||
use environment::{Environment, Output};
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn replay_npm(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn crate::libs::ArtifactStore,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let (runtime, _) = self.inputs.runtime(Ecosystem::Npm)?;
|
||||
if runtime.pack != receipt.runtime {
|
||||
return Err("locked npm build runtime changed".into());
|
||||
}
|
||||
let environment = Environment::new(self.inputs, &runtime)?;
|
||||
if environment.descriptor != receipt.tools
|
||||
|| Some(&environment.python_runtime) != receipt.auxiliary_runtime.as_ref()
|
||||
{
|
||||
return Err("locked npm build tools or auxiliary runtime changed".into());
|
||||
}
|
||||
let mut graph = receipt
|
||||
.inputs
|
||||
.npm
|
||||
.first()
|
||||
.ok_or("npm build input graph missing")?
|
||||
.clone();
|
||||
let (files, executables) = installation_inputs(store, &receipt.inputs, &graph)?;
|
||||
let output = environment.run(self.inputs, &graph, &files, &executables)?;
|
||||
prune_failed(&mut graph, &output.failed)?;
|
||||
npm::validate_built_files(&graph, &output.files)?;
|
||||
if Some(&graph) != receipt.graph.as_ref()
|
||||
|| output.native != receipt.native_entries
|
||||
|| output.rejections != receipt.native_rejections
|
||||
{
|
||||
return Err(
|
||||
"npm fixed build replay changed graph, optional outcomes or native admission"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
archive(&output)
|
||||
}
|
||||
|
||||
pub(crate) fn npm(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
@@ -21,26 +58,7 @@ impl Session<'_> {
|
||||
) -> Result<(), String> {
|
||||
let mut lock = LockFile::new(libraries.clone())?;
|
||||
lock.npm.push(graph.clone());
|
||||
let archives = libraries
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
resolver.store.verify_exact(
|
||||
&artifact.sha256,
|
||||
artifact.bytes,
|
||||
&artifact.target,
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, &lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
let (files, executables) = installation_inputs(resolver.store, &lock, graph)?;
|
||||
if !npm::requires_build(&files)? {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -66,7 +84,6 @@ impl Session<'_> {
|
||||
retain_archives(libraries, graph);
|
||||
return Ok(());
|
||||
}
|
||||
let executables = npm::build_executables(graph, &archives)?;
|
||||
let environment = Environment::new(self.inputs, runtime)?;
|
||||
let output = environment.run(self.inputs, graph, &files, &executables)?;
|
||||
prune_failed(graph, &output.failed)?;
|
||||
@@ -82,6 +99,7 @@ impl Session<'_> {
|
||||
path: format!("build/npm/{digest}/installation.tgz"),
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: digest,
|
||||
source: None,
|
||||
};
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
@@ -97,7 +115,7 @@ impl Session<'_> {
|
||||
frontend: super::frontend(&Ecosystem::Npm)?,
|
||||
python: None,
|
||||
dynamic_requires: vec![],
|
||||
dependencies: Box::new(LockFile::new(vec![])?),
|
||||
inputs: Box::new(lock),
|
||||
config_settings: BTreeMap::new(),
|
||||
output: output_artifact,
|
||||
};
|
||||
@@ -114,6 +132,33 @@ impl Session<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
type InstallationInputs = (Vec<(String, Vec<u8>)>, BTreeSet<String>);
|
||||
|
||||
fn installation_inputs(
|
||||
store: &dyn crate::libs::ArtifactStore,
|
||||
lock: &LockFile,
|
||||
graph: &npm::Environment,
|
||||
) -> Result<InstallationInputs, String> {
|
||||
let archives = lock
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
Ok((files, npm::build_executables(graph, &archives)?))
|
||||
}
|
||||
|
||||
fn archive(output: &Output) -> Result<Vec<u8>, String> {
|
||||
let encoder = flate2::GzBuilder::new()
|
||||
.mtime(0)
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
|
||||
use crate::libs::{
|
||||
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
|
||||
sha256,
|
||||
ArtifactStore, Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime,
|
||||
RegistrySource, archive_files, sha256,
|
||||
};
|
||||
use crate::workers::{python_wheel::Wheel, runtime};
|
||||
use serde_json::{Value, json};
|
||||
@@ -23,6 +23,12 @@ struct Environment {
|
||||
descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
struct Execution<'a> {
|
||||
store: &'a dyn ArtifactStore,
|
||||
runtime: &'a RegistryRuntime,
|
||||
inputs: &'a dyn super::Inputs,
|
||||
}
|
||||
|
||||
struct Active<'a> {
|
||||
session: &'a Session<'a>,
|
||||
key: String,
|
||||
@@ -34,12 +40,69 @@ impl Drop for Active<'_> {
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn replay_python(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let (runtime, _) = self.inputs.runtime(Ecosystem::Pip)?;
|
||||
if runtime.pack != receipt.runtime || runtime.python_markers != receipt.python_markers {
|
||||
return Err("locked Python build runtime or marker policy changed".into());
|
||||
}
|
||||
let environment = Environment::new(self, &runtime)?;
|
||||
let execution = Execution {
|
||||
store,
|
||||
runtime: &runtime,
|
||||
inputs: self.inputs,
|
||||
};
|
||||
if environment.descriptor != receipt.tools {
|
||||
return Err("locked Python build tools changed".into());
|
||||
}
|
||||
let artifact = receipt
|
||||
.sources
|
||||
.first()
|
||||
.ok_or("Python build source missing")?;
|
||||
let origin = artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or("Python build source locator missing")?;
|
||||
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let source = source_files(&origin.filename, &bytes)?;
|
||||
let system = receipt
|
||||
.python
|
||||
.as_ref()
|
||||
.ok_or("Python build backend missing")?;
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected = environment.run(
|
||||
&execution,
|
||||
&empty,
|
||||
&source,
|
||||
json!({"phase":"inspect"}),
|
||||
None,
|
||||
)?;
|
||||
let actual: PythonSystem =
|
||||
serde_json::from_value(inspected.response).map_err(|error| error.to_string())?;
|
||||
if &actual != system {
|
||||
return Err("locked Python build backend configuration changed".into());
|
||||
}
|
||||
// The locked closure is final. Replaying never asks the resolver to
|
||||
// discover or select static/dynamic build requirements again.
|
||||
let (_, bytes) = environment.wheel(
|
||||
&execution,
|
||||
&receipt.inputs,
|
||||
&source,
|
||||
system,
|
||||
receipt.roots.first().ok_or("Python build root missing")?,
|
||||
)?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn python(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
spec: &LibSpec,
|
||||
runtime: &RegistryRuntime,
|
||||
filename: &str,
|
||||
source_origin: RegistrySource,
|
||||
source: &[u8],
|
||||
) -> Result<Product, String> {
|
||||
let key = format!(
|
||||
@@ -62,11 +125,21 @@ impl Session<'_> {
|
||||
session: self,
|
||||
key: key.clone(),
|
||||
};
|
||||
let sources = source_files(filename, source)?;
|
||||
let sources = source_files(&source_origin.filename, source)?;
|
||||
let environment = Environment::new(self, runtime)?;
|
||||
let execution = Execution {
|
||||
store: resolver.store,
|
||||
runtime,
|
||||
inputs: self.inputs,
|
||||
};
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected =
|
||||
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
|
||||
let inspected = environment.run(
|
||||
&execution,
|
||||
&empty,
|
||||
&sources,
|
||||
json!({"phase":"inspect"}),
|
||||
None,
|
||||
)?;
|
||||
let system: PythonSystem = serde_json::from_value(inspected.response)
|
||||
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
|
||||
let mut requirements = system.requires.clone();
|
||||
@@ -75,7 +148,7 @@ impl Session<'_> {
|
||||
let mut discovered = BTreeSet::new();
|
||||
for _ in 0..8 {
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&execution,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"requires","system":system}),
|
||||
@@ -99,38 +172,8 @@ impl Session<'_> {
|
||||
dependencies = next;
|
||||
}
|
||||
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
|
||||
let metadata = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
let (filename, bytes) =
|
||||
environment.wheel(&execution, &dependencies, &sources, &system, spec)?;
|
||||
let output = artifact(
|
||||
runtime,
|
||||
format!(
|
||||
@@ -140,11 +183,12 @@ impl Session<'_> {
|
||||
),
|
||||
&bytes,
|
||||
);
|
||||
let source_artifact = artifact(
|
||||
let mut source_artifact = artifact(
|
||||
runtime,
|
||||
format!("build/source/{}/source", sha256(source)),
|
||||
source,
|
||||
);
|
||||
source_artifact.source = Some(source_origin);
|
||||
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
|
||||
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
|
||||
return Err("build artifact cache identity mismatch".into());
|
||||
@@ -165,13 +209,13 @@ impl Session<'_> {
|
||||
frontend: super::frontend(&Ecosystem::Pip)?,
|
||||
python: Some(system),
|
||||
dynamic_requires: dynamic,
|
||||
dependencies: Box::new(dependencies),
|
||||
inputs: Box::new(dependencies),
|
||||
config_settings: BTreeMap::new(),
|
||||
output,
|
||||
};
|
||||
let product = Product {
|
||||
bytes,
|
||||
filename: filename.into(),
|
||||
filename,
|
||||
receipt,
|
||||
};
|
||||
self.remember(key, product.clone())?;
|
||||
@@ -185,6 +229,7 @@ fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArti
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,6 +281,54 @@ struct ResultFiles {
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
fn wheel(
|
||||
&self,
|
||||
execution: &Execution<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
system: &PythonSystem,
|
||||
spec: &LibSpec,
|
||||
) -> Result<(String, Vec<u8>), String> {
|
||||
let metadata = self.run(
|
||||
execution,
|
||||
dependencies,
|
||||
source,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = self.run(
|
||||
execution,
|
||||
dependencies,
|
||||
source,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(
|
||||
spec,
|
||||
Some(filename),
|
||||
&execution.runtime.python_wheel_tags,
|
||||
&bytes,
|
||||
)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
Ok((filename.into(), bytes))
|
||||
}
|
||||
|
||||
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
|
||||
super::require_execution_target(&requested.target, "Python source build")?;
|
||||
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
|
||||
@@ -291,12 +384,17 @@ impl Environment {
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
execution: &Execution<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
mut request: Value,
|
||||
prepared: Option<&BTreeMap<String, Vec<u8>>>,
|
||||
) -> Result<ResultFiles, String> {
|
||||
let Execution {
|
||||
store,
|
||||
runtime,
|
||||
inputs,
|
||||
} = execution;
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
@@ -322,10 +420,6 @@ impl Environment {
|
||||
}
|
||||
}
|
||||
let mut installed = BTreeMap::<String, Vec<u8>>::new();
|
||||
let runtime = resolver
|
||||
.runtimes
|
||||
.get(&Ecosystem::Pip)
|
||||
.ok_or("build dependency runtime is missing")?;
|
||||
let mut distributions = BTreeSet::new();
|
||||
for lib in &dependencies.libs {
|
||||
super::validate_python_policy(
|
||||
@@ -342,10 +436,7 @@ impl Environment {
|
||||
);
|
||||
}
|
||||
let artifact = &lib.artifacts[0];
|
||||
let bytes =
|
||||
resolver
|
||||
.store
|
||||
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
|
||||
wheel.validate_dependencies(
|
||||
lib,
|
||||
@@ -432,9 +523,7 @@ impl Environment {
|
||||
"isolated Python build hook",
|
||||
&work,
|
||||
)?;
|
||||
if let Some(session) = resolver.build {
|
||||
session.inputs.diagnostic(&stage.diagnostics()?);
|
||||
}
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
let outputs = output_files(&work.join("output"))?;
|
||||
let response = serde_json::from_slice(
|
||||
outputs
|
||||
|
||||
@@ -10,7 +10,7 @@ pub(crate) struct ProjectMutation {
|
||||
|
||||
impl ProjectMutation {
|
||||
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
|
||||
if !matches!(command, "add" | "update" | "remove") {
|
||||
if !matches!(command, "add" | "update" | "remove" | "install") {
|
||||
return Ok(None);
|
||||
}
|
||||
let directory = fs::symlink_metadata(root)
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod sumdb;
|
||||
|
||||
use super::{
|
||||
ArtifactStore, Ecosystem, LibResolver, LibSpec, LockFile, LockedArtifact, LockedDependency,
|
||||
LockedLib, ProviderManifest, RuntimePack, sha256, validate_artifact,
|
||||
LockedLib, ProviderManifest, RegistrySource, RuntimePack, sha256, validate_artifact,
|
||||
};
|
||||
|
||||
const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024;
|
||||
@@ -553,7 +553,7 @@ impl RegistryResolver<'_> {
|
||||
runtime,
|
||||
"whl",
|
||||
&candidate.bytes,
|
||||
"pypi-wheel-v1",
|
||||
candidate.source.clone(),
|
||||
)?;
|
||||
lib.build = candidate.build.clone();
|
||||
distributions.insert(base.to_owned(), lib.clone());
|
||||
@@ -925,6 +925,11 @@ impl RegistryResolver<'_> {
|
||||
}
|
||||
Ok(Some(PythonCandidate {
|
||||
build: None,
|
||||
source: Some(RegistrySource {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
locator: url.into(),
|
||||
filename: field(file, "filename")?.into(),
|
||||
}),
|
||||
bytes,
|
||||
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
|
||||
}))
|
||||
@@ -965,7 +970,17 @@ impl RegistryResolver<'_> {
|
||||
if sha256(&bytes) != expected {
|
||||
return Err("PyPI source SHA-256 mismatch".into());
|
||||
}
|
||||
let product = session.python(self, spec, runtime, field(file, "filename")?, &bytes)?;
|
||||
let product = session.python(
|
||||
self,
|
||||
spec,
|
||||
runtime,
|
||||
RegistrySource {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
locator: field(file, "url")?.into(),
|
||||
filename: field(file, "filename")?.into(),
|
||||
},
|
||||
&bytes,
|
||||
)?;
|
||||
let wheel = crate::workers::python_wheel::Wheel::parse(
|
||||
spec,
|
||||
Some(&product.filename),
|
||||
@@ -980,6 +995,7 @@ impl RegistryResolver<'_> {
|
||||
}
|
||||
Ok(Some(PythonCandidate {
|
||||
bytes: product.bytes,
|
||||
source: None,
|
||||
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
|
||||
build: Some(product.receipt.identity()?),
|
||||
}))
|
||||
@@ -1012,7 +1028,7 @@ impl RegistryResolver<'_> {
|
||||
runtime: &RegistryRuntime,
|
||||
extension: &str,
|
||||
bytes: &[u8],
|
||||
schema: &str,
|
||||
source: Option<RegistrySource>,
|
||||
) -> Result<LockedLib, String> {
|
||||
let files = archive_files(extension, bytes)?;
|
||||
if spec.ecosystem == Ecosystem::Go {
|
||||
@@ -1035,6 +1051,7 @@ impl RegistryResolver<'_> {
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
source,
|
||||
};
|
||||
validate_artifact(&artifact)?;
|
||||
let published = self.store.publish(bytes, &runtime.target)?;
|
||||
@@ -1052,7 +1069,12 @@ impl RegistryResolver<'_> {
|
||||
dependencies,
|
||||
runtime: runtime.pack.clone(),
|
||||
artifacts: vec![artifact],
|
||||
schema: schema.into(),
|
||||
schema: match spec.ecosystem {
|
||||
Ecosystem::Pip => "pypi-wheel-v1",
|
||||
Ecosystem::Npm => "npm-instances-v1",
|
||||
Ecosystem::Go => "go-sumdb-v1",
|
||||
}
|
||||
.into(),
|
||||
build: None,
|
||||
})
|
||||
}
|
||||
@@ -1126,7 +1148,22 @@ impl RegistryResolver<'_> {
|
||||
MAX_ARTIFACT_BYTES,
|
||||
)?;
|
||||
sumdb.verify_zip(name, version, &zip)?;
|
||||
self.lock_archive(&spec, dependencies, runtime, "zip", &zip, "go-sumdb-v1")
|
||||
self.lock_archive(
|
||||
&spec,
|
||||
dependencies,
|
||||
runtime,
|
||||
"zip",
|
||||
&zip,
|
||||
Some(RegistrySource {
|
||||
ecosystem: Ecosystem::Go,
|
||||
locator: format!(
|
||||
"/{}/@v/v{}.zip",
|
||||
go_path(name),
|
||||
version.trim_start_matches('v')
|
||||
),
|
||||
filename: format!("v{}.zip", version.trim_start_matches('v')),
|
||||
}),
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
return Ok((libs, sumdb.finish()));
|
||||
@@ -1179,6 +1216,7 @@ struct PythonCandidate {
|
||||
bytes: Vec<u8>,
|
||||
info: Value,
|
||||
build: Option<String>,
|
||||
source: Option<RegistrySource>,
|
||||
}
|
||||
|
||||
struct SearchState<'a> {
|
||||
|
||||
@@ -128,6 +128,18 @@ impl Catalog<'_, '_> {
|
||||
"npm archive cache exceeds its 256 MiB/package budget".into(),
|
||||
));
|
||||
}
|
||||
let locator = document
|
||||
.get("dist")
|
||||
.and_then(|dist| dist.get("tarball"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or_else(|| SearchError::Fatal("npm tarball locator missing".into()))?;
|
||||
let filename = locator
|
||||
.split(['?', '#'])
|
||||
.next()
|
||||
.unwrap_or(locator)
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.ok_or_else(|| SearchError::Fatal("npm tarball filename missing".into()))?;
|
||||
let lib = self
|
||||
.resolver
|
||||
.lock_archive(
|
||||
@@ -136,7 +148,11 @@ impl Catalog<'_, '_> {
|
||||
self.runtime,
|
||||
"tgz",
|
||||
&bytes,
|
||||
"npm-instances-v1",
|
||||
Some(crate::libs::RegistrySource {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
locator: locator.into(),
|
||||
filename: filename.into(),
|
||||
}),
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.archives.insert(spec.clone(), (lib, archive));
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
//! Exact restoration is a preparation operation, never a dependency resolver.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use super::{
|
||||
ArtifactStore, FixtureRegistry, LockFile, LockedArtifact, RegistryTransport, build, sumdb,
|
||||
};
|
||||
|
||||
pub(super) fn install(
|
||||
project_root: &Path,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<super::PreparationReport, String> {
|
||||
super::reject_declaration_symlinks(project_root)?;
|
||||
let lock_bytes = fs::read(project_root.join("dever.lock"))
|
||||
.map_err(|error| format!("dever lib install requires an existing dever.lock: {error}"))?;
|
||||
let setting_bytes = setting_snapshot(project_root)?;
|
||||
let lock = LockFile::decode(&lock_bytes)?;
|
||||
super::doctor(&lock)?;
|
||||
validate_target(&lock, target.platform())?;
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
if lock.packages.is_empty()
|
||||
&& lock
|
||||
.libs
|
||||
.iter()
|
||||
.all(|lib| fixtures.package(&lib.spec).is_some())
|
||||
&& lock.workers.iter().all(|worker| worker.ecosystem == "exec")
|
||||
{
|
||||
let store = super::FixtureArtifactStore::default();
|
||||
validate_project(project_root, &lock, &store, target, false)?;
|
||||
restore_fixtures(&lock, &store)?;
|
||||
} else {
|
||||
let inputs = super::PreparingRegistry(super::InstalledRegistry::current(target)?);
|
||||
let store = super::ManagedArtifactStore {
|
||||
layout: inputs.0.layout.clone(),
|
||||
};
|
||||
crate::packages::restore_locked(project_root, &lock, &store)?;
|
||||
validate_project(project_root, &lock, &store, target, true)?;
|
||||
for worker in lock
|
||||
.workers
|
||||
.iter()
|
||||
.filter(|worker| worker.ecosystem != "exec")
|
||||
{
|
||||
let (runtime, _) = build::Inputs::runtime(&inputs, worker.ecosystem.parse()?)?;
|
||||
if worker.runtime.as_ref() != Some(&runtime.pack) {
|
||||
return Err("locked Worker runtime differs from signed release".into());
|
||||
}
|
||||
}
|
||||
if target != crate::toolchain::BuildTarget::host()? && !lock.workers.is_empty() {
|
||||
inputs.ensure(crate::toolchain::ExtensionKind::Target)?;
|
||||
}
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
&super::HttpRegistry::official(),
|
||||
&store,
|
||||
&inputs,
|
||||
target.platform(),
|
||||
&sumdb::Verifier::official(),
|
||||
)?;
|
||||
super::embedded_resources_for_target(project_root, target)?;
|
||||
}
|
||||
if fs::read(project_root.join("dever.lock")).map_err(|error| error.to_string())? != lock_bytes
|
||||
|| setting_snapshot(project_root)? != setting_bytes
|
||||
{
|
||||
return Err("project setting or lock changed during exact restoration".into());
|
||||
}
|
||||
let artifacts = lock
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| &lib.artifacts)
|
||||
.chain(build::npm_outputs(&lock).map(|receipt| &receipt.output))
|
||||
.collect::<BTreeSet<_>>();
|
||||
Ok(super::PreparationReport {
|
||||
libs: lock.libs.len(),
|
||||
artifacts: artifacts.len(),
|
||||
runtimes: lock
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| &lib.runtime)
|
||||
.chain(
|
||||
lock.workers
|
||||
.iter()
|
||||
.filter_map(|worker| worker.runtime.as_ref()),
|
||||
)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len(),
|
||||
artifact_bytes: artifacts.iter().map(|artifact| artifact.bytes).sum(),
|
||||
})
|
||||
}
|
||||
|
||||
fn setting_snapshot(project_root: &Path) -> Result<Option<Vec<u8>>, String> {
|
||||
match fs::read(project_root.join("config/setting.json")) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(format!("cannot read project setting snapshot: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_project(
|
||||
project_root: &Path,
|
||||
lock: &LockFile,
|
||||
store: &dyn ArtifactStore,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
prepare: bool,
|
||||
) -> Result<(), String> {
|
||||
let sources = crate::packages::sources_with(project_root, store)?;
|
||||
let workers = super::unbound_source_workers(project_root, &sources)?;
|
||||
let mut unbound = lock.clone();
|
||||
for worker in &mut unbound.workers {
|
||||
worker.runtime = None;
|
||||
}
|
||||
super::validate_worker_bindings(&unbound, &workers)?;
|
||||
let mut roots = super::read_declarations(project_root)?;
|
||||
roots.extend(crate::packages::declared_libs_with(project_root, store)?);
|
||||
if super::retain_locked_libs(lock, &roots, lock.workers.clone())? != *lock {
|
||||
return Err("dever.lock contains Lib entries or environments outside the declared closure; run 'dever lib update <project-root>'".into());
|
||||
}
|
||||
let workers = super::bind_workers(workers, target, prepare)?;
|
||||
super::validate_worker_bindings(lock, &workers)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_target(lock: &LockFile, target: &str) -> Result<(), String> {
|
||||
for artifact in lock.libs.iter().flat_map(|lib| &lib.artifacts).chain(
|
||||
lock.builds
|
||||
.iter()
|
||||
.flat_map(|receipt| receipt.sources.iter().chain([&receipt.output])),
|
||||
) {
|
||||
if artifact.target != target && artifact.target != "host" {
|
||||
return Err(format!(
|
||||
"locked artifact targets {}, not {target}; install never switches lock targets",
|
||||
artifact.target
|
||||
));
|
||||
}
|
||||
}
|
||||
for receipt in &lock.builds {
|
||||
validate_target(&receipt.inputs, target)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_fixtures(lock: &LockFile, store: &dyn ArtifactStore) -> Result<(), String> {
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
for lib in &lock.libs {
|
||||
if let Some(package) = fixtures.package(&lib.spec) {
|
||||
if package.locked() != *lib {
|
||||
return Err("locked fixture metadata differs from builtin package".into());
|
||||
}
|
||||
for artifact in &package.artifacts {
|
||||
let expected = super::sha256(&artifact.bytes);
|
||||
if store.publish(&artifact.bytes, &artifact.target)? != expected {
|
||||
return Err("fixture artifact cache identity mismatch".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shared by production preparation and isolated registry fixtures. Every
|
||||
/// network request names the recorded archive; no metadata/version query runs.
|
||||
pub fn restore_artifacts(
|
||||
lock: &LockFile,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
inputs: &dyn build::Inputs,
|
||||
target: &str,
|
||||
verifier: &sumdb::Verifier,
|
||||
) -> Result<(), String> {
|
||||
super::doctor_with_go_verifier(lock, verifier)?;
|
||||
validate_target(lock, target)?;
|
||||
let session = build::Session::new(inputs);
|
||||
restore_at(lock, transport, store, inputs, &session)?;
|
||||
super::verify_locked_artifacts_with_go_verifier(lock, store, target, verifier)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_at(
|
||||
lock: &LockFile,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
inputs: &dyn build::Inputs,
|
||||
session: &build::Session<'_>,
|
||||
) -> Result<(), String> {
|
||||
restore_fixtures(lock, store)?;
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
for lib in &lock.libs {
|
||||
if fixtures.package(&lib.spec).is_some() {
|
||||
continue;
|
||||
}
|
||||
let (runtime, _) = inputs.runtime(lib.spec.ecosystem.clone())?;
|
||||
if runtime.pack != lib.runtime
|
||||
|| lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.target != runtime.target)
|
||||
{
|
||||
return Err(format!(
|
||||
"{} locked runtime differs from signed release",
|
||||
lib.spec.key()
|
||||
));
|
||||
}
|
||||
if let Some(markers) = &runtime.python_markers {
|
||||
build::validate_python_policy(lock, lib, markers)?;
|
||||
}
|
||||
for artifact in &lib.artifacts {
|
||||
if artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| source.ecosystem != lib.spec.ecosystem)
|
||||
{
|
||||
return Err("locked registry source ecosystem differs from its Lib".into());
|
||||
}
|
||||
if artifact.source.is_some() {
|
||||
restore_archive(artifact, transport, store)?;
|
||||
} else if lib.build.is_none() {
|
||||
return Err(format!("{} lacks an exact registry source", lib.spec.key()));
|
||||
}
|
||||
}
|
||||
}
|
||||
for receipt in &lock.builds {
|
||||
if cached(&receipt.output, store)? {
|
||||
continue;
|
||||
}
|
||||
restore_at(&receipt.inputs, transport, store, inputs, session)?;
|
||||
for source in &receipt.sources {
|
||||
restore_archive(source, transport, store)?;
|
||||
}
|
||||
session.replay(receipt, store)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cached(artifact: &LockedArtifact, store: &dyn ArtifactStore) -> Result<bool, String> {
|
||||
match store.get_exact(&artifact.sha256, artifact.bytes, &artifact.target)? {
|
||||
Some(bytes) => {
|
||||
validate_bytes(artifact, &bytes)?;
|
||||
Ok(true)
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bytes(artifact: &LockedArtifact, bytes: &[u8]) -> Result<(), String> {
|
||||
if bytes.len() as u64 != artifact.bytes || super::sha256(bytes) != artifact.sha256 {
|
||||
return Err(format!(
|
||||
"locked archive '{}' failed exact byte/hash verification",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_archive(
|
||||
artifact: &LockedArtifact,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
if cached(artifact, store)? {
|
||||
return Ok(());
|
||||
}
|
||||
let source = artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or("locked archive source is absent")?;
|
||||
let limit =
|
||||
usize::try_from(artifact.bytes).map_err(|_| "locked archive exceeds address space")?;
|
||||
if limit > 64 * 1024 * 1024 {
|
||||
return Err("locked registry archive exceeds byte budget".into());
|
||||
}
|
||||
let bytes = transport.get(source.ecosystem.clone(), &source.locator, limit)?;
|
||||
validate_bytes(artifact, &bytes)?;
|
||||
if store.publish(&bytes, &artifact.target)? != artifact.sha256 {
|
||||
return Err("restored archive cache identity mismatch".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -18,7 +18,7 @@ mod worker;
|
||||
#[path = "../../../test/dever-cli-tests/build_arguments.rs"]
|
||||
mod build_arguments;
|
||||
|
||||
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
|
||||
const USAGE: &str = "Usage:\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever run <project-root> --tenant <positive-tenant-id> -- <component>.<domain>.<cmd> '<json-object>'\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant migrate <project-root> <positive-tenant-id>\n dever tenant owner <project-root> <positive-tenant-id> <site> <positive-user-id>\n dever tenant component enable <project-root> <positive-tenant-id> <component>\n dever tenant component disable <project-root> <positive-tenant-id> <component>\n dever lib add|install|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n\nApplications use API, Job, and CMD declarations under module/ and config/setting.json. Native builds and tests are offline.";
|
||||
|
||||
enum Action {
|
||||
Check,
|
||||
@@ -435,7 +435,7 @@ fn arguments_for(arguments: &[OsString]) -> Result<(PathBuf, Action), String> {
|
||||
if command == "lib"
|
||||
&& matches!(
|
||||
operation.to_str(),
|
||||
Some("add" | "list" | "update" | "remove" | "doctor")
|
||||
Some("add" | "install" | "list" | "update" | "remove" | "doctor")
|
||||
) =>
|
||||
{
|
||||
let command = operation.to_str().unwrap().to_owned();
|
||||
|
||||
@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use url::Url;
|
||||
|
||||
use crate::libs::{self, ArtifactStore, LibSpec};
|
||||
use crate::libs::{self, ArtifactStore, LibSpec, LockFile};
|
||||
|
||||
const INDEX_FORMAT: &str = "dever-package-index-v1";
|
||||
const MANIFEST_FORMAT: &str = "dever-package-v1";
|
||||
@@ -406,6 +406,64 @@ fn settings(project_root: &Path) -> Result<(String, Vec<PackageRequest>), String
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn restore_locked(
|
||||
project_root: &Path,
|
||||
lock: &LockFile,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
let (origin, roots) = settings(project_root)?;
|
||||
validate_locked_roots(&roots, &lock.packages)?;
|
||||
if lock.packages.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
restore_locked_with(
|
||||
project_root,
|
||||
lock,
|
||||
&HttpPackageTransport::new(&origin)?,
|
||||
store,
|
||||
)
|
||||
}
|
||||
|
||||
/// Exact Package archives are restored before source/Worker checking. No index
|
||||
/// is consulted: both dependency versions and archive identities are locked.
|
||||
pub fn restore_locked_with(
|
||||
project_root: &Path,
|
||||
lock: &LockFile,
|
||||
transport: &dyn PackageTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
validate_locked_roots(&settings(project_root)?.1, &lock.packages)?;
|
||||
for package in &lock.packages {
|
||||
let cached = store.get_exact(&package.sha256, package.bytes, "package")?;
|
||||
let missing = cached.is_none();
|
||||
let bytes = match cached {
|
||||
Some(bytes) => bytes,
|
||||
None => transport.get(
|
||||
&format!("/v1/packages/{}/{}.zip", package.name, package.version),
|
||||
package.bytes as usize,
|
||||
)?,
|
||||
};
|
||||
if bytes.len() as u64 != package.bytes || digest(&bytes) != package.sha256 {
|
||||
return Err(format!(
|
||||
"Package {}@{} differs from dever.lock",
|
||||
package.name, package.version
|
||||
));
|
||||
}
|
||||
let archive = manifest(&bytes, &package.name, &package.version)?;
|
||||
if archive.manifest_sha256 != package.manifest_sha256 {
|
||||
return Err(format!(
|
||||
"Package {} manifest differs from dever.lock",
|
||||
package.name
|
||||
));
|
||||
}
|
||||
if missing && store.publish(&bytes, "package")? != package.sha256 {
|
||||
return Err("restored Package cache identity mismatch".into());
|
||||
}
|
||||
}
|
||||
owned_files_with(project_root, store)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct SelectedVersion {
|
||||
index: IndexVersion,
|
||||
@@ -1169,7 +1227,12 @@ fn execute_locked(
|
||||
resolve_with(&roots, transport, store)?
|
||||
};
|
||||
check_project_collisions(project_root, &closure.packages, store)?;
|
||||
let workers = libs::source_workers_with_packages(
|
||||
let worker_loader = if command == "remove" {
|
||||
libs::source_workers_with_packages
|
||||
} else {
|
||||
libs::prepare_source_workers_with_packages
|
||||
};
|
||||
let workers = worker_loader(
|
||||
project_root,
|
||||
&closure.sources,
|
||||
crate::toolchain::BuildTarget::host()?,
|
||||
|
||||
@@ -235,14 +235,33 @@ impl CacheStore {
|
||||
}
|
||||
|
||||
pub fn artifact(&self, expected: &ArtifactReceipt) -> Result<StoredArtifact, String> {
|
||||
self.artifact_optional(expected)?
|
||||
.ok_or_else(|| "shared artifact unavailable: artifact is not cached".into())
|
||||
}
|
||||
|
||||
pub fn artifact_optional(
|
||||
&self,
|
||||
expected: &ArtifactReceipt,
|
||||
) -> Result<Option<StoredArtifact>, String> {
|
||||
expected.validate()?;
|
||||
let operation = CacheOperation::acquire(&self.root)?;
|
||||
let directory = self.root.join("artifacts");
|
||||
for path in [&directory, &directory.join(&expected.sha256)] {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(_) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
||||
Err(error) => return Err(format!("cannot inspect shared artifact: {error}")),
|
||||
}
|
||||
if path == &directory {
|
||||
ensure_real_directory(&directory)?;
|
||||
}
|
||||
}
|
||||
let file = self.read_artifact_file(expected)?;
|
||||
Ok(StoredArtifact {
|
||||
Ok(Some(StoredArtifact {
|
||||
receipt: expected.clone(),
|
||||
file,
|
||||
_operation: operation,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn read_artifact_file(&self, expected: &ArtifactReceipt) -> Result<File, String> {
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
//! One signed catalog owns both the base release and immutable optional resources.
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::BuildTarget;
|
||||
use super::release::{
|
||||
self, Artifact, InstallLock, Layout, MachineManager, ReleaseManifest, Version,
|
||||
};
|
||||
use crate::libs::Ecosystem;
|
||||
|
||||
/// Explicit author/test transport. Product IPC always uses the compiled official source.
|
||||
pub trait ExtensionSource {
|
||||
fn open(&self, version: &Version, extension: &ExtensionId)
|
||||
-> Result<Box<dyn io::Read>, String>;
|
||||
}
|
||||
|
||||
pub(super) const PAYLOAD_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
|
||||
pub(super) const ARTIFACT_LIMIT: usize = 4096;
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(
|
||||
tag = "type",
|
||||
content = "ecosystem",
|
||||
rename_all = "snake_case",
|
||||
deny_unknown_fields
|
||||
)]
|
||||
pub enum ExtensionKind {
|
||||
Runtime(Ecosystem),
|
||||
Build(Ecosystem),
|
||||
Target,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ExtensionId {
|
||||
pub kind: ExtensionKind,
|
||||
pub target: BuildTarget,
|
||||
}
|
||||
|
||||
impl ExtensionId {
|
||||
pub fn name(&self) -> String {
|
||||
let kind = match &self.kind {
|
||||
ExtensionKind::Runtime(ecosystem) => format!("runtime-{}", ecosystem.as_str()),
|
||||
ExtensionKind::Build(ecosystem) => format!("build-{}", ecosystem.as_str()),
|
||||
ExtensionKind::Target => "target".into(),
|
||||
};
|
||||
format!("{kind}-{}", self.target.platform())
|
||||
}
|
||||
|
||||
pub fn asset_suffix(&self) -> String {
|
||||
format!("ext-{}.tar.zst", self.name())
|
||||
}
|
||||
|
||||
fn permits(&self, path: &str) -> bool {
|
||||
let target = self.target.platform();
|
||||
match &self.kind {
|
||||
ExtensionKind::Runtime(ecosystem) => {
|
||||
path.starts_with(&format!("runtime/{}/{target}/", ecosystem.as_str()))
|
||||
}
|
||||
ExtensionKind::Build(ecosystem) => {
|
||||
path.starts_with(&format!("build/{}/{target}/", ecosystem.as_str()))
|
||||
}
|
||||
ExtensionKind::Target => {
|
||||
path.starts_with(&format!("runtime/{target}/"))
|
||||
|| path.starts_with(&format!("sandbox/{target}/"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn missing(&self) -> String {
|
||||
match self.kind {
|
||||
ExtensionKind::Target => format!(
|
||||
"target resources are not installed; run 'dever target add {}'",
|
||||
self.target.platform()
|
||||
),
|
||||
_ => format!(
|
||||
"{} resources are not installed; run 'dever lib install <project-root> --target {}' (or 'dever lib add' for new dependencies)",
|
||||
self.name(),
|
||||
self.target.platform()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Extension {
|
||||
pub kind: ExtensionKind,
|
||||
pub target: BuildTarget,
|
||||
pub artifacts: Vec<Artifact>,
|
||||
}
|
||||
|
||||
impl Extension {
|
||||
pub fn id(&self) -> ExtensionId {
|
||||
ExtensionId {
|
||||
kind: self.kind.clone(),
|
||||
target: self.target,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
|
||||
Version::parse(manifest.version.as_str())?;
|
||||
if manifest.format != "dever-release-v2" || manifest.platform != release::platform_identity() {
|
||||
return Err("unsupported release format or host platform".into());
|
||||
}
|
||||
if manifest.extensions.len() > 12 {
|
||||
return Err("release extension count exceeds its limit".into());
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
validate_artifacts(&manifest.artifacts, &mut paths)?;
|
||||
let mut ids = BTreeSet::new();
|
||||
for extension in &manifest.extensions {
|
||||
let id = extension.id();
|
||||
if !ids.insert(id.clone())
|
||||
|| matches!(id.kind, ExtensionKind::Build(Ecosystem::Go))
|
||||
|| matches!(id.kind, ExtensionKind::Target) && id.target == BuildTarget::host()?
|
||||
{
|
||||
return Err("release has a duplicate or unsupported extension identity".into());
|
||||
}
|
||||
validate_artifacts(&extension.artifacts, &mut paths)?;
|
||||
if extension
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| !id.permits(&artifact.path))
|
||||
{
|
||||
return Err(format!(
|
||||
"extension '{}' contains a file outside its resource namespace",
|
||||
id.name()
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn validate_artifacts<'a>(
|
||||
artifacts: &'a [Artifact],
|
||||
paths: &mut BTreeSet<&'a str>,
|
||||
) -> Result<(), String> {
|
||||
if artifacts.is_empty() || artifacts.len() > ARTIFACT_LIMIT {
|
||||
return Err("release artifact count is outside 1..=4096".into());
|
||||
}
|
||||
let mut bytes = 0_u64;
|
||||
for artifact in artifacts {
|
||||
if artifact.path.is_empty()
|
||||
|| artifact.path.len() > 1024
|
||||
|| artifact
|
||||
.path
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
|| artifact.path.contains(['\\', ':', '\0'])
|
||||
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|
||||
|| !paths.insert(&artifact.path)
|
||||
{
|
||||
return Err("release has an unsafe or duplicate artifact path".into());
|
||||
}
|
||||
if artifact.sha256.len() != 64
|
||||
|| !artifact
|
||||
.sha256
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err("release artifact has an invalid SHA-256".into());
|
||||
}
|
||||
bytes = bytes
|
||||
.checked_add(artifact.bytes)
|
||||
.ok_or("release size overflow")?;
|
||||
if bytes > PAYLOAD_LIMIT {
|
||||
return Err("release exceeds its two-GiB unpacked size limit".into());
|
||||
}
|
||||
}
|
||||
// A signed file cannot also be the parent directory of another file.
|
||||
for path in paths.iter() {
|
||||
for (index, _) in path.match_indices('/') {
|
||||
if paths.contains(&path[..index]) {
|
||||
return Err("release artifact paths overlap as file and directory".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SignedResources {
|
||||
layout: Layout,
|
||||
base: PathBuf,
|
||||
manifest: ReleaseManifest,
|
||||
catalog: String,
|
||||
}
|
||||
|
||||
impl SignedResources {
|
||||
pub fn load(layout: &Layout, version: &Version) -> Result<Self, String> {
|
||||
Version::parse(version.as_str())?;
|
||||
layout.validate_machine_permissions()?;
|
||||
let base = layout.versions().join(version.as_str());
|
||||
let manifest = MachineManager::new(layout.clone()).verify_release(&base, Some(version))?;
|
||||
release::validate_installed_release(layout, &base, &manifest)?;
|
||||
let catalog = release::sha256_file(&base.join("manifest.json"))?;
|
||||
Ok(Self {
|
||||
layout: layout.clone(),
|
||||
base,
|
||||
manifest,
|
||||
catalog,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn manifest(&self) -> &ReleaseManifest {
|
||||
&self.manifest
|
||||
}
|
||||
|
||||
pub fn root(&self, logical_path: &str) -> Result<PathBuf, String> {
|
||||
if let Some(artifact) = self
|
||||
.manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.find(|artifact| artifact.path == logical_path)
|
||||
{
|
||||
verify_file(&self.base, artifact)?;
|
||||
return Ok(self.base.clone());
|
||||
}
|
||||
let extension = self
|
||||
.manifest
|
||||
.extensions
|
||||
.iter()
|
||||
.find(|extension| {
|
||||
extension
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == logical_path)
|
||||
})
|
||||
.ok_or_else(|| format!("signed release does not declare resource '{logical_path}'"))?;
|
||||
self.extension_root(extension)
|
||||
}
|
||||
|
||||
pub fn path(&self, logical_path: &str) -> Result<PathBuf, String> {
|
||||
Ok(self.root(logical_path)?.join(logical_path))
|
||||
}
|
||||
|
||||
pub fn read(&self, logical_path: &str) -> Result<Vec<u8>, String> {
|
||||
let path = self.path(logical_path)?;
|
||||
let artifact = self
|
||||
.manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.chain(
|
||||
self.manifest
|
||||
.extensions
|
||||
.iter()
|
||||
.flat_map(|extension| &extension.artifacts),
|
||||
)
|
||||
.find(|artifact| artifact.path == logical_path)
|
||||
.expect("root required signed artifact");
|
||||
let bytes =
|
||||
fs::read(path).map_err(|error| format!("cannot read signed resource: {error}"))?;
|
||||
use sha2::{Digest, Sha256};
|
||||
if bytes.len() as u64 != artifact.bytes
|
||||
|| release::hex(&Sha256::digest(&bytes)) != artifact.sha256
|
||||
{
|
||||
return Err("signed resource changed while reading".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub fn artifacts(&self, prefix: &str) -> Result<Vec<Artifact>, String> {
|
||||
let mut artifacts = Vec::new();
|
||||
for artifact in self
|
||||
.manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.filter(|artifact| artifact.path.starts_with(prefix))
|
||||
{
|
||||
verify_file(&self.base, artifact)?;
|
||||
artifacts.push(artifact.clone());
|
||||
}
|
||||
for extension in &self.manifest.extensions {
|
||||
if extension
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.path.starts_with(prefix))
|
||||
{
|
||||
self.extension_root(extension)?;
|
||||
artifacts.extend(
|
||||
extension
|
||||
.artifacts
|
||||
.iter()
|
||||
.filter(|artifact| artifact.path.starts_with(prefix))
|
||||
.cloned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(artifacts)
|
||||
}
|
||||
|
||||
fn store(&self) -> PathBuf {
|
||||
self.layout
|
||||
.extensions()
|
||||
.join(self.manifest.version.as_str())
|
||||
.join(&self.catalog)
|
||||
}
|
||||
|
||||
fn destination(&self, id: &ExtensionId) -> PathBuf {
|
||||
self.store().join(id.name())
|
||||
}
|
||||
|
||||
fn extension(&self, id: &ExtensionId) -> Result<&Extension, String> {
|
||||
self.manifest
|
||||
.extensions
|
||||
.iter()
|
||||
.find(|extension| extension.id() == *id)
|
||||
.ok_or_else(|| {
|
||||
format!(
|
||||
"Dever {} does not declare extension '{}'",
|
||||
self.manifest.version,
|
||||
id.name()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn extension_root(&self, extension: &Extension) -> Result<PathBuf, String> {
|
||||
let id = extension.id();
|
||||
let root = self.destination(&id);
|
||||
if !self.store_present()? || !exists(&root)? {
|
||||
return Err(id.missing());
|
||||
}
|
||||
verify_extension(&root, &extension.artifacts)?;
|
||||
Ok(root)
|
||||
}
|
||||
|
||||
fn store_present(&self) -> Result<bool, String> {
|
||||
let mut parent = self.layout.cache();
|
||||
for directory in [
|
||||
self.layout.extensions(),
|
||||
self.layout
|
||||
.extensions()
|
||||
.join(self.manifest.version.as_str()),
|
||||
self.store(),
|
||||
] {
|
||||
if !exists(&directory)? {
|
||||
return Ok(false);
|
||||
}
|
||||
release::validate_shared_directory(&directory, &parent, "extension store")?;
|
||||
parent = directory;
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub(super) fn installed_extensions(&self) -> Result<Vec<ExtensionId>, String> {
|
||||
if !self.store_present()? {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut installed = Vec::new();
|
||||
for entry in fs::read_dir(self.store()).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let extension = self
|
||||
.manifest
|
||||
.extensions
|
||||
.iter()
|
||||
.find(|extension| entry.file_name() == extension.id().name().as_str())
|
||||
.ok_or("extension store contains an undeclared entry")?;
|
||||
self.extension_root(extension)?;
|
||||
installed.push(extension.id());
|
||||
}
|
||||
Ok(installed)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn exists(path: &Path) -> Result<bool, String> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(_) => Ok(true),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
|
||||
Err(error) => Err(format!(
|
||||
"cannot inspect extension path '{}': {error}",
|
||||
path.display()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_file(root: &Path, artifact: &Artifact) -> Result<(), String> {
|
||||
let mut parent = root.to_path_buf();
|
||||
let relative = Path::new(&artifact.path);
|
||||
for component in relative.parent().into_iter().flat_map(Path::components) {
|
||||
let directory = parent.join(component);
|
||||
release::validate_shared_directory(&directory, &parent, "signed resource directory")?;
|
||||
parent = directory;
|
||||
}
|
||||
let path = root.join(relative);
|
||||
release::validate_shared_file(&path, &parent, "signed resource", false)?;
|
||||
if fs::metadata(&path)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len()
|
||||
!= artifact.bytes
|
||||
|| release::sha256_file(&path)? != artifact.sha256
|
||||
{
|
||||
return Err(format!(
|
||||
"signed resource '{}' failed integrity verification",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn verify_extension(root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
|
||||
release::validate_shared_directory(
|
||||
root,
|
||||
root.parent().ok_or("extension root has no parent")?,
|
||||
"installed extension",
|
||||
)?;
|
||||
for artifact in artifacts {
|
||||
verify_file(root, artifact)?;
|
||||
}
|
||||
fn entries(root: &Path, directory: &Path, expected: &BTreeSet<&str>) -> Result<(), String> {
|
||||
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let path = entry.path();
|
||||
let kind = entry.file_type().map_err(|error| error.to_string())?;
|
||||
let relative = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.to_str()
|
||||
.ok_or("extension path is not UTF-8")?;
|
||||
if kind.is_dir() {
|
||||
release::validate_shared_directory(&path, directory, "extension directory")?;
|
||||
if !expected
|
||||
.iter()
|
||||
.any(|artifact| artifact.starts_with(&format!("{relative}/")))
|
||||
{
|
||||
return Err("installed extension contains an unsigned directory".into());
|
||||
}
|
||||
entries(root, &path, expected)?;
|
||||
} else if !kind.is_file() || !expected.contains(relative) {
|
||||
return Err("installed extension contains an unsigned or non-regular file".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
entries(
|
||||
root,
|
||||
root,
|
||||
&artifacts
|
||||
.iter()
|
||||
.map(|artifact| artifact.path.as_str())
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn ensure(layout: &Layout, version: &Version, id: &ExtensionId) -> Result<(), String> {
|
||||
prepare_extension_with(
|
||||
layout,
|
||||
version,
|
||||
id,
|
||||
&super::release_source::ReleaseSource::official(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn prepare_extension_with(
|
||||
layout: &Layout,
|
||||
version: &Version,
|
||||
id: &ExtensionId,
|
||||
source: &dyn ExtensionSource,
|
||||
) -> Result<(), String> {
|
||||
Version::parse(version.as_str())?;
|
||||
let _lock = InstallLock::acquire_wait(layout)?;
|
||||
ensure_locked(layout, version, id, source)
|
||||
}
|
||||
|
||||
pub(super) fn ensure_locked(
|
||||
layout: &Layout,
|
||||
version: &Version,
|
||||
id: &ExtensionId,
|
||||
source: &dyn ExtensionSource,
|
||||
) -> Result<(), String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
super::bootstrap::require_no_pending(layout)?;
|
||||
let resources = SignedResources::load(layout, version)?;
|
||||
let extension = resources.extension(id)?;
|
||||
let destination = resources.destination(id);
|
||||
if exists(&destination)? {
|
||||
resources.extension_root(extension)?;
|
||||
return Ok(());
|
||||
}
|
||||
let mut parent = layout.cache();
|
||||
for directory in [
|
||||
layout.extensions(),
|
||||
layout.extensions().join(version.as_str()),
|
||||
resources.store(),
|
||||
] {
|
||||
if !exists(&directory)? {
|
||||
fs::create_dir(&directory)
|
||||
.map_err(|error| format!("cannot create extension store: {error}"))?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&directory, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
release::sync_directory(&parent).map_err(|error| error.to_string())?;
|
||||
}
|
||||
release::validate_shared_directory(&directory, &parent, "extension store")?;
|
||||
parent = directory;
|
||||
}
|
||||
super::release_source::prepare_extension(layout, version, extension, &destination, source)
|
||||
}
|
||||
@@ -5,7 +5,7 @@ use std::process::Command;
|
||||
|
||||
use super::release::{Layout, MachineManager, Version, platform_contract};
|
||||
|
||||
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
|
||||
const USAGE: &str = "Usage:\n dever install <version|latest>\n dever update\n dever target add linux-aarch64\n dever use <version>\n dever uninstall <version>\n dever version [project-root]\n dever cache status\n dever cache clean\n dever lib add|list|update|install|remove|doctor <project-root> [--target linux-x86_64|linux-aarch64] [spec ...]\n dever package add|list|update|remove|doctor <project-root> [spec ...]\n dever check <project-root>\n dever api <project-root> [--output <new-file>]\n dever fmt <project-root> [--check]\n dever test <project-root>\n dever run <project-root> [-- <component>.<domain>.<cmd> '<json-object>']\n dever build <project-root> --output <new-file> [--target linux-x86_64|linux-aarch64]\n dever clean <project-root>\n dever tenant ...";
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CommandResult {
|
||||
@@ -30,6 +30,20 @@ pub fn execute_in(layout: &Layout, arguments: &[OsString]) -> Result<CommandResu
|
||||
super::bootstrap::require_no_pending(layout)?;
|
||||
let manager = MachineManager::new(layout.clone());
|
||||
match arguments {
|
||||
[command, operation, target] if command == "target" && operation == "add" => {
|
||||
let target = utf8(target, "target")?.parse::<super::BuildTarget>()?;
|
||||
if target != super::BuildTarget::LinuxAarch64 {
|
||||
return Err("target add supports linux-aarch64".into());
|
||||
}
|
||||
let version = manager
|
||||
.active_version()?
|
||||
.ok_or("no active Dever version; run 'dever install latest'")?;
|
||||
super::ensure_extension(layout, &version, super::ExtensionKind::Target, target)?;
|
||||
Ok(CommandResult::Message(format!(
|
||||
"prepared {} for Dever {version}",
|
||||
target.platform()
|
||||
)))
|
||||
}
|
||||
[command, requested] if command == "install" => {
|
||||
let requested = utf8(requested, "version")?;
|
||||
super::release_source::prepare(layout, requested)?;
|
||||
|
||||
@@ -3,15 +3,22 @@ pub mod bootstrap;
|
||||
mod cache;
|
||||
pub mod compilation;
|
||||
mod core_libraries;
|
||||
mod extensions;
|
||||
mod launcher;
|
||||
pub mod packaging;
|
||||
mod release;
|
||||
mod release_archive;
|
||||
mod release_source;
|
||||
pub mod runtime_pack;
|
||||
mod service;
|
||||
mod skill;
|
||||
mod target;
|
||||
|
||||
pub use extensions::{
|
||||
Extension, ExtensionId, ExtensionKind, ExtensionSource, SignedResources,
|
||||
prepare_extension_with, validate_catalog,
|
||||
};
|
||||
pub use release_source::extract_config;
|
||||
pub use target::BuildTarget;
|
||||
|
||||
pub use cache::{
|
||||
@@ -22,4 +29,7 @@ pub use release::{
|
||||
Artifact, DownloadedReleases, Layout, MachineManager, PlatformContract, ReleaseManifest,
|
||||
Version, platform_contract, platform_identity, sha256_file,
|
||||
};
|
||||
pub use service::{artifact_get, artifact_put, cache_clean, cache_status, compile, serve};
|
||||
pub use service::{
|
||||
artifact_get, artifact_get_optional, artifact_put, cache_clean, cache_status, compile,
|
||||
ensure_extension, serve,
|
||||
};
|
||||
@@ -12,7 +12,10 @@ use serde::Deserialize;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::runtime_pack::{self, Manifest, Profile, Profiles, RuntimeArtifact};
|
||||
use super::{Artifact, BuildTarget, ReleaseManifest, Version, sha256_file};
|
||||
use super::{
|
||||
Artifact, BuildTarget, Extension, ExtensionKind, ReleaseManifest, Version, sha256_file,
|
||||
validate_catalog,
|
||||
};
|
||||
|
||||
static NEXT_STAGING: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
@@ -205,33 +208,71 @@ fn assemble(
|
||||
end: settings.end,
|
||||
},
|
||||
)?);
|
||||
let mut extensions = Vec::new();
|
||||
let mut target_artifacts = BTreeMap::new();
|
||||
for (target, inputs) in settings.native_targets {
|
||||
artifacts.extend(assemble_native(&mut writer, &version, target, inputs)?);
|
||||
target_artifacts.insert(
|
||||
target,
|
||||
assemble_native(&mut writer, &version, target, inputs)?,
|
||||
);
|
||||
}
|
||||
for (ecosystem, input) in settings.runtimes.entries() {
|
||||
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, host)?);
|
||||
extensions.push(Extension {
|
||||
kind: ExtensionKind::Runtime(ecosystem.clone()),
|
||||
target: host,
|
||||
artifacts: ecosystems::assemble(&mut writer, ecosystem, input, host)?,
|
||||
});
|
||||
}
|
||||
for (target, inputs) in settings.ecosystem_targets {
|
||||
for (ecosystem, input) in inputs.entries() {
|
||||
artifacts.extend(ecosystems::assemble(&mut writer, ecosystem, input, target)?);
|
||||
extensions.push(Extension {
|
||||
kind: ExtensionKind::Runtime(ecosystem.clone()),
|
||||
target,
|
||||
artifacts: ecosystems::assemble(&mut writer, ecosystem, input, target)?,
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(inputs) = settings.sandbox {
|
||||
artifacts.extend(sandbox::assemble(&mut writer, inputs, host)?);
|
||||
}
|
||||
for (target, inputs) in settings.sandbox_targets {
|
||||
artifacts.extend(sandbox::assemble(&mut writer, inputs, target)?);
|
||||
let selected = target_artifacts
|
||||
.get_mut(&target)
|
||||
.ok_or("target sandbox requires a matching native target")?;
|
||||
selected.extend(sandbox::assemble(&mut writer, inputs, target)?);
|
||||
}
|
||||
for (target, artifacts) in target_artifacts {
|
||||
extensions.push(Extension {
|
||||
kind: ExtensionKind::Target,
|
||||
target,
|
||||
artifacts,
|
||||
});
|
||||
}
|
||||
for (ecosystem, input) in settings.builds.entries() {
|
||||
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, host)?);
|
||||
extensions.push(Extension {
|
||||
kind: ExtensionKind::Build(ecosystem.clone()),
|
||||
target: host,
|
||||
artifacts: builds::assemble(&mut writer, ecosystem, input, host)?,
|
||||
});
|
||||
}
|
||||
for (target, inputs) in settings.build_targets {
|
||||
for (ecosystem, input) in inputs.entries() {
|
||||
artifacts.extend(builds::assemble(&mut writer, ecosystem, input, target)?);
|
||||
extensions.push(Extension {
|
||||
kind: ExtensionKind::Build(ecosystem.clone()),
|
||||
target,
|
||||
artifacts: builds::assemble(&mut writer, ecosystem, input, target)?,
|
||||
});
|
||||
}
|
||||
}
|
||||
artifacts.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
let manifest = ReleaseManifest::new(version, artifacts);
|
||||
for extension in &mut extensions {
|
||||
extension
|
||||
.artifacts
|
||||
.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
}
|
||||
let mut manifest = ReleaseManifest::new(version, artifacts);
|
||||
manifest.extensions = extensions;
|
||||
validate_catalog(&manifest)?;
|
||||
let bytes = write_metadata(&staging.join("manifest.json"), &manifest)?;
|
||||
let signature = key.pair.sign(&bytes);
|
||||
let hex: String = signature
|
||||
|
||||
@@ -74,15 +74,17 @@ pub struct ReleaseManifest {
|
||||
pub version: Version,
|
||||
pub platform: String,
|
||||
pub artifacts: Vec<Artifact>,
|
||||
pub extensions: Vec<super::extensions::Extension>,
|
||||
}
|
||||
|
||||
impl ReleaseManifest {
|
||||
pub fn new(version: Version, artifacts: Vec<Artifact>) -> Self {
|
||||
Self {
|
||||
format: "dever-release-v1".into(),
|
||||
format: "dever-release-v2".into(),
|
||||
version,
|
||||
platform: platform_identity(),
|
||||
artifacts,
|
||||
extensions: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -129,6 +131,9 @@ impl Layout {
|
||||
pub fn cache(&self) -> PathBuf {
|
||||
self.root.join("cache")
|
||||
}
|
||||
pub fn extensions(&self) -> PathBuf {
|
||||
self.cache().join("extensions")
|
||||
}
|
||||
pub fn downloads(&self) -> PathBuf {
|
||||
self.cache().join("downloads")
|
||||
}
|
||||
@@ -307,6 +312,11 @@ impl MachineManager {
|
||||
}
|
||||
|
||||
pub fn update(&self) -> Result<Version, String> {
|
||||
self.update_with(&super::release_source::ReleaseSource::official())
|
||||
}
|
||||
|
||||
/// Uses the ordinary install transaction with an explicitly supplied author transport.
|
||||
pub fn update_with(&self, source: &dyn super::ExtensionSource) -> Result<Version, String> {
|
||||
let version = self.releases.latest()?;
|
||||
let _lock = InstallLock::acquire(&self.layout)?;
|
||||
#[cfg(target_os = "linux")]
|
||||
@@ -321,6 +331,12 @@ impl MachineManager {
|
||||
}
|
||||
self.cleanup_staging()?;
|
||||
self.install_locked(&version)?;
|
||||
if let Some(active) = self.active_version()? {
|
||||
let resources = super::extensions::SignedResources::load(&self.layout, &active)?;
|
||||
for extension in resources.installed_extensions()? {
|
||||
super::extensions::ensure_locked(&self.layout, &version, &extension, source)?;
|
||||
}
|
||||
}
|
||||
self.activate_locked(&version)?;
|
||||
Ok(version)
|
||||
}
|
||||
@@ -468,6 +484,7 @@ impl MachineManager {
|
||||
Ok(InstalledCompiler {
|
||||
core,
|
||||
manifest,
|
||||
resources: super::SignedResources::load(&self.layout, version)?,
|
||||
identity: sha256_file(&root.join(MANIFEST_NAME))?,
|
||||
_lease: lease,
|
||||
})
|
||||
@@ -486,6 +503,14 @@ impl MachineManager {
|
||||
if destination.exists() {
|
||||
let manifest = self.verify_release(&destination, Some(version))?;
|
||||
validate_installed_release(&self.layout, &destination, &manifest)?;
|
||||
if package.exists() {
|
||||
self.verify_release(package, Some(version))?;
|
||||
if read_regular_file(&destination.join(MANIFEST_NAME))?
|
||||
!= read_regular_file(&package.join(MANIFEST_NAME))?
|
||||
{
|
||||
return Err("published Dever version changed its signed manifest".into());
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let manifest = self.verify_release(package, Some(version))?;
|
||||
@@ -548,7 +573,7 @@ impl MachineManager {
|
||||
.map_err(|error| format!("cannot activate Dever {version}: {error}"))
|
||||
}
|
||||
|
||||
fn verify_release(
|
||||
pub(super) fn verify_release(
|
||||
&self,
|
||||
root: &Path,
|
||||
expected: Option<&Version>,
|
||||
@@ -601,7 +626,7 @@ impl MachineManager {
|
||||
manifest_path.display()
|
||||
)
|
||||
})?;
|
||||
if manifest.format != "dever-release-v1" {
|
||||
if manifest.format != "dever-release-v2" {
|
||||
return Err("unsupported Dever release manifest format".into());
|
||||
}
|
||||
if manifest.platform != platform_identity() {
|
||||
@@ -611,6 +636,7 @@ impl MachineManager {
|
||||
platform_identity()
|
||||
));
|
||||
}
|
||||
super::extensions::validate_catalog(&manifest)?;
|
||||
Ok(manifest)
|
||||
}
|
||||
|
||||
@@ -706,6 +732,7 @@ pub(crate) struct InstalledCompiler {
|
||||
pub(crate) core: PathBuf,
|
||||
pub(crate) manifest: ReleaseManifest,
|
||||
pub(crate) identity: String,
|
||||
pub(crate) resources: super::SignedResources,
|
||||
_lease: InstallLock,
|
||||
}
|
||||
|
||||
@@ -713,14 +740,18 @@ pub(super) struct InstallLock(File);
|
||||
|
||||
impl InstallLock {
|
||||
pub(super) fn acquire(layout: &Layout) -> Result<Self, String> {
|
||||
Self::open(layout, false)
|
||||
Self::open(layout, false, false)
|
||||
}
|
||||
|
||||
pub(super) fn acquire_wait(layout: &Layout) -> Result<Self, String> {
|
||||
Self::open(layout, false, true)
|
||||
}
|
||||
|
||||
fn shared(layout: &Layout) -> Result<Self, String> {
|
||||
Self::open(layout, true)
|
||||
Self::open(layout, true, false)
|
||||
}
|
||||
|
||||
fn open(layout: &Layout, shared: bool) -> Result<Self, String> {
|
||||
fn open(layout: &Layout, shared: bool, wait: bool) -> Result<Self, String> {
|
||||
let path = layout.state().join("install.lock");
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true).write(true).create(true).truncate(false);
|
||||
@@ -750,14 +781,29 @@ impl InstallLock {
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
let _ = current;
|
||||
let locked = if shared {
|
||||
fs2::FileExt::try_lock_shared(&file)
|
||||
} else {
|
||||
fs2::FileExt::try_lock_exclusive(&file)
|
||||
};
|
||||
locked.map_err(|error| {
|
||||
format!("another Dever installation operation or compilation is active: {error}")
|
||||
})?;
|
||||
let started = std::time::Instant::now();
|
||||
loop {
|
||||
let locked = if shared {
|
||||
fs2::FileExt::try_lock_shared(&file)
|
||||
} else {
|
||||
fs2::FileExt::try_lock_exclusive(&file)
|
||||
};
|
||||
match locked {
|
||||
Ok(()) => break,
|
||||
Err(error)
|
||||
if wait
|
||||
&& error.kind() == io::ErrorKind::WouldBlock
|
||||
&& started.elapsed() < std::time::Duration::from_secs(300) =>
|
||||
{
|
||||
std::thread::sleep(std::time::Duration::from_millis(50))
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(format!(
|
||||
"another Dever installation operation or compilation is active: {error}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Self(file))
|
||||
}
|
||||
}
|
||||
@@ -912,7 +958,17 @@ fn read_regular_file(path: &Path) -> Result<Vec<u8>, String> {
|
||||
if !metadata.file_type().is_file() {
|
||||
return Err(format!("'{}' must be a regular file", path.display()));
|
||||
}
|
||||
fs::read(path).map_err(|error| format!("cannot read '{}': {error}", path.display()))
|
||||
const LIMIT: u64 = 2 * 1024 * 1024;
|
||||
let mut bytes = Vec::new();
|
||||
File::open(path)
|
||||
.map_err(|error| format!("cannot read '{}': {error}", path.display()))?
|
||||
.take(LIMIT + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
if bytes.len() as u64 > LIMIT {
|
||||
return Err("release metadata exceeds its size limit".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(super) fn copy_file(source: &Path, destination: &Path, executable: bool) -> Result<(), String> {
|
||||
@@ -964,7 +1020,7 @@ pub(super) fn sync_tree(root: &Path) -> Result<(), String> {
|
||||
sync_directory(root).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn validate_installed_release(
|
||||
pub(super) fn validate_installed_release(
|
||||
layout: &Layout,
|
||||
root: &Path,
|
||||
manifest: &ReleaseManifest,
|
||||
@@ -995,7 +1051,11 @@ fn validate_installed_release(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_shared_directory(path: &Path, parent: &Path, label: &str) -> Result<(), String> {
|
||||
pub(super) fn validate_shared_directory(
|
||||
path: &Path,
|
||||
parent: &Path,
|
||||
label: &str,
|
||||
) -> Result<(), String> {
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.map_err(|error| format!("cannot inspect {label} '{}': {error}", path.display()))?;
|
||||
if metadata.file_type().is_symlink() || !metadata.is_dir() {
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
//! Bounded single-frame Zstandard and exact signed USTAR extraction.
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::{self, BufRead, BufReader, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use super::extensions::{ARTIFACT_LIMIT, PAYLOAD_LIMIT, validate_artifacts};
|
||||
use super::release::{Artifact, set_shared_file_permissions, sha256_file};
|
||||
|
||||
pub(super) fn extract(
|
||||
reader: impl Read,
|
||||
root: &Path,
|
||||
artifacts: &[Artifact],
|
||||
) -> Result<(), String> {
|
||||
validate_artifacts(artifacts, &mut BTreeSet::new())?;
|
||||
let mut input = Bounded {
|
||||
input: reader,
|
||||
remaining: PAYLOAD_LIMIT,
|
||||
};
|
||||
let mut header = [0_u8; 5];
|
||||
input
|
||||
.read_exact(&mut header)
|
||||
.map_err(|error| format!("cannot read Zstandard header: {error}"))?;
|
||||
if header[..4] != [0x28, 0xb5, 0x2f, 0xfd] || header[4] & 4 == 0 {
|
||||
return Err("release requires a Zstandard frame with a content checksum".into());
|
||||
}
|
||||
let mut decoder = zstd::stream::read::Decoder::with_buffer(BufReader::new(
|
||||
io::Cursor::new(header).chain(input),
|
||||
))
|
||||
.map_err(|error| format!("cannot initialize Zstandard decoder: {error}"))?
|
||||
.single_frame();
|
||||
decoder
|
||||
.window_log_max(27)
|
||||
.map_err(|error| format!("cannot bound Zstandard window: {error}"))?;
|
||||
unpack(&mut decoder, root, artifacts)?;
|
||||
// unpack drains the decoder through its checksum, including errors after TAR EOF.
|
||||
let mut input = decoder.finish();
|
||||
if !input
|
||||
.fill_buf()
|
||||
.map_err(|error| format!("cannot finish release frame: {error}"))?
|
||||
.is_empty()
|
||||
{
|
||||
return Err("release contains data after its single Zstandard frame".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn unpack(reader: impl Read, root: &Path, artifacts: &[Artifact]) -> Result<(), String> {
|
||||
validate_artifacts(artifacts, &mut BTreeSet::new())?;
|
||||
let expected: BTreeMap<_, _> = artifacts
|
||||
.iter()
|
||||
.map(|artifact| (artifact.path.as_str(), artifact))
|
||||
.collect();
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut archive = Bounded {
|
||||
input: reader,
|
||||
remaining: PAYLOAD_LIMIT + ARTIFACT_LIMIT as u64 * 1024 + 10240,
|
||||
};
|
||||
while let Some(header) = read_header(&mut archive)? {
|
||||
let name = std::str::from_utf8(&header.path_bytes())
|
||||
.map_err(|_| "release archive path is not UTF-8")?
|
||||
.to_owned();
|
||||
let artifact = expected
|
||||
.get(name.as_str())
|
||||
.ok_or("release archive contains an unsigned file")?;
|
||||
if !seen.insert(name) || header.size().map_err(|error| error.to_string())? != artifact.bytes
|
||||
{
|
||||
return Err("release archive contains a duplicate or wrong-sized file".into());
|
||||
}
|
||||
extract_file(&mut archive, root, artifact)?;
|
||||
}
|
||||
if seen.len() != expected.len() {
|
||||
return Err("release archive is missing signed artifacts".into());
|
||||
}
|
||||
verify_trailer(&mut archive)
|
||||
}
|
||||
|
||||
fn read_header(reader: &mut impl Read) -> Result<Option<tar::Header>, String> {
|
||||
let mut header = tar::Header::new_ustar();
|
||||
reader
|
||||
.read_exact(header.as_mut_bytes())
|
||||
.map_err(|error| format!("incomplete release TAR header: {error}"))?;
|
||||
if header.as_bytes().iter().all(|byte| *byte == 0) {
|
||||
return Ok(None);
|
||||
}
|
||||
let checksum = header.as_bytes()[..148]
|
||||
.iter()
|
||||
.chain(&header.as_bytes()[156..])
|
||||
.map(|byte| u32::from(*byte))
|
||||
.sum::<u32>()
|
||||
+ 8 * u32::from(b' ');
|
||||
if header.cksum().map_err(|error| error.to_string())? != checksum {
|
||||
return Err("release TAR header checksum is invalid".into());
|
||||
}
|
||||
if !header.entry_type().is_file() || header.as_ustar().is_none() {
|
||||
return Err("release archives may contain only USTAR regular files".into());
|
||||
}
|
||||
Ok(Some(header))
|
||||
}
|
||||
|
||||
fn extract_file(reader: &mut impl Read, root: &Path, artifact: &Artifact) -> Result<(), String> {
|
||||
let path = root.join(&artifact.path);
|
||||
let mut parent = root.to_path_buf();
|
||||
for part in Path::new(&artifact.path)
|
||||
.parent()
|
||||
.into_iter()
|
||||
.flat_map(Path::components)
|
||||
{
|
||||
parent.push(part);
|
||||
match fs::create_dir(&parent) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {
|
||||
let metadata = fs::symlink_metadata(&parent).map_err(|error| error.to_string())?;
|
||||
if !metadata.is_dir() || metadata.file_type().is_symlink() {
|
||||
return Err("release artifact parent is not a real directory".into());
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(format!("cannot create release artifact directory: {error}")),
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&parent, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot create release artifact: {error}"))?;
|
||||
if io::copy(&mut reader.by_ref().take(artifact.bytes), &mut file)
|
||||
.map_err(|error| format!("cannot extract release artifact: {error}"))?
|
||||
!= artifact.bytes
|
||||
{
|
||||
return Err("release archive contains a truncated file".into());
|
||||
}
|
||||
let padding_length = ((512 - artifact.bytes % 512) % 512) as usize;
|
||||
let mut padding = [0; 512];
|
||||
reader
|
||||
.read_exact(&mut padding[..padding_length])
|
||||
.map_err(|error| format!("incomplete TAR entry padding: {error}"))?;
|
||||
if padding[..padding_length].iter().any(|byte| *byte != 0) {
|
||||
return Err("release TAR entry padding must be zero".into());
|
||||
}
|
||||
if sha256_file(&path)? != artifact.sha256 {
|
||||
return Err("release artifact failed integrity verification".into());
|
||||
}
|
||||
// First-install invokes the extracted signed bootstrap before machine install.
|
||||
// Archive mode bits are untrusted; executable roles come from fixed paths.
|
||||
let executable = matches!(
|
||||
artifact.path.as_str(),
|
||||
"dever-core" | "bootstrap/dever" | "bootstrap/deverd"
|
||||
);
|
||||
set_shared_file_permissions(&path, executable)?;
|
||||
file.sync_all().map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn verify_trailer(reader: &mut impl Read) -> Result<(), String> {
|
||||
let mut tail_bytes = 0_usize;
|
||||
let mut tail = [0_u8; 4096];
|
||||
loop {
|
||||
let length = reader
|
||||
.read(&mut tail)
|
||||
.map_err(|error| format!("invalid release archive trailer: {error}"))?;
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
tail_bytes += length;
|
||||
if tail_bytes > 10240 || tail[..length].iter().any(|byte| *byte != 0) {
|
||||
return Err("release archive contains an invalid TAR trailer".into());
|
||||
}
|
||||
}
|
||||
// The first zero header was consumed by read_header; require the second.
|
||||
if tail_bytes < 512 || !tail_bytes.is_multiple_of(512) {
|
||||
return Err("release archive is missing its complete TAR terminator".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Bounded<R> {
|
||||
input: R,
|
||||
remaining: u64,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for Bounded<R> {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
||||
if buffer.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
if self.remaining == 0 {
|
||||
let mut probe = [0];
|
||||
return match self.input.read(&mut probe)? {
|
||||
0 => Ok(0),
|
||||
_ => Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"release byte limit exceeded",
|
||||
)),
|
||||
};
|
||||
}
|
||||
let limit = (self.remaining.min(buffer.len() as u64)) as usize;
|
||||
let length = self.input.read(&mut buffer[..limit])?;
|
||||
self.remaining -= length as u64;
|
||||
Ok(length)
|
||||
}
|
||||
}
|
||||
@@ -1,31 +1,35 @@
|
||||
//! Explicit installation/update downloads. Project commands never enter here.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use super::extensions::Extension;
|
||||
#[cfg(test)]
|
||||
use super::extensions::PAYLOAD_LIMIT as RELEASE_LIMIT;
|
||||
#[cfg(test)]
|
||||
use super::release::ReleaseManifest;
|
||||
use super::release::{
|
||||
InstallLock, Layout, MachineManager, ReleaseManifest, Version, decode_hex_file,
|
||||
ensure_real_directory, platform_identity, reject_symlink_ancestors,
|
||||
set_shared_file_permissions, sync_directory, sync_tree, validate_shared_file,
|
||||
InstallLock, Layout, MachineManager, Version, decode_hex_file, ensure_real_directory,
|
||||
platform_identity, reject_symlink_ancestors, set_shared_file_permissions, sync_directory,
|
||||
sync_tree, validate_shared_file,
|
||||
};
|
||||
use super::validate_catalog;
|
||||
|
||||
const OFFICIAL_RELEASES: &str = "https://github.com/shemic/dever-main/releases";
|
||||
const DOWNLOAD_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const METADATA_LIMIT: u64 = 2 * 1024 * 1024;
|
||||
const RELEASE_LIMIT: u64 = 2 * 1024 * 1024 * 1024;
|
||||
static NEXT_DOWNLOAD: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
struct ReleaseSource {
|
||||
pub(super) struct ReleaseSource {
|
||||
base: String,
|
||||
agent: ureq::Agent,
|
||||
}
|
||||
|
||||
impl ReleaseSource {
|
||||
fn official() -> Self {
|
||||
pub(super) fn official() -> Self {
|
||||
Self::new(OFFICIAL_RELEASES.to_owned())
|
||||
}
|
||||
|
||||
@@ -122,6 +126,23 @@ impl ReleaseSource {
|
||||
}
|
||||
}
|
||||
|
||||
impl super::ExtensionSource for ReleaseSource {
|
||||
fn open(
|
||||
&self,
|
||||
version: &Version,
|
||||
extension: &super::ExtensionId,
|
||||
) -> Result<Box<dyn Read>, String> {
|
||||
Ok(Box::new(
|
||||
self.response(
|
||||
self.asset(version.as_str(), &extension.asset_suffix()),
|
||||
Instant::now(),
|
||||
)?
|
||||
.into_body()
|
||||
.into_reader(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn redirect(current: &str, location: &str) -> Result<String, String> {
|
||||
let next = url::Url::parse(current)
|
||||
.and_then(|base| base.join(location))
|
||||
@@ -218,12 +239,11 @@ fn prepare_from(
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {
|
||||
let mut response =
|
||||
source.response(source.asset(version.as_str(), "tar.gz"), started)?;
|
||||
let compressed = response.body_mut().as_reader().take(RELEASE_LIMIT + 1);
|
||||
unpack(
|
||||
flate2::read::GzDecoder::new(compressed),
|
||||
source.response(source.asset(version.as_str(), "tar.zst"), started)?;
|
||||
super::release_archive::extract(
|
||||
response.body_mut().as_reader(),
|
||||
&staging.0,
|
||||
&manifest,
|
||||
&manifest.artifacts,
|
||||
)?;
|
||||
MachineManager::verify_package(&staging.0, Some(&version), &key)?;
|
||||
sync_tree(&staging.0)?;
|
||||
@@ -250,80 +270,111 @@ fn prepare_from(
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
fn validate_catalog(manifest: &ReleaseManifest) -> Result<(), String> {
|
||||
if manifest.artifacts.is_empty() || manifest.artifacts.len() > 4096 {
|
||||
return Err("release artifact count is outside 1..=4096".into());
|
||||
}
|
||||
let mut paths = BTreeSet::new();
|
||||
let mut size = 0_u64;
|
||||
for artifact in &manifest.artifacts {
|
||||
if artifact
|
||||
.path
|
||||
.split('/')
|
||||
.any(|part| part.is_empty() || part == "." || part == "..")
|
||||
|| artifact.path.contains(['\\', ':', '\0'])
|
||||
|| matches!(artifact.path.as_str(), "manifest.json" | "manifest.sig")
|
||||
|| !paths.insert(&artifact.path)
|
||||
{
|
||||
return Err("release has an unsafe or duplicate artifact path".into());
|
||||
}
|
||||
size = size
|
||||
.checked_add(artifact.bytes)
|
||||
.ok_or("release size overflow")?;
|
||||
if size > RELEASE_LIMIT {
|
||||
return Err("release exceeds its two-GiB unpacked size limit".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn unpack(reader: impl Read, root: &Path, manifest: &ReleaseManifest) -> Result<(), String> {
|
||||
validate_catalog(manifest)?;
|
||||
let expected: BTreeMap<_, _> = manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.map(|artifact| (artifact.path.as_str(), artifact))
|
||||
.collect();
|
||||
let mut seen = BTreeSet::new();
|
||||
// Raw entries reject PAX/GNU extensions before tar can buffer arbitrary metadata.
|
||||
let mut archive = tar::Archive::new(reader.take(RELEASE_LIMIT + 4096 * 1024));
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|error| format!("invalid release archive: {error}"))?
|
||||
.raw(true)
|
||||
super::release_archive::unpack(reader, root, &manifest.artifacts)
|
||||
}
|
||||
|
||||
pub(super) fn prepare_extension(
|
||||
layout: &Layout,
|
||||
version: &Version,
|
||||
extension: &Extension,
|
||||
destination: &Path,
|
||||
source: &dyn super::ExtensionSource,
|
||||
) -> Result<(), String> {
|
||||
let staging = Download::new(&layout.staging())?;
|
||||
super::release_archive::extract(
|
||||
source.open(version, &extension.id())?,
|
||||
&staging.0,
|
||||
&extension.artifacts,
|
||||
)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let mut entry = entry.map_err(|error| format!("invalid release archive entry: {error}"))?;
|
||||
if !entry.header().entry_type().is_file() {
|
||||
return Err("release archives may contain only regular files".into());
|
||||
}
|
||||
let bytes = entry.path_bytes();
|
||||
let name = std::str::from_utf8(&bytes)
|
||||
.map_err(|_| "release archive path is not UTF-8")?
|
||||
.to_owned();
|
||||
let artifact = expected
|
||||
.get(name.as_str())
|
||||
.ok_or("release archive contains an unsigned file")?;
|
||||
if !seen.insert(name.clone()) || entry.size() != artifact.bytes {
|
||||
return Err("release archive contains a duplicate or wrong-sized file".into());
|
||||
}
|
||||
let path = root.join(&name);
|
||||
fs::create_dir_all(path.parent().expect("artifact has a parent"))
|
||||
.map_err(|error| format!("cannot create release artifact directory: {error}"))?;
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot create release artifact: {error}"))?;
|
||||
let bytes = io::copy(&mut entry, &mut file)
|
||||
.map_err(|error| format!("cannot extract release artifact: {error}"))?;
|
||||
if bytes != artifact.bytes {
|
||||
return Err("release archive contains a truncated file".into());
|
||||
}
|
||||
set_shared_file_permissions(&path, false)?;
|
||||
file.sync_all().map_err(|error| error.to_string())?;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&staging.0, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
if seen.len() != expected.len() {
|
||||
return Err("release archive is missing signed artifacts".into());
|
||||
super::extensions::verify_extension(&staging.0, &extension.artifacts)?;
|
||||
if matches!(extension.kind, super::ExtensionKind::Target) {
|
||||
super::runtime_pack::validate(&staging.0, version.as_str(), extension.target)?;
|
||||
}
|
||||
sync_tree(&staging.0)?;
|
||||
publish(&staging.0, destination)?;
|
||||
sync_directory(destination.parent().ok_or("extension has no parent")?)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
/// Private, independently authenticated installer entry; never accepted over IPC.
|
||||
pub fn extract_config(config: &Path) -> Result<(), String> {
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Extraction {
|
||||
release: PathBuf,
|
||||
archive: PathBuf,
|
||||
trusted_key: PathBuf,
|
||||
}
|
||||
trusted_input(config, false)?;
|
||||
let mut bytes = Vec::new();
|
||||
File::open(config)
|
||||
.map_err(|error| error.to_string())?
|
||||
.take(METADATA_LIMIT + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
if bytes.len() as u64 > METADATA_LIMIT {
|
||||
return Err("extraction config exceeds its size limit".into());
|
||||
}
|
||||
let settings: Extraction = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("invalid extraction config: {error}"))?;
|
||||
trusted_input(&settings.release, true)?;
|
||||
trusted_input(&settings.archive, false)?;
|
||||
trusted_input(&settings.trusted_key, false)?;
|
||||
if settings.trusted_key.starts_with(&settings.release) {
|
||||
return Err("trusted key must be supplied independently outside the release".into());
|
||||
}
|
||||
let key = decode_hex_file(&settings.trusted_key, 32)?;
|
||||
let manifest = MachineManager::signed_manifest(&settings.release, &key)?;
|
||||
let archive = File::open(&settings.archive).map_err(|error| error.to_string())?;
|
||||
super::release_archive::extract(archive, &settings.release, &manifest.artifacts)?;
|
||||
MachineManager::verify_package(&settings.release, Some(&manifest.version), &key)?;
|
||||
sync_tree(&settings.release)
|
||||
}
|
||||
|
||||
fn trusted_input(path: &Path, directory: bool) -> Result<(), String> {
|
||||
if !path.is_absolute()
|
||||
|| path.components().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
std::path::Component::ParentDir | std::path::Component::CurDir
|
||||
)
|
||||
})
|
||||
{
|
||||
return Err("extraction paths must be absolute and normalized".into());
|
||||
}
|
||||
reject_symlink_ancestors(path)?;
|
||||
let metadata = fs::symlink_metadata(path)
|
||||
.map_err(|error| format!("cannot inspect extraction input: {error}"))?;
|
||||
if directory && !metadata.is_dir() || !directory && !metadata.is_file() {
|
||||
return Err("extraction input has the wrong file type".into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
if metadata.uid() != 0
|
||||
|| metadata.mode() & 0o022 != 0
|
||||
|| !directory && metadata.nlink() != 1
|
||||
{
|
||||
return Err(
|
||||
"extraction inputs must be root-owned and not writable by other users".into(),
|
||||
);
|
||||
}
|
||||
for ancestor in path.ancestors().skip(1) {
|
||||
let metadata = fs::symlink_metadata(ancestor).map_err(|error| error.to_string())?;
|
||||
if metadata.uid() != 0 || metadata.mode() & 0o022 != 0 && metadata.mode() & 0o1000 == 0
|
||||
{
|
||||
return Err("extraction input ancestor is not protected by root".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -101,7 +101,13 @@ enum Operation {
|
||||
Clean,
|
||||
ArtifactPut(ArtifactReceipt),
|
||||
ArtifactGet(ArtifactReceipt),
|
||||
Compile { bytes: u64 },
|
||||
EnsureExtension {
|
||||
version: Version,
|
||||
extension: super::ExtensionId,
|
||||
},
|
||||
Compile {
|
||||
bytes: u64,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -325,6 +331,15 @@ pub fn artifact_put(layout: &Layout, bytes: &[u8]) -> Result<ArtifactReceipt, St
|
||||
}
|
||||
|
||||
pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>, String> {
|
||||
artifact_get_optional(layout, sha256, bytes)?
|
||||
.ok_or_else(|| "shared artifact unavailable: artifact is not cached".into())
|
||||
}
|
||||
|
||||
pub fn artifact_get_optional(
|
||||
layout: &Layout,
|
||||
sha256: &str,
|
||||
bytes: u64,
|
||||
) -> Result<Option<Vec<u8>>, String> {
|
||||
let expected = ArtifactReceipt {
|
||||
sha256: sha256.to_owned(),
|
||||
bytes,
|
||||
@@ -341,7 +356,11 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
|
||||
installed: Vec::new(),
|
||||
},
|
||||
)?;
|
||||
require_artifact(read_frame(&mut stream)?, &expected)?;
|
||||
let response: Response = read_frame(&mut stream)?;
|
||||
if response.ok && response.artifact.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
require_artifact(response, &expected)?;
|
||||
let mut bytes = vec![0; expected.bytes as usize];
|
||||
stream
|
||||
.read_exact(&mut bytes)
|
||||
@@ -349,7 +368,7 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
|
||||
if hex(&Sha256::digest(&bytes)) != expected.sha256 {
|
||||
return Err("deverd artifact download failed SHA-256 verification".into());
|
||||
}
|
||||
Ok(bytes)
|
||||
Ok(Some(bytes))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
@@ -358,6 +377,30 @@ pub fn artifact_get(layout: &Layout, sha256: &str, bytes: u64) -> Result<Vec<u8>
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ensure_extension(
|
||||
layout: &Layout,
|
||||
version: &Version,
|
||||
kind: super::ExtensionKind,
|
||||
target: super::BuildTarget,
|
||||
) -> Result<(), String> {
|
||||
Version::parse(version.as_str())?;
|
||||
let response = request(
|
||||
layout,
|
||||
Operation::EnsureExtension {
|
||||
version: version.clone(),
|
||||
extension: super::ExtensionId { kind, target },
|
||||
},
|
||||
&[],
|
||||
)?;
|
||||
if response.ok {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(response
|
||||
.error
|
||||
.unwrap_or_else(|| "extension preparation failed".into()))
|
||||
}
|
||||
}
|
||||
|
||||
fn require_artifact(response: Response, expected: &ArtifactReceipt) -> Result<(), String> {
|
||||
if !response.ok {
|
||||
return Err(response
|
||||
@@ -418,6 +461,11 @@ fn request(
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let (mut stream, token) = connect(layout)?;
|
||||
if matches!(operation, Operation::EnsureExtension { .. }) {
|
||||
stream
|
||||
.set_read_timeout(Some(std::time::Duration::from_secs(620)))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
let request = Request {
|
||||
token,
|
||||
operation,
|
||||
@@ -513,8 +561,16 @@ fn handle(
|
||||
};
|
||||
}
|
||||
Operation::ArtifactGet(expected) => {
|
||||
let mut artifact = match store.artifact(&expected) {
|
||||
Ok(artifact) => artifact,
|
||||
let mut artifact = match store.artifact_optional(&expected) {
|
||||
Ok(Some(artifact)) => artifact,
|
||||
Ok(None) => {
|
||||
return Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
artifact: None,
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
Err(error) => return failure(error),
|
||||
};
|
||||
if let Err(error) = write_frame(stream, &artifact_response(artifact.receipt)) {
|
||||
@@ -530,6 +586,17 @@ fn handle(
|
||||
error: None,
|
||||
};
|
||||
}
|
||||
Operation::EnsureExtension { version, extension } => {
|
||||
return match super::extensions::ensure(layout, &version, &extension) {
|
||||
Ok(()) => Response {
|
||||
ok: true,
|
||||
status: None,
|
||||
artifact: None,
|
||||
error: None,
|
||||
},
|
||||
Err(error) => failure(error),
|
||||
};
|
||||
}
|
||||
Operation::Compile { bytes } => {
|
||||
return match compile::serve(layout, stream, bytes) {
|
||||
Ok(()) => Response {
|
||||
|
||||
@@ -112,13 +112,15 @@ fn verify_pack(
|
||||
target: super::super::BuildTarget,
|
||||
) -> Result<(), String> {
|
||||
let root = compiler
|
||||
.core
|
||||
.parent()
|
||||
.ok_or("installed compiler has no directory")?;
|
||||
.resources
|
||||
.root(&format!("runtime/{}/manifest.json", target.platform()))?;
|
||||
let artifacts =
|
||||
super::super::runtime_pack::validate(root, compiler.manifest.version.as_str(), target)?;
|
||||
super::super::runtime_pack::validate(&root, compiler.manifest.version.as_str(), target)?;
|
||||
let signed_artifacts = compiler
|
||||
.resources
|
||||
.artifacts(&format!("runtime/{}/", target.platform()))?;
|
||||
for input in artifacts {
|
||||
if !compiler.manifest.artifacts.iter().any(|signed| {
|
||||
if !signed_artifacts.iter().any(|signed| {
|
||||
signed.path == input.path
|
||||
&& signed.bytes == input.bytes
|
||||
&& signed.sha256 == input.sha256
|
||||
|
||||
Reference in new issue
Block a user