feat: ship slim releases with shared on-demand extensions and locked restore
This commit is contained in:
1 parent
83b78a0db6
commit
4fda7c8c1a
58 files changed
+3774
-425
No files matched your search
@@ -63,7 +63,7 @@ pub struct Receipt {
|
||||
pub frontend: String,
|
||||
pub python: Option<PythonSystem>,
|
||||
pub dynamic_requires: Vec<String>,
|
||||
pub dependencies: Box<LockFile>,
|
||||
pub inputs: Box<LockFile>,
|
||||
pub config_settings: BTreeMap<String, serde_json::Value>,
|
||||
pub output: LockedArtifact,
|
||||
}
|
||||
@@ -90,6 +90,29 @@ pub struct Session<'a> {
|
||||
}
|
||||
|
||||
impl<'a> Session<'a> {
|
||||
pub(crate) fn replay(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn super::ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
if receipt.frontend != frontend(&receipt.ecosystem)? {
|
||||
return Err("locked build frontend differs from this toolchain".into());
|
||||
}
|
||||
let bytes = match receipt.ecosystem {
|
||||
Ecosystem::Pip => self.replay_python(receipt, store)?,
|
||||
Ecosystem::Npm => self.replay_npm(receipt, store)?,
|
||||
Ecosystem::Go => return Err("Go source build receipts are unsupported".into()),
|
||||
};
|
||||
if bytes.len() as u64 != receipt.output.bytes
|
||||
|| super::sha256(&bytes) != receipt.output.sha256
|
||||
{
|
||||
return Err("fixed build replay is not byte reproducible: output differs from dever.lock; lock was not changed".into());
|
||||
}
|
||||
if store.publish(&bytes, &receipt.output.target)? != receipt.output.sha256 {
|
||||
return Err("restored build cache identity mismatch".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn new(inputs: &'a dyn Inputs) -> Self {
|
||||
Self {
|
||||
inputs,
|
||||
@@ -229,6 +252,9 @@ pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
}
|
||||
|
||||
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if lock.format != super::LOCK_FORMAT || lock.toolchain != super::TOOLCHAIN {
|
||||
return Err("build input lock format or toolchain differs from this release".into());
|
||||
}
|
||||
if depth > 16 {
|
||||
return Err("build dependency depth exceeds limit".into());
|
||||
}
|
||||
@@ -361,7 +387,6 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
&& (receipt.python.is_some()
|
||||
|| receipt.python_markers.is_some()
|
||||
|| !receipt.dynamic_requires.is_empty()
|
||||
|| !receipt.dependencies.libs.is_empty()
|
||||
|| receipt.graph.is_none())
|
||||
{
|
||||
return Err("npm build receipt contains Python backend state".into());
|
||||
@@ -372,20 +397,50 @@ fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if let Some(system) = &receipt.python {
|
||||
validate_requirements(receipt, system)?;
|
||||
}
|
||||
if !receipt.dependencies.workers.is_empty()
|
||||
|| !receipt.dependencies.packages.is_empty()
|
||||
if !receipt.inputs.workers.is_empty()
|
||||
|| !receipt.inputs.packages.is_empty()
|
||||
|| receipt
|
||||
.dependencies
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
|
||||
.any(|lib| lib.spec.ecosystem != receipt.ecosystem)
|
||||
{
|
||||
return Err("build dependency closure is not a separate Python environment".into());
|
||||
return Err("build input closure contains unrelated environments".into());
|
||||
}
|
||||
doctor_at(&receipt.dependencies, depth + 1)?;
|
||||
if receipt.output.source.is_some()
|
||||
|| receipt.sources.iter().any(|source| {
|
||||
source
|
||||
.source
|
||||
.as_ref()
|
||||
.is_none_or(|origin| origin.ecosystem != receipt.ecosystem)
|
||||
})
|
||||
{
|
||||
return Err("build receipt lacks exact registry source identity".into());
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Npm {
|
||||
let mut sources = receipt
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| lib.artifacts.clone())
|
||||
.collect::<Vec<_>>();
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
if sources != receipt.sources
|
||||
|| receipt.inputs.npm.len() != 1
|
||||
|| receipt.inputs.npm[0].roots != receipt.roots
|
||||
|| receipt.inputs.npm[0].build.is_some()
|
||||
|| !receipt.inputs.builds.is_empty()
|
||||
{
|
||||
return Err("npm build input graph differs from its original sources".into());
|
||||
}
|
||||
} else if receipt.sources.len() != 1 || !receipt.inputs.npm.is_empty() {
|
||||
return Err("Python build receipt has invalid source inputs".into());
|
||||
}
|
||||
doctor_at(&receipt.inputs, depth + 1)?;
|
||||
// Full graph checks belong to the ordinary lock owner; recursion is
|
||||
// bounded above before invoking that owner on a dependency lock.
|
||||
let mut dependencies = (*receipt.dependencies).clone();
|
||||
let mut dependencies = (*receipt.inputs).clone();
|
||||
dependencies.builds.clear();
|
||||
for lib in &mut dependencies.libs {
|
||||
lib.build = None;
|
||||
@@ -447,7 +502,7 @@ fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(),
|
||||
let document = serde_json::json!({"info":{"requires_dist":requirements}});
|
||||
let expected = super::python_dependencies(&document, markers, &[])?;
|
||||
let libraries = receipt
|
||||
.dependencies
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| (lib.spec.name.as_str(), lib))
|
||||
|
||||
@@ -12,6 +12,43 @@ mod environment;
|
||||
use environment::{Environment, Output};
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn replay_npm(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn crate::libs::ArtifactStore,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let (runtime, _) = self.inputs.runtime(Ecosystem::Npm)?;
|
||||
if runtime.pack != receipt.runtime {
|
||||
return Err("locked npm build runtime changed".into());
|
||||
}
|
||||
let environment = Environment::new(self.inputs, &runtime)?;
|
||||
if environment.descriptor != receipt.tools
|
||||
|| Some(&environment.python_runtime) != receipt.auxiliary_runtime.as_ref()
|
||||
{
|
||||
return Err("locked npm build tools or auxiliary runtime changed".into());
|
||||
}
|
||||
let mut graph = receipt
|
||||
.inputs
|
||||
.npm
|
||||
.first()
|
||||
.ok_or("npm build input graph missing")?
|
||||
.clone();
|
||||
let (files, executables) = installation_inputs(store, &receipt.inputs, &graph)?;
|
||||
let output = environment.run(self.inputs, &graph, &files, &executables)?;
|
||||
prune_failed(&mut graph, &output.failed)?;
|
||||
npm::validate_built_files(&graph, &output.files)?;
|
||||
if Some(&graph) != receipt.graph.as_ref()
|
||||
|| output.native != receipt.native_entries
|
||||
|| output.rejections != receipt.native_rejections
|
||||
{
|
||||
return Err(
|
||||
"npm fixed build replay changed graph, optional outcomes or native admission"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
archive(&output)
|
||||
}
|
||||
|
||||
pub(crate) fn npm(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
@@ -21,26 +58,7 @@ impl Session<'_> {
|
||||
) -> Result<(), String> {
|
||||
let mut lock = LockFile::new(libraries.clone())?;
|
||||
lock.npm.push(graph.clone());
|
||||
let archives = libraries
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
resolver.store.verify_exact(
|
||||
&artifact.sha256,
|
||||
artifact.bytes,
|
||||
&artifact.target,
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, &lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
let (files, executables) = installation_inputs(resolver.store, &lock, graph)?;
|
||||
if !npm::requires_build(&files)? {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -66,7 +84,6 @@ impl Session<'_> {
|
||||
retain_archives(libraries, graph);
|
||||
return Ok(());
|
||||
}
|
||||
let executables = npm::build_executables(graph, &archives)?;
|
||||
let environment = Environment::new(self.inputs, runtime)?;
|
||||
let output = environment.run(self.inputs, graph, &files, &executables)?;
|
||||
prune_failed(graph, &output.failed)?;
|
||||
@@ -82,6 +99,7 @@ impl Session<'_> {
|
||||
path: format!("build/npm/{digest}/installation.tgz"),
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: digest,
|
||||
source: None,
|
||||
};
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
@@ -97,7 +115,7 @@ impl Session<'_> {
|
||||
frontend: super::frontend(&Ecosystem::Npm)?,
|
||||
python: None,
|
||||
dynamic_requires: vec![],
|
||||
dependencies: Box::new(LockFile::new(vec![])?),
|
||||
inputs: Box::new(lock),
|
||||
config_settings: BTreeMap::new(),
|
||||
output: output_artifact,
|
||||
};
|
||||
@@ -114,6 +132,33 @@ impl Session<'_> {
|
||||
}
|
||||
}
|
||||
|
||||
type InstallationInputs = (Vec<(String, Vec<u8>)>, BTreeSet<String>);
|
||||
|
||||
fn installation_inputs(
|
||||
store: &dyn crate::libs::ArtifactStore,
|
||||
lock: &LockFile,
|
||||
graph: &npm::Environment,
|
||||
) -> Result<InstallationInputs, String> {
|
||||
let archives = lock
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
Ok((files, npm::build_executables(graph, &archives)?))
|
||||
}
|
||||
|
||||
fn archive(output: &Output) -> Result<Vec<u8>, String> {
|
||||
let encoder = flate2::GzBuilder::new()
|
||||
.mtime(0)
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
|
||||
use crate::libs::{
|
||||
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
|
||||
sha256,
|
||||
ArtifactStore, Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime,
|
||||
RegistrySource, archive_files, sha256,
|
||||
};
|
||||
use crate::workers::{python_wheel::Wheel, runtime};
|
||||
use serde_json::{Value, json};
|
||||
@@ -23,6 +23,12 @@ struct Environment {
|
||||
descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
struct Execution<'a> {
|
||||
store: &'a dyn ArtifactStore,
|
||||
runtime: &'a RegistryRuntime,
|
||||
inputs: &'a dyn super::Inputs,
|
||||
}
|
||||
|
||||
struct Active<'a> {
|
||||
session: &'a Session<'a>,
|
||||
key: String,
|
||||
@@ -34,12 +40,69 @@ impl Drop for Active<'_> {
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn replay_python(
|
||||
&self,
|
||||
receipt: &Receipt,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let (runtime, _) = self.inputs.runtime(Ecosystem::Pip)?;
|
||||
if runtime.pack != receipt.runtime || runtime.python_markers != receipt.python_markers {
|
||||
return Err("locked Python build runtime or marker policy changed".into());
|
||||
}
|
||||
let environment = Environment::new(self, &runtime)?;
|
||||
let execution = Execution {
|
||||
store,
|
||||
runtime: &runtime,
|
||||
inputs: self.inputs,
|
||||
};
|
||||
if environment.descriptor != receipt.tools {
|
||||
return Err("locked Python build tools changed".into());
|
||||
}
|
||||
let artifact = receipt
|
||||
.sources
|
||||
.first()
|
||||
.ok_or("Python build source missing")?;
|
||||
let origin = artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or("Python build source locator missing")?;
|
||||
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let source = source_files(&origin.filename, &bytes)?;
|
||||
let system = receipt
|
||||
.python
|
||||
.as_ref()
|
||||
.ok_or("Python build backend missing")?;
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected = environment.run(
|
||||
&execution,
|
||||
&empty,
|
||||
&source,
|
||||
json!({"phase":"inspect"}),
|
||||
None,
|
||||
)?;
|
||||
let actual: PythonSystem =
|
||||
serde_json::from_value(inspected.response).map_err(|error| error.to_string())?;
|
||||
if &actual != system {
|
||||
return Err("locked Python build backend configuration changed".into());
|
||||
}
|
||||
// The locked closure is final. Replaying never asks the resolver to
|
||||
// discover or select static/dynamic build requirements again.
|
||||
let (_, bytes) = environment.wheel(
|
||||
&execution,
|
||||
&receipt.inputs,
|
||||
&source,
|
||||
system,
|
||||
receipt.roots.first().ok_or("Python build root missing")?,
|
||||
)?;
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn python(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
spec: &LibSpec,
|
||||
runtime: &RegistryRuntime,
|
||||
filename: &str,
|
||||
source_origin: RegistrySource,
|
||||
source: &[u8],
|
||||
) -> Result<Product, String> {
|
||||
let key = format!(
|
||||
@@ -62,11 +125,21 @@ impl Session<'_> {
|
||||
session: self,
|
||||
key: key.clone(),
|
||||
};
|
||||
let sources = source_files(filename, source)?;
|
||||
let sources = source_files(&source_origin.filename, source)?;
|
||||
let environment = Environment::new(self, runtime)?;
|
||||
let execution = Execution {
|
||||
store: resolver.store,
|
||||
runtime,
|
||||
inputs: self.inputs,
|
||||
};
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected =
|
||||
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
|
||||
let inspected = environment.run(
|
||||
&execution,
|
||||
&empty,
|
||||
&sources,
|
||||
json!({"phase":"inspect"}),
|
||||
None,
|
||||
)?;
|
||||
let system: PythonSystem = serde_json::from_value(inspected.response)
|
||||
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
|
||||
let mut requirements = system.requires.clone();
|
||||
@@ -75,7 +148,7 @@ impl Session<'_> {
|
||||
let mut discovered = BTreeSet::new();
|
||||
for _ in 0..8 {
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&execution,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"requires","system":system}),
|
||||
@@ -99,38 +172,8 @@ impl Session<'_> {
|
||||
dependencies = next;
|
||||
}
|
||||
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
|
||||
let metadata = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
let (filename, bytes) =
|
||||
environment.wheel(&execution, &dependencies, &sources, &system, spec)?;
|
||||
let output = artifact(
|
||||
runtime,
|
||||
format!(
|
||||
@@ -140,11 +183,12 @@ impl Session<'_> {
|
||||
),
|
||||
&bytes,
|
||||
);
|
||||
let source_artifact = artifact(
|
||||
let mut source_artifact = artifact(
|
||||
runtime,
|
||||
format!("build/source/{}/source", sha256(source)),
|
||||
source,
|
||||
);
|
||||
source_artifact.source = Some(source_origin);
|
||||
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
|
||||
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
|
||||
return Err("build artifact cache identity mismatch".into());
|
||||
@@ -165,13 +209,13 @@ impl Session<'_> {
|
||||
frontend: super::frontend(&Ecosystem::Pip)?,
|
||||
python: Some(system),
|
||||
dynamic_requires: dynamic,
|
||||
dependencies: Box::new(dependencies),
|
||||
inputs: Box::new(dependencies),
|
||||
config_settings: BTreeMap::new(),
|
||||
output,
|
||||
};
|
||||
let product = Product {
|
||||
bytes,
|
||||
filename: filename.into(),
|
||||
filename,
|
||||
receipt,
|
||||
};
|
||||
self.remember(key, product.clone())?;
|
||||
@@ -185,6 +229,7 @@ fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArti
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -236,6 +281,54 @@ struct ResultFiles {
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
fn wheel(
|
||||
&self,
|
||||
execution: &Execution<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
system: &PythonSystem,
|
||||
spec: &LibSpec,
|
||||
) -> Result<(String, Vec<u8>), String> {
|
||||
let metadata = self.run(
|
||||
execution,
|
||||
dependencies,
|
||||
source,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = self.run(
|
||||
execution,
|
||||
dependencies,
|
||||
source,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(
|
||||
spec,
|
||||
Some(filename),
|
||||
&execution.runtime.python_wheel_tags,
|
||||
&bytes,
|
||||
)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
Ok((filename.into(), bytes))
|
||||
}
|
||||
|
||||
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
|
||||
super::require_execution_target(&requested.target, "Python source build")?;
|
||||
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
|
||||
@@ -291,12 +384,17 @@ impl Environment {
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
execution: &Execution<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
mut request: Value,
|
||||
prepared: Option<&BTreeMap<String, Vec<u8>>>,
|
||||
) -> Result<ResultFiles, String> {
|
||||
let Execution {
|
||||
store,
|
||||
runtime,
|
||||
inputs,
|
||||
} = execution;
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
@@ -322,10 +420,6 @@ impl Environment {
|
||||
}
|
||||
}
|
||||
let mut installed = BTreeMap::<String, Vec<u8>>::new();
|
||||
let runtime = resolver
|
||||
.runtimes
|
||||
.get(&Ecosystem::Pip)
|
||||
.ok_or("build dependency runtime is missing")?;
|
||||
let mut distributions = BTreeSet::new();
|
||||
for lib in &dependencies.libs {
|
||||
super::validate_python_policy(
|
||||
@@ -342,10 +436,7 @@ impl Environment {
|
||||
);
|
||||
}
|
||||
let artifact = &lib.artifacts[0];
|
||||
let bytes =
|
||||
resolver
|
||||
.store
|
||||
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let bytes = store.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
|
||||
wheel.validate_dependencies(
|
||||
lib,
|
||||
@@ -432,9 +523,7 @@ impl Environment {
|
||||
"isolated Python build hook",
|
||||
&work,
|
||||
)?;
|
||||
if let Some(session) = resolver.build {
|
||||
session.inputs.diagnostic(&stage.diagnostics()?);
|
||||
}
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
let outputs = output_files(&work.join("output"))?;
|
||||
let response = serde_json::from_slice(
|
||||
outputs
|
||||
|
||||
@@ -10,7 +10,7 @@ pub(crate) struct ProjectMutation {
|
||||
|
||||
impl ProjectMutation {
|
||||
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
|
||||
if !matches!(command, "add" | "update" | "remove") {
|
||||
if !matches!(command, "add" | "update" | "remove" | "install") {
|
||||
return Ok(None);
|
||||
}
|
||||
let directory = fs::symlink_metadata(root)
|
||||
|
||||
@@ -19,7 +19,7 @@ pub mod sumdb;
|
||||
|
||||
use super::{
|
||||
ArtifactStore, Ecosystem, LibResolver, LibSpec, LockFile, LockedArtifact, LockedDependency,
|
||||
LockedLib, ProviderManifest, RuntimePack, sha256, validate_artifact,
|
||||
LockedLib, ProviderManifest, RegistrySource, RuntimePack, sha256, validate_artifact,
|
||||
};
|
||||
|
||||
const MAX_METADATA_BYTES: usize = 16 * 1024 * 1024;
|
||||
@@ -553,7 +553,7 @@ impl RegistryResolver<'_> {
|
||||
runtime,
|
||||
"whl",
|
||||
&candidate.bytes,
|
||||
"pypi-wheel-v1",
|
||||
candidate.source.clone(),
|
||||
)?;
|
||||
lib.build = candidate.build.clone();
|
||||
distributions.insert(base.to_owned(), lib.clone());
|
||||
@@ -925,6 +925,11 @@ impl RegistryResolver<'_> {
|
||||
}
|
||||
Ok(Some(PythonCandidate {
|
||||
build: None,
|
||||
source: Some(RegistrySource {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
locator: url.into(),
|
||||
filename: field(file, "filename")?.into(),
|
||||
}),
|
||||
bytes,
|
||||
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
|
||||
}))
|
||||
@@ -965,7 +970,17 @@ impl RegistryResolver<'_> {
|
||||
if sha256(&bytes) != expected {
|
||||
return Err("PyPI source SHA-256 mismatch".into());
|
||||
}
|
||||
let product = session.python(self, spec, runtime, field(file, "filename")?, &bytes)?;
|
||||
let product = session.python(
|
||||
self,
|
||||
spec,
|
||||
runtime,
|
||||
RegistrySource {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
locator: field(file, "url")?.into(),
|
||||
filename: field(file, "filename")?.into(),
|
||||
},
|
||||
&bytes,
|
||||
)?;
|
||||
let wheel = crate::workers::python_wheel::Wheel::parse(
|
||||
spec,
|
||||
Some(&product.filename),
|
||||
@@ -980,6 +995,7 @@ impl RegistryResolver<'_> {
|
||||
}
|
||||
Ok(Some(PythonCandidate {
|
||||
bytes: product.bytes,
|
||||
source: None,
|
||||
info: serde_json::json!({"requires_dist":wheel.requirements,"provides_extra":wheel.extras}),
|
||||
build: Some(product.receipt.identity()?),
|
||||
}))
|
||||
@@ -1012,7 +1028,7 @@ impl RegistryResolver<'_> {
|
||||
runtime: &RegistryRuntime,
|
||||
extension: &str,
|
||||
bytes: &[u8],
|
||||
schema: &str,
|
||||
source: Option<RegistrySource>,
|
||||
) -> Result<LockedLib, String> {
|
||||
let files = archive_files(extension, bytes)?;
|
||||
if spec.ecosystem == Ecosystem::Go {
|
||||
@@ -1035,6 +1051,7 @@ impl RegistryResolver<'_> {
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
source,
|
||||
};
|
||||
validate_artifact(&artifact)?;
|
||||
let published = self.store.publish(bytes, &runtime.target)?;
|
||||
@@ -1052,7 +1069,12 @@ impl RegistryResolver<'_> {
|
||||
dependencies,
|
||||
runtime: runtime.pack.clone(),
|
||||
artifacts: vec![artifact],
|
||||
schema: schema.into(),
|
||||
schema: match spec.ecosystem {
|
||||
Ecosystem::Pip => "pypi-wheel-v1",
|
||||
Ecosystem::Npm => "npm-instances-v1",
|
||||
Ecosystem::Go => "go-sumdb-v1",
|
||||
}
|
||||
.into(),
|
||||
build: None,
|
||||
})
|
||||
}
|
||||
@@ -1126,7 +1148,22 @@ impl RegistryResolver<'_> {
|
||||
MAX_ARTIFACT_BYTES,
|
||||
)?;
|
||||
sumdb.verify_zip(name, version, &zip)?;
|
||||
self.lock_archive(&spec, dependencies, runtime, "zip", &zip, "go-sumdb-v1")
|
||||
self.lock_archive(
|
||||
&spec,
|
||||
dependencies,
|
||||
runtime,
|
||||
"zip",
|
||||
&zip,
|
||||
Some(RegistrySource {
|
||||
ecosystem: Ecosystem::Go,
|
||||
locator: format!(
|
||||
"/{}/@v/v{}.zip",
|
||||
go_path(name),
|
||||
version.trim_start_matches('v')
|
||||
),
|
||||
filename: format!("v{}.zip", version.trim_start_matches('v')),
|
||||
}),
|
||||
)
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
return Ok((libs, sumdb.finish()));
|
||||
@@ -1179,6 +1216,7 @@ struct PythonCandidate {
|
||||
bytes: Vec<u8>,
|
||||
info: Value,
|
||||
build: Option<String>,
|
||||
source: Option<RegistrySource>,
|
||||
}
|
||||
|
||||
struct SearchState<'a> {
|
||||
|
||||
@@ -128,6 +128,18 @@ impl Catalog<'_, '_> {
|
||||
"npm archive cache exceeds its 256 MiB/package budget".into(),
|
||||
));
|
||||
}
|
||||
let locator = document
|
||||
.get("dist")
|
||||
.and_then(|dist| dist.get("tarball"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.ok_or_else(|| SearchError::Fatal("npm tarball locator missing".into()))?;
|
||||
let filename = locator
|
||||
.split(['?', '#'])
|
||||
.next()
|
||||
.unwrap_or(locator)
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.ok_or_else(|| SearchError::Fatal("npm tarball filename missing".into()))?;
|
||||
let lib = self
|
||||
.resolver
|
||||
.lock_archive(
|
||||
@@ -136,7 +148,11 @@ impl Catalog<'_, '_> {
|
||||
self.runtime,
|
||||
"tgz",
|
||||
&bytes,
|
||||
"npm-instances-v1",
|
||||
Some(crate::libs::RegistrySource {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
locator: locator.into(),
|
||||
filename: filename.into(),
|
||||
}),
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.archives.insert(spec.clone(), (lib, archive));
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
//! Exact restoration is a preparation operation, never a dependency resolver.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
use super::{
|
||||
ArtifactStore, FixtureRegistry, LockFile, LockedArtifact, RegistryTransport, build, sumdb,
|
||||
};
|
||||
|
||||
pub(super) fn install(
|
||||
project_root: &Path,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
) -> Result<super::PreparationReport, String> {
|
||||
super::reject_declaration_symlinks(project_root)?;
|
||||
let lock_bytes = fs::read(project_root.join("dever.lock"))
|
||||
.map_err(|error| format!("dever lib install requires an existing dever.lock: {error}"))?;
|
||||
let setting_bytes = setting_snapshot(project_root)?;
|
||||
let lock = LockFile::decode(&lock_bytes)?;
|
||||
super::doctor(&lock)?;
|
||||
validate_target(&lock, target.platform())?;
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
if lock.packages.is_empty()
|
||||
&& lock
|
||||
.libs
|
||||
.iter()
|
||||
.all(|lib| fixtures.package(&lib.spec).is_some())
|
||||
&& lock.workers.iter().all(|worker| worker.ecosystem == "exec")
|
||||
{
|
||||
let store = super::FixtureArtifactStore::default();
|
||||
validate_project(project_root, &lock, &store, target, false)?;
|
||||
restore_fixtures(&lock, &store)?;
|
||||
} else {
|
||||
let inputs = super::PreparingRegistry(super::InstalledRegistry::current(target)?);
|
||||
let store = super::ManagedArtifactStore {
|
||||
layout: inputs.0.layout.clone(),
|
||||
};
|
||||
crate::packages::restore_locked(project_root, &lock, &store)?;
|
||||
validate_project(project_root, &lock, &store, target, true)?;
|
||||
for worker in lock
|
||||
.workers
|
||||
.iter()
|
||||
.filter(|worker| worker.ecosystem != "exec")
|
||||
{
|
||||
let (runtime, _) = build::Inputs::runtime(&inputs, worker.ecosystem.parse()?)?;
|
||||
if worker.runtime.as_ref() != Some(&runtime.pack) {
|
||||
return Err("locked Worker runtime differs from signed release".into());
|
||||
}
|
||||
}
|
||||
if target != crate::toolchain::BuildTarget::host()? && !lock.workers.is_empty() {
|
||||
inputs.ensure(crate::toolchain::ExtensionKind::Target)?;
|
||||
}
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
&super::HttpRegistry::official(),
|
||||
&store,
|
||||
&inputs,
|
||||
target.platform(),
|
||||
&sumdb::Verifier::official(),
|
||||
)?;
|
||||
super::embedded_resources_for_target(project_root, target)?;
|
||||
}
|
||||
if fs::read(project_root.join("dever.lock")).map_err(|error| error.to_string())? != lock_bytes
|
||||
|| setting_snapshot(project_root)? != setting_bytes
|
||||
{
|
||||
return Err("project setting or lock changed during exact restoration".into());
|
||||
}
|
||||
let artifacts = lock
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| &lib.artifacts)
|
||||
.chain(build::npm_outputs(&lock).map(|receipt| &receipt.output))
|
||||
.collect::<BTreeSet<_>>();
|
||||
Ok(super::PreparationReport {
|
||||
libs: lock.libs.len(),
|
||||
artifacts: artifacts.len(),
|
||||
runtimes: lock
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| &lib.runtime)
|
||||
.chain(
|
||||
lock.workers
|
||||
.iter()
|
||||
.filter_map(|worker| worker.runtime.as_ref()),
|
||||
)
|
||||
.collect::<BTreeSet<_>>()
|
||||
.len(),
|
||||
artifact_bytes: artifacts.iter().map(|artifact| artifact.bytes).sum(),
|
||||
})
|
||||
}
|
||||
|
||||
fn setting_snapshot(project_root: &Path) -> Result<Option<Vec<u8>>, String> {
|
||||
match fs::read(project_root.join("config/setting.json")) {
|
||||
Ok(bytes) => Ok(Some(bytes)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(error) => Err(format!("cannot read project setting snapshot: {error}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_project(
|
||||
project_root: &Path,
|
||||
lock: &LockFile,
|
||||
store: &dyn ArtifactStore,
|
||||
target: crate::toolchain::BuildTarget,
|
||||
prepare: bool,
|
||||
) -> Result<(), String> {
|
||||
let sources = crate::packages::sources_with(project_root, store)?;
|
||||
let workers = super::unbound_source_workers(project_root, &sources)?;
|
||||
let mut unbound = lock.clone();
|
||||
for worker in &mut unbound.workers {
|
||||
worker.runtime = None;
|
||||
}
|
||||
super::validate_worker_bindings(&unbound, &workers)?;
|
||||
let mut roots = super::read_declarations(project_root)?;
|
||||
roots.extend(crate::packages::declared_libs_with(project_root, store)?);
|
||||
if super::retain_locked_libs(lock, &roots, lock.workers.clone())? != *lock {
|
||||
return Err("dever.lock contains Lib entries or environments outside the declared closure; run 'dever lib update <project-root>'".into());
|
||||
}
|
||||
let workers = super::bind_workers(workers, target, prepare)?;
|
||||
super::validate_worker_bindings(lock, &workers)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_target(lock: &LockFile, target: &str) -> Result<(), String> {
|
||||
for artifact in lock.libs.iter().flat_map(|lib| &lib.artifacts).chain(
|
||||
lock.builds
|
||||
.iter()
|
||||
.flat_map(|receipt| receipt.sources.iter().chain([&receipt.output])),
|
||||
) {
|
||||
if artifact.target != target && artifact.target != "host" {
|
||||
return Err(format!(
|
||||
"locked artifact targets {}, not {target}; install never switches lock targets",
|
||||
artifact.target
|
||||
));
|
||||
}
|
||||
}
|
||||
for receipt in &lock.builds {
|
||||
validate_target(&receipt.inputs, target)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_fixtures(lock: &LockFile, store: &dyn ArtifactStore) -> Result<(), String> {
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
for lib in &lock.libs {
|
||||
if let Some(package) = fixtures.package(&lib.spec) {
|
||||
if package.locked() != *lib {
|
||||
return Err("locked fixture metadata differs from builtin package".into());
|
||||
}
|
||||
for artifact in &package.artifacts {
|
||||
let expected = super::sha256(&artifact.bytes);
|
||||
if store.publish(&artifact.bytes, &artifact.target)? != expected {
|
||||
return Err("fixture artifact cache identity mismatch".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shared by production preparation and isolated registry fixtures. Every
|
||||
/// network request names the recorded archive; no metadata/version query runs.
|
||||
pub fn restore_artifacts(
|
||||
lock: &LockFile,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
inputs: &dyn build::Inputs,
|
||||
target: &str,
|
||||
verifier: &sumdb::Verifier,
|
||||
) -> Result<(), String> {
|
||||
super::doctor_with_go_verifier(lock, verifier)?;
|
||||
validate_target(lock, target)?;
|
||||
let session = build::Session::new(inputs);
|
||||
restore_at(lock, transport, store, inputs, &session)?;
|
||||
super::verify_locked_artifacts_with_go_verifier(lock, store, target, verifier)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_at(
|
||||
lock: &LockFile,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
inputs: &dyn build::Inputs,
|
||||
session: &build::Session<'_>,
|
||||
) -> Result<(), String> {
|
||||
restore_fixtures(lock, store)?;
|
||||
let fixtures = FixtureRegistry::builtin();
|
||||
for lib in &lock.libs {
|
||||
if fixtures.package(&lib.spec).is_some() {
|
||||
continue;
|
||||
}
|
||||
let (runtime, _) = inputs.runtime(lib.spec.ecosystem.clone())?;
|
||||
if runtime.pack != lib.runtime
|
||||
|| lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.target != runtime.target)
|
||||
{
|
||||
return Err(format!(
|
||||
"{} locked runtime differs from signed release",
|
||||
lib.spec.key()
|
||||
));
|
||||
}
|
||||
if let Some(markers) = &runtime.python_markers {
|
||||
build::validate_python_policy(lock, lib, markers)?;
|
||||
}
|
||||
for artifact in &lib.artifacts {
|
||||
if artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.is_some_and(|source| source.ecosystem != lib.spec.ecosystem)
|
||||
{
|
||||
return Err("locked registry source ecosystem differs from its Lib".into());
|
||||
}
|
||||
if artifact.source.is_some() {
|
||||
restore_archive(artifact, transport, store)?;
|
||||
} else if lib.build.is_none() {
|
||||
return Err(format!("{} lacks an exact registry source", lib.spec.key()));
|
||||
}
|
||||
}
|
||||
}
|
||||
for receipt in &lock.builds {
|
||||
if cached(&receipt.output, store)? {
|
||||
continue;
|
||||
}
|
||||
restore_at(&receipt.inputs, transport, store, inputs, session)?;
|
||||
for source in &receipt.sources {
|
||||
restore_archive(source, transport, store)?;
|
||||
}
|
||||
session.replay(receipt, store)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cached(artifact: &LockedArtifact, store: &dyn ArtifactStore) -> Result<bool, String> {
|
||||
match store.get_exact(&artifact.sha256, artifact.bytes, &artifact.target)? {
|
||||
Some(bytes) => {
|
||||
validate_bytes(artifact, &bytes)?;
|
||||
Ok(true)
|
||||
}
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_bytes(artifact: &LockedArtifact, bytes: &[u8]) -> Result<(), String> {
|
||||
if bytes.len() as u64 != artifact.bytes || super::sha256(bytes) != artifact.sha256 {
|
||||
return Err(format!(
|
||||
"locked archive '{}' failed exact byte/hash verification",
|
||||
artifact.path
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn restore_archive(
|
||||
artifact: &LockedArtifact,
|
||||
transport: &dyn RegistryTransport,
|
||||
store: &dyn ArtifactStore,
|
||||
) -> Result<(), String> {
|
||||
if cached(artifact, store)? {
|
||||
return Ok(());
|
||||
}
|
||||
let source = artifact
|
||||
.source
|
||||
.as_ref()
|
||||
.ok_or("locked archive source is absent")?;
|
||||
let limit =
|
||||
usize::try_from(artifact.bytes).map_err(|_| "locked archive exceeds address space")?;
|
||||
if limit > 64 * 1024 * 1024 {
|
||||
return Err("locked registry archive exceeds byte budget".into());
|
||||
}
|
||||
let bytes = transport.get(source.ecosystem.clone(), &source.locator, limit)?;
|
||||
validate_bytes(artifact, &bytes)?;
|
||||
if store.publish(&bytes, &artifact.target)? != artifact.sha256 {
|
||||
return Err("restored archive cache identity mismatch".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user