feat: publish Dever core with project scaffolding and versioned AI skills

This commit is contained in:
Antigravity committed 2026-10-04 13:37:30 +00:00
1 parent 70fa73cdc2
commit ac458ffa7f
960 files changed
+221319 -1163

No files matched your search

+526
View File
@@ -0,0 +1,526 @@
//! Explicit Lib preparation: shared owned staging and bounded tool execution.
mod npm;
pub mod pack;
pub(crate) mod process;
mod python;
use super::{Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryRuntime, RuntimePack};
use serde::{Deserialize, Serialize};
use std::cell::RefCell;
use std::collections::{BTreeMap, BTreeSet};
fn require_execution_target(target: &str, operation: &str) -> Result<(), String> {
if target != crate::toolchain::BuildTarget::host()?.platform() {
return Err(format!(
"{operation} for {target} requires prepared target output and a complete build receipt; prepare the signed target inputs in a matching isolated execution environment before the offline build"
));
}
Ok(())
}
/// Production supplies only authenticated release resources. Explicit author
/// tests inject the same immutable inputs without creating a host-tool fallback.
pub trait Inputs {
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String>;
fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String>;
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String>;
/// Author tools may display bounded hook output without retaining temporary
/// staging trees or putting diagnostics in reproducible build identities.
fn diagnostic(&self, _output: &[u8]) {}
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct PythonSystem {
pub requires: Vec<String>,
pub backend: String,
pub backend_path: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NativeRejection {
Format,
Target,
Closure,
Unloadable,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Receipt {
pub ecosystem: Ecosystem,
pub roots: Vec<LibSpec>,
pub sources: Vec<LockedArtifact>,
pub graph: Option<super::npm::Environment>,
pub runtime: RuntimePack,
pub auxiliary_runtime: Option<RuntimePack>,
pub native_entries: BTreeSet<String>,
pub native_rejections: BTreeMap<String, NativeRejection>,
pub python_markers: Option<pep508_rs::MarkerEnvironment>,
pub tools: pack::Descriptor,
pub frontend: String,
pub python: Option<PythonSystem>,
pub dynamic_requires: Vec<String>,
pub dependencies: Box<LockFile>,
pub config_settings: BTreeMap<String, serde_json::Value>,
pub output: LockedArtifact,
}
impl Receipt {
pub fn identity(&self) -> Result<String, String> {
serde_json::to_vec(self)
.map(|bytes| super::sha256(&bytes))
.map_err(|error| error.to_string())
}
}
#[derive(Clone)]
pub(crate) struct Product {
pub bytes: Vec<u8>,
pub filename: String,
pub receipt: Receipt,
}
pub struct Session<'a> {
inputs: &'a dyn Inputs,
active: RefCell<BTreeSet<String>>,
products: RefCell<BTreeMap<String, Product>>,
}
impl<'a> Session<'a> {
pub fn new(inputs: &'a dyn Inputs) -> Self {
Self {
inputs,
active: RefCell::new(BTreeSet::new()),
products: RefCell::new(BTreeMap::new()),
}
}
pub(crate) fn receipts(&self, identities: &BTreeSet<String>) -> Result<Vec<Receipt>, String> {
let receipts = self
.products
.borrow()
.values()
.map(|product| Ok((product.receipt.identity()?, product.receipt.clone())))
.collect::<Result<BTreeMap<_, _>, String>>()?;
identities
.iter()
.map(|identity| {
receipts
.get(identity)
.cloned()
.ok_or("prepared build receipt is missing".into())
})
.collect()
}
fn remember(&self, key: String, product: Product) -> Result<(), String> {
let total = self
.products
.borrow()
.values()
.try_fold(product.bytes.len(), |total, cached| {
total.checked_add(cached.bytes.len())
})
.ok_or("build candidate cache byte overflow")?;
if total > crate::workers::MAX_TREE_BYTES {
return Err("build candidate cache exceeds byte budget".into());
}
self.products.borrow_mut().insert(key, product);
Ok(())
}
}
pub fn normalize(receipts: &mut Vec<Receipt>) -> Result<(), String> {
let mut indexed = BTreeMap::new();
for receipt in receipts.drain(..) {
indexed.insert(receipt.identity()?, receipt);
}
*receipts = indexed.into_values().collect();
Ok(())
}
pub fn npm_outputs(lock: &LockFile) -> impl Iterator<Item = &Receipt> {
lock.builds
.iter()
.filter(|receipt| receipt.ecosystem == Ecosystem::Npm)
}
pub(crate) fn npm_receipt<'a>(
lock: &'a LockFile,
environment: &super::npm::Environment,
) -> Result<Option<&'a Receipt>, String> {
let Some(identity) = &environment.build else {
return Ok(None);
};
lock.builds
.iter()
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
.map(Some)
.ok_or("npm installation receipt is missing".into())
}
/// Offline consumption never invokes a frontend. Removing roots projects the
/// already built graph and output; it does not rerun lifecycle side effects.
pub fn npm_installation(
lock: &LockFile,
environment: &super::npm::Environment,
bytes: &[u8],
) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
let receipt = npm_receipt(lock, environment)?.ok_or("npm installation has no build receipt")?;
if bytes.len() as u64 != receipt.output.bytes || super::sha256(bytes) != receipt.output.sha256 {
return Err("npm built installation differs from its locked artifact".into());
}
let graph = receipt
.graph
.as_ref()
.ok_or("npm receipt lacks its exact graph")?;
let files = super::archive_files("tgz", bytes)?;
super::npm::validate_built_files(graph, &files)?;
let candidates = files
.iter()
.filter(|(path, _)| path.ends_with(".node"))
.map(|(path, _)| path.clone())
.collect::<BTreeSet<_>>();
let accounted = receipt
.native_entries
.iter()
.chain(receipt.native_rejections.keys())
.cloned()
.collect::<BTreeSet<_>>();
if candidates != accounted
|| receipt
.native_rejections
.keys()
.any(|path| receipt.native_entries.contains(path))
{
return Err("npm native admission does not cover its exact addon files".into());
}
for path in &receipt.native_entries {
if !files.iter().any(|(name, bytes)| {
name == path && path.ends_with(".node") && bytes.starts_with(b"\x7fELF")
}) {
return Err("npm locked native entry point is absent from output".into());
}
}
let executable = super::registry::archive_executables("tgz", bytes)?;
let retained = environment
.instances
.iter()
.map(|node| node.path.as_str())
.collect::<BTreeSet<_>>();
let mut output = Vec::new();
for (path, bytes) in files {
if path.ends_with(".node") && !receipt.native_entries.contains(&path) {
continue;
}
if retained.contains(super::npm::installed_owner(&path)?.as_str()) {
let mode = executable.contains(&path);
output.push((path, bytes, mode));
}
}
Ok(output)
}
pub fn doctor(lock: &LockFile) -> Result<(), String> {
doctor_at(lock, 0)
}
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
if depth > 16 {
return Err("build dependency depth exceeds limit".into());
}
let records = lock
.builds
.iter()
.map(|receipt| Ok((receipt.identity()?, receipt)))
.collect::<Result<BTreeMap<_, _>, String>>()?;
if records.len() != lock.builds.len() {
return Err("duplicate build receipt".into());
}
let mut used = BTreeSet::new();
for lib in &lock.libs {
if let Some(identity) = &lib.build {
let receipt = records
.get(identity)
.ok_or("Python Lib has no exact build receipt")?;
if lib.spec.ecosystem != Ecosystem::Pip
|| receipt.ecosystem != Ecosystem::Pip
|| receipt.runtime != lib.runtime
|| !lib.artifacts.contains(&receipt.output)
|| receipt.roots.len() != 1
|| receipt.roots[0].distribution_name() != lib.spec.distribution_name()
|| receipt.roots[0].version != lib.spec.version
{
return Err(
"Python Lib build receipt differs from locked artifact identity".into(),
);
}
used.insert(identity.clone());
}
}
for environment in &lock.npm {
if let Some(identity) = &environment.build {
let receipt = records
.get(identity)
.ok_or("npm environment has no exact build receipt")?;
let graph = receipt
.graph
.as_ref()
.ok_or("npm build receipt lacks its graph")?;
if receipt.ecosystem != Ecosystem::Npm
|| receipt.roots != graph.roots
|| graph.build.is_some()
|| !environment
.roots
.iter()
.all(|root| graph.roots.contains(root))
|| !environment
.instances
.iter()
.all(|node| graph.instances.contains(node))
|| !environment.instances.iter().all(|node| {
lock.libs.iter().any(|lib| {
lib.spec == node.source.archive
&& lib.runtime == receipt.runtime
&& lib
.artifacts
.iter()
.all(|artifact| receipt.sources.contains(artifact))
})
})
{
return Err("npm installation build receipt differs from locked graph".into());
}
used.insert(identity.clone());
}
}
if used.len() != records.len() {
return Err("lock contains an unreferenced build receipt".into());
}
for receipt in records.values() {
super::validate_runtime(&receipt.runtime)?;
if receipt.tools.runtime_sha256 != receipt.runtime.sha256
|| receipt.tools.ecosystem != receipt.ecosystem
|| receipt.sources.is_empty()
|| receipt.output.target != receipt.tools.target
|| receipt.roots.is_empty()
|| receipt.frontend != frontend(&receipt.ecosystem)?
{
return Err("build receipt has inconsistent tools, frontend, or target".into());
}
for artifact in receipt.sources.iter().chain([&receipt.output]) {
super::validate_artifact(artifact)?;
if artifact.target != receipt.tools.target {
return Err("build artifact target differs from its tools".into());
}
}
if receipt.ecosystem == Ecosystem::Pip
&& (receipt.python.is_none()
|| receipt.graph.is_some()
|| receipt.auxiliary_runtime.is_some())
{
return Err("Python build receipt lacks its backend configuration".into());
}
if receipt.ecosystem == Ecosystem::Pip
&& (!receipt.native_entries.is_empty() || !receipt.native_rejections.is_empty())
{
return Err("Python receipt cannot declare Node native entry points".into());
}
for path in receipt
.native_entries
.iter()
.chain(receipt.native_rejections.keys())
{
crate::workers::valid_path(path)?;
}
let policy = RegistryRuntime {
pack: receipt.runtime.clone(),
target: receipt.tools.target.clone(),
python_markers: receipt.python_markers.clone(),
python_wheel_tags: vec![],
npm_libc: None,
};
let auxiliary = receipt
.auxiliary_runtime
.as_ref()
.map(|runtime| RegistryRuntime {
pack: runtime.clone(),
target: receipt.tools.target.clone(),
python_markers: None,
python_wheel_tags: vec![],
npm_libc: None,
});
if let Some(runtime) = &receipt.auxiliary_runtime {
super::validate_runtime(runtime)?;
}
receipt.tools.validate(&policy, auxiliary.as_ref())?;
if receipt.ecosystem == Ecosystem::Npm
&& (receipt.python.is_some()
|| receipt.python_markers.is_some()
|| !receipt.dynamic_requires.is_empty()
|| !receipt.dependencies.libs.is_empty()
|| receipt.graph.is_none())
{
return Err("npm build receipt contains Python backend state".into());
}
if !receipt.config_settings.is_empty() {
return Err("unsupported source-build configuration settings".into());
}
if let Some(system) = &receipt.python {
validate_requirements(receipt, system)?;
}
if !receipt.dependencies.workers.is_empty()
|| !receipt.dependencies.packages.is_empty()
|| receipt
.dependencies
.libs
.iter()
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
{
return Err("build dependency closure is not a separate Python environment".into());
}
doctor_at(&receipt.dependencies, depth + 1)?;
// Full graph checks belong to the ordinary lock owner; recursion is
// bounded above before invoking that owner on a dependency lock.
let mut dependencies = (*receipt.dependencies).clone();
dependencies.builds.clear();
for lib in &mut dependencies.libs {
lib.build = None;
}
super::doctor(&dependencies)?;
}
Ok(())
}
fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(), String> {
let identifier = |path: &str| {
!path.is_empty()
&& path.split('.').all(|part| {
let mut bytes = part.bytes();
bytes
.next()
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
})
};
let (module, function) = system
.backend
.split_once(':')
.map(|(module, function)| (module, Some(function)))
.unwrap_or((&system.backend, None));
if !identifier(module)
|| function.is_some_and(|function| !identifier(function))
|| system.requires.len() > 1024
|| receipt.dynamic_requires.len() > 1024
|| system.backend_path.len() > 64
{
return Err("build receipt has an invalid backend or requirement set".into());
}
for path in &system.backend_path {
if path.is_empty() || path.starts_with('/') || path.contains(['\\', '\0']) {
return Err("invalid backend-path in receipt".into());
}
let mut depth = 0usize;
for part in path.split('/') {
match part {
"" | "." => {}
".." => {
depth = depth.checked_sub(1).ok_or("backend-path escapes source")?;
}
_ => depth += 1,
}
}
}
let markers = receipt
.python_markers
.as_ref()
.ok_or("Python build receipt lacks marker policy")?;
let requirements = system
.requires
.iter()
.chain(&receipt.dynamic_requires)
.cloned()
.collect::<Vec<_>>();
let document = serde_json::json!({"info":{"requires_dist":requirements}});
let expected = super::python_dependencies(&document, markers, &[])?;
let libraries = receipt
.dependencies
.libs
.iter()
.map(|lib| (lib.spec.name.as_str(), lib))
.collect::<BTreeMap<_, _>>();
let mut remaining = Vec::new();
for (name, rule) in expected {
let lib = libraries
.get(name.as_str())
.ok_or("Python build receipt omits a required dependency")?;
let version = lib
.spec
.version
.parse::<pep440_rs::Version>()
.map_err(|error| error.to_string())?;
if !rule.contains(&version) {
return Err("Python build dependency does not satisfy recorded requirements".into());
}
remaining.push(lib.spec.name.as_str());
}
let mut reached = BTreeSet::new();
while let Some(name) = remaining.pop() {
if !reached.insert(name) {
continue;
}
let lib = libraries
.get(name)
.ok_or("build dependency closure is incomplete")?;
if lib.runtime != receipt.runtime
|| lib
.artifacts
.iter()
.any(|artifact| artifact.target != receipt.tools.target)
{
return Err("build dependency runtime or target differs from its receipt".into());
}
remaining.extend(lib.dependencies.iter().map(|edge| edge.spec.name.as_str()));
}
if reached.len() != libraries.len() {
return Err("build dependency lock contains unrelated libraries".into());
}
Ok(())
}
/// Offline consumers bind recorded marker evaluation to the immutable runtime
/// manifest they have just verified, never to a lock's self-declared policy.
pub fn validate_python_policy(
lock: &LockFile,
lib: &super::LockedLib,
markers: &pep508_rs::MarkerEnvironment,
) -> Result<(), String> {
if let Some(identity) = &lib.build {
let receipt = lock
.builds
.iter()
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
.ok_or("Python build receipt missing")?;
if receipt.python_markers.as_ref() != Some(markers) {
return Err("Python build marker policy differs from signed runtime".into());
}
}
Ok(())
}
fn frontend(ecosystem: &Ecosystem) -> Result<String, String> {
match ecosystem {
Ecosystem::Pip => Ok(super::sha256(include_bytes!("build/python.py"))),
Ecosystem::Npm => Ok(super::sha256(
&[
include_bytes!("build/npm.js").as_slice(),
include_bytes!("build/npm/probe.cjs").as_slice(),
]
.concat(),
)),
_ => Err("source-build frontend is not available for this ecosystem".into()),
}
}
+103
View File
@@ -0,0 +1,103 @@
// Fixed registry-package lifecycle frontend. Runs only in the build sandbox.
const fs = require('node:fs');
const path = require('node:path');
const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script');
const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links');
async function main() {
const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8'));
const root = '/data/build/output/install';
fs.mkdirSync('/data/build/output');
fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true });
fs.mkdirSync('/data/build/tmp');
fs.mkdirSync('/data/build/home');
Object.assign(process.env, {
PATH: '/worker/runtime/bin:/usr/bin',
CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar',
PYTHON: request.python,
npm_config_python: request.python,
npm_config_nodedir: '/worker/npm',
npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js',
npm_config_build_from_source: 'true',
npm_config_cache: '/data/build/npm-cache',
TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0',
HOME: '/data/build/home',
});
const nodes = new Map(request.instances.map(node => [node.path, node]));
for (const node of request.instances) {
const directory = path.join(root, node.path);
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8'));
await binLinks({ path: directory, pkg, global: false, top: false, force: false });
}
const visited = new Set();
const required = new Set(request.required);
const failed = new Set();
const nativeEntries = new Set();
async function install(name) {
if (visited.has(name)) return;
visited.add(name);
const node = nodes.get(name);
try {
for (const edge of node.edges) {
if (edge.target && nodes.has(edge.target)) {
await install(edge.target);
if (failed.has(edge.target) && ['dependency', 'peer'].includes(edge.kind)) {
throw new Error('required lifecycle dependency failed: ' + edge.target);
}
}
}
const directory = path.join(root, name);
for (const event of ['preinstall', 'install', 'postinstall']) {
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' });
}
const native = path.join(directory, 'build', 'Release');
if (fs.existsSync(native)) {
for (const file of fs.readdirSync(native)) {
if (file.endsWith('.node')) {
const addon = path.join(native, file);
require(addon);
nativeEntries.add(path.relative(root, addon));
}
}
}
} catch (error) {
if (required.has(name)) throw error;
process.stderr.write('optional npm lifecycle failed: ' + name + ': ' + String(error.message || error) + '\n');
failed.add(name);
}
}
for (const node of request.instances) await install(node.path);
let changed = true;
while (changed) {
changed = false;
for (const node of request.instances) {
if (!failed.has(node.path) && node.edges.some(edge => ['dependency', 'peer'].includes(edge.kind) && failed.has(edge.target))) {
if (required.has(node.path)) throw new Error('required lifecycle dependency failed: ' + node.path);
failed.add(node.path);
changed = true;
}
}
}
const retained = new Set();
function retain(name) {
if (retained.has(name) || failed.has(name)) return;
retained.add(name);
for (const edge of nodes.get(name).edges) {
if (edge.target) retain(edge.target);
}
}
for (const node of request.roots) retain(node);
for (const node of request.instances) {
if (!retained.has(node.path)) fs.rmSync(path.join(root, node.path), { recursive: true, force: true });
}
// Bin links are a build-time PATH input. The immutable Worker tree stores
// package files only; it never persists symlinks created by installer hooks.
for (const node of request.instances) {
fs.rmSync(path.join(root, node.path, 'node_modules', '.bin'), { recursive: true, force: true });
fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true });
}
fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true });
fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
}
main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; });
+230
View File
@@ -0,0 +1,230 @@
//! Registry lifecycle scripts use the signed npm frontend and exact instance graph.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::io::Cursor;
use super::{Product, Receipt, Session};
use crate::libs::{
Ecosystem, LockFile, LockedArtifact, LockedLib, RegistryResolver, RegistryRuntime, npm, sha256,
};
mod environment;
use environment::{Environment, Output};
impl Session<'_> {
pub(crate) fn npm(
&self,
resolver: &RegistryResolver<'_>,
libraries: &mut Vec<LockedLib>,
graph: &mut npm::Environment,
runtime: &RegistryRuntime,
) -> Result<(), String> {
let mut lock = LockFile::new(libraries.clone())?;
lock.npm.push(graph.clone());
let archives = libraries
.iter()
.map(|lib| {
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
Ok((
lib.spec.clone(),
resolver.store.verify_exact(
&artifact.sha256,
artifact.bytes,
&artifact.target,
)?,
))
})
.collect::<Result<BTreeMap<_, _>, String>>()?;
let files = npm::build_files(graph, &lock, |lib| {
archives
.get(&lib.spec)
.map(Vec::as_slice)
.ok_or("npm source archive missing".into())
})?;
if !npm::requires_build(&files)? {
return Ok(());
}
let mut sources = libraries
.iter()
.flat_map(|lib| lib.artifacts.clone())
.collect::<Vec<_>>();
sources.sort();
sources.dedup();
let key = format!(
"npm:{}:{}:{}",
sha256(&serde_json::to_vec(graph).map_err(|error| error.to_string())?),
runtime.pack.sha256,
sha256(&serde_json::to_vec(&sources).map_err(|error| error.to_string())?)
);
if let Some(product) = self.products.borrow().get(&key) {
*graph = product
.receipt
.graph
.clone()
.ok_or("cached npm graph missing")?;
graph.build = Some(product.receipt.identity()?);
retain_archives(libraries, graph);
return Ok(());
}
let executables = npm::build_executables(graph, &archives)?;
let environment = Environment::new(self.inputs, runtime)?;
let output = environment.run(self.inputs, graph, &files, &executables)?;
prune_failed(graph, &output.failed)?;
retain_archives(libraries, graph);
npm::validate_built_files(graph, &output.files)?;
let bytes = archive(&output)?;
let digest = resolver.store.publish(&bytes, &runtime.target)?;
if digest != sha256(&bytes) {
return Err("built npm cache identity mismatch".into());
}
let output_artifact = LockedArtifact {
target: runtime.target.clone(),
path: format!("build/npm/{digest}/installation.tgz"),
bytes: bytes.len() as u64,
sha256: digest,
};
let receipt = Receipt {
ecosystem: Ecosystem::Npm,
roots: graph.roots.clone(),
sources,
graph: Some(graph.clone()),
runtime: runtime.pack.clone(),
auxiliary_runtime: Some(environment.python_runtime),
native_entries: output.native,
native_rejections: output.rejections,
python_markers: None,
tools: environment.descriptor,
frontend: super::frontend(&Ecosystem::Npm)?,
python: None,
dynamic_requires: vec![],
dependencies: Box::new(LockFile::new(vec![])?),
config_settings: BTreeMap::new(),
output: output_artifact,
};
graph.build = Some(receipt.identity()?);
self.remember(
key,
Product {
bytes,
filename: "installation.tgz".into(),
receipt,
},
)?;
Ok(())
}
}
fn archive(output: &Output) -> Result<Vec<u8>, String> {
let encoder = flate2::GzBuilder::new()
.mtime(0)
.operating_system(255)
.write(Vec::new(), flate2::Compression::default());
let mut archive = tar::Builder::new(encoder);
for (path, bytes) in &output.files {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(if output.executables.contains(path) {
0o755
} else {
0o644
});
header.set_uid(0);
header.set_gid(0);
header.set_mtime(0);
header.set_cksum();
archive
.append_data(&mut header, path, Cursor::new(bytes))
.map_err(|error| error.to_string())?;
}
archive
.into_inner()
.and_then(|encoder| encoder.finish())
.map_err(|error| error.to_string())
}
fn retain_archives(libraries: &mut Vec<LockedLib>, graph: &npm::Environment) {
let used = graph
.instances
.iter()
.map(|node| &node.source.archive)
.collect::<BTreeSet<_>>();
libraries.retain(|lib| used.contains(&lib.spec));
}
fn required_nodes(graph: &npm::Environment) -> Result<BTreeSet<String>, String> {
reachable(graph, true)
}
fn reachable(graph: &npm::Environment, mandatory_only: bool) -> Result<BTreeSet<String>, String> {
let mut remaining = graph
.roots
.iter()
.map(|root| format!("node_modules/{}", root.name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = remaining.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = graph
.instances
.iter()
.find(|node| node.path == path)
.ok_or("npm reachable instance missing")?;
remaining.extend(
node.edges
.iter()
.filter(|edge| {
!mandatory_only || matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
})
.filter_map(|edge| edge.target.clone()),
);
}
Ok(reached)
}
fn prune_failed(graph: &mut npm::Environment, failed: &BTreeSet<String>) -> Result<(), String> {
if failed
.iter()
.any(|path| !graph.instances.iter().any(|node| &node.path == path))
{
return Err("npm build reported an unknown failed instance".into());
}
let mut failed = failed.clone();
loop {
let before = failed.len();
for node in &graph.instances {
if node.edges.iter().any(|edge| {
matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
&& edge
.target
.as_ref()
.is_some_and(|path| failed.contains(path))
}) {
failed.insert(node.path.clone());
}
}
if before == failed.len() {
break;
}
}
if !required_nodes(graph)?.is_disjoint(&failed) {
return Err("required npm lifecycle failed".into());
}
graph.instances.retain(|node| !failed.contains(&node.path));
for node in &mut graph.instances {
for edge in &mut node.edges {
if edge
.target
.as_ref()
.is_some_and(|path| failed.contains(path))
{
edge.target = None;
edge.omission = Some(npm::Omission::BuildFailure);
}
}
}
let reached = reachable(graph, false)?;
graph.instances.retain(|node| reached.contains(&node.path));
Ok(())
}
@@ -0,0 +1,326 @@
//! One immutable Node/Python/tool environment for registry lifecycle execution.
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
use crate::libs::build::NativeRejection;
use crate::libs::build::{Inputs, pack, process::Stage};
use crate::libs::{Ecosystem, RegistryRuntime, RuntimePack, archive_files, npm, sha256};
use crate::workers::runtime;
pub(super) struct Environment {
files: BTreeMap<String, (Vec<u8>, bool)>,
assets: Vec<(String, Vec<u8>)>,
node: String,
node_paths: BTreeSet<String>,
python: String,
pub python_runtime: RuntimePack,
pub descriptor: pack::Descriptor,
}
pub(super) struct Output {
pub files: Vec<(String, Vec<u8>)>,
pub executables: BTreeSet<String>,
pub failed: BTreeSet<String>,
pub native: BTreeSet<String>,
pub rejections: BTreeMap<String, NativeRejection>,
}
#[derive(serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct ResultMetadata {
failed: BTreeSet<String>,
native: BTreeSet<String>,
}
impl Environment {
pub fn new(inputs: &dyn Inputs, runtime: &RegistryRuntime) -> Result<Self, String> {
crate::libs::build::require_execution_target(&runtime.target, "npm lifecycle build")?;
let (node, bytes) = inputs.runtime(Ecosystem::Npm)?;
if node.pack != runtime.pack || sha256(&bytes) != runtime.pack.sha256 {
return Err("npm build runtime differs from resolver".into());
}
let node_files = archive_files("tgz", &bytes)?;
let node_manifest = runtime::validate(&node_files, &Ecosystem::Npm, &runtime.target)?;
let node_paths = node_files
.iter()
.map(|(path, _)| format!("runtime/{path}"))
.collect();
let node = format!(
"runtime/{}",
node_manifest
.executable
.as_deref()
.ok_or("Node executable missing")?
);
let (python, bytes) = inputs.runtime(Ecosystem::Pip)?;
if sha256(&bytes) != python.pack.sha256 {
return Err("auxiliary Python runtime hash mismatch".into());
}
let python_files = archive_files("tgz", &bytes)?;
let python_manifest = runtime::validate(&python_files, &Ecosystem::Pip, &runtime.target)?;
let python_executable = format!(
"runtime/{}",
python_manifest
.executable
.as_deref()
.ok_or("build Python executable missing")?
);
let (descriptor, bytes) = inputs.tools(Ecosystem::Npm)?;
descriptor.validate(runtime, Some(&python))?;
let (manifest, tools) = pack::unpack(&bytes, &descriptor)?;
let assets = inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
pack::validate_tools(&manifest, &tools, &assets)?;
let mut files = BTreeMap::new();
for (path, bytes) in node_files {
let executable = node_manifest.executable(&path);
files.insert(format!("runtime/{path}"), (bytes, executable));
}
for (path, bytes) in python_files {
if path == "dever-runtime.json" {
continue;
}
let executable = python_manifest.executable(&path);
let path = format!("runtime/{path}");
if let Some((existing, _)) = files.get(&path) {
if existing != &bytes {
return Err("Node and auxiliary Python runtime paths conflict".into());
}
} else {
files.insert(path, (bytes, executable));
}
}
for (path, bytes) in tools {
let executable = manifest.executables.contains(&path);
if files.insert(path, (bytes, executable)).is_some() {
return Err("npm tools overlap runtimes".into());
}
}
Ok(Self {
files,
assets,
node,
node_paths,
python: python_executable,
python_runtime: python.pack,
descriptor,
})
}
pub fn run(
&self,
inputs: &dyn Inputs,
graph: &npm::Environment,
files: &[(String, Vec<u8>)],
executables: &BTreeSet<String>,
) -> Result<Output, String> {
let stage = Stage::new()?;
let input = stage.0.join("input");
for (path, (bytes, executable)) in &self.files {
stage.write(&format!("input/{path}"), bytes, *executable)?;
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
bytes,
dever_sandbox::asset_executable(path),
)?;
}
for (path, bytes) in files {
stage.write(
&format!("input/install/{path}"),
bytes,
executables.contains(path),
)?;
}
let request = serde_json::json!({
"instances":graph.instances, "required":super::required_nodes(graph)?,
"roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(),
"python":format!("/worker/{}", self.python),
});
stage.write(
"input/request.json",
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
false,
)?;
stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [input.join("frontend.js").into_os_string()];
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated npm lifecycle",
&work,
)?;
inputs.diagnostic(&stage.diagnostics()?);
let mut outputs = super::super::python::output_files(&work.join("output"))?;
let response: ResultMetadata = serde_json::from_slice(
&outputs
.remove("result.json")
.ok_or("npm build result missing")?,
)
.map_err(|error| error.to_string())?;
let mut files = Vec::new();
let mut executables = BTreeSet::new();
for (path, bytes) in outputs {
let path = path
.strip_prefix("install/")
.ok_or("npm lifecycle output is outside its installation")?
.to_owned();
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if fs::symlink_metadata(work.join("output/install").join(&path))
.map_err(|error| error.to_string())?
.permissions()
.mode()
& 0o111
!= 0
{
executables.insert(path.clone());
}
}
files.push((path, bytes));
}
let mut output = Output {
files,
executables,
failed: response.failed,
native: response.native,
rejections: BTreeMap::new(),
};
// Release the large compiler tree before staging the much smaller
// runtime-only native admission sandbox.
drop(stage);
self.admit_native(inputs, &mut output)?;
Ok(output)
}
fn admit_native(&self, inputs: &dyn Inputs, output: &mut Output) -> Result<(), String> {
let required = std::mem::take(&mut output.native);
let candidates = output
.files
.iter()
.filter(|(path, _)| path.ends_with(".node"))
.collect::<Vec<_>>();
if candidates.is_empty() {
return if required.is_empty() {
Ok(())
} else {
Err("npm generated native entry point is missing".into())
};
}
let machine = match self.descriptor.target.as_str() {
"linux-x86_64" => goblin::elf::header::EM_X86_64,
"linux-aarch64" => goblin::elf::header::EM_AARCH64,
_ => return Err("Node native admission target is unsupported".into()),
};
let mut files = self
.files
.iter()
.filter(|(path, _)| self.node_paths.contains(*path))
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
files.extend(
self.assets
.iter()
.map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())),
);
files.extend(
output
.files
.iter()
.map(|(path, bytes)| (path.clone(), bytes.as_slice())),
);
let stage = Stage::new()?;
for (path, bytes) in &files {
let relative = if path.starts_with("sandbox/") {
path.clone()
} else {
format!("input/{path}")
};
let executable = path == &self.node
|| path
.strip_prefix("sandbox/")
.is_some_and(dever_sandbox::asset_executable);
stage.write(&relative, bytes, executable)?;
}
stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?;
let input = stage.0.join("input");
let executable = input.join(&self.node);
let assets = stage.0.join("sandbox");
for (path, bytes) in candidates {
let rejection = match goblin::elf::Elf::parse(bytes) {
Err(_) => Some(NativeRejection::Format),
Ok(elf) if !elf.is_64 || !elf.little_endian || elf.header.e_machine != machine => {
Some(NativeRejection::Target)
}
Ok(_) => {
if let Err(error) = crate::workers::python_wheel::validate_native(
&self.descriptor.target,
&[],
&self.node,
&files,
&BTreeSet::from([path.clone()]),
) {
inputs.diagnostic(
format!("npm addon '{path}' has no packaged loader closure: {error}\n")
.as_bytes(),
);
Some(NativeRejection::Closure)
} else {
let arguments = [
input.join("probe.cjs").into_os_string(),
input.join(path).into_os_string(),
];
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities::default(),
grants: &[],
};
let admitted = stage.probe(launch.command()?)?;
inputs.diagnostic(&stage.diagnostics()?);
if admitted {
None
} else {
Some(NativeRejection::Unloadable)
}
}
}
};
if let Some(reason) = rejection {
output.rejections.insert(path.clone(), reason);
} else {
output.native.insert(path.clone());
}
}
if !required.is_subset(&output.native) {
return Err("source-built npm addon did not pass runtime-only native admission".into());
}
Ok(())
}
}
@@ -0,0 +1,9 @@
// No package entry modules are executed. Probe exactly one immutable addon in
// an isolated runtime-only process, so addon globals cannot affect its peers.
try {
require(process.argv[2]);
process.exit(0);
} catch (error) {
process.stderr.write(String(error.stack || error) + '\n');
process.exit(10);
}
+166
View File
@@ -0,0 +1,166 @@
//! Signed, target-specific source-build tools. Never sourced from host PATH.
use crate::libs::{Ecosystem, RegistryRuntime};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};
pub const MAX_PACK_BYTES: usize = 256 * 1024 * 1024;
pub const MAX_EXPANDED_BYTES: usize = 512 * 1024 * 1024;
pub const MAX_FILES: usize = 65_536;
pub type PackFiles = Vec<(String, Vec<u8>)>;
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Manifest {
pub format: String,
pub ecosystem: Ecosystem,
pub target: String,
pub runtime_sha256: String,
pub python_runtime_sha256: Option<String>,
pub executables: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Descriptor {
pub format: String,
pub ecosystem: Ecosystem,
pub target: String,
pub runtime_sha256: String,
pub python_runtime_sha256: Option<String>,
pub sha256: String,
}
fn hash(value: &str) -> Result<(), String> {
if value.len() != 64
|| !value
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
{
return Err("build pack identity must be canonical SHA-256".into());
}
Ok(())
}
impl Descriptor {
pub fn validate(
&self,
runtime: &RegistryRuntime,
python: Option<&RegistryRuntime>,
) -> Result<(), String> {
if self.format != "dever-registry-build-v1"
|| self.ecosystem == Ecosystem::Go
|| self.target != runtime.target
|| self.runtime_sha256 != runtime.pack.sha256
{
return Err("signed build pack does not match its runtime and target".into());
}
hash(&self.sha256)?;
hash(&self.runtime_sha256)?;
match (&self.ecosystem, &self.python_runtime_sha256, python) {
(Ecosystem::Pip, None, _) => Ok(()),
(Ecosystem::Npm, Some(identity), Some(python))
if identity == &python.pack.sha256 && python.target == self.target =>
{
hash(identity)
}
_ => Err("build pack has no exact auxiliary Python runtime binding".into()),
}
}
}
impl Manifest {
pub fn validate(
&self,
files: &[(String, Vec<u8>)],
descriptor: &Descriptor,
) -> Result<(), String> {
if self.format != "dever-lib-build-v1"
|| self.ecosystem != descriptor.ecosystem
|| self.target != descriptor.target
|| self.runtime_sha256 != descriptor.runtime_sha256
|| self.python_runtime_sha256 != descriptor.python_runtime_sha256
{
return Err("build pack manifest differs from its signed descriptor".into());
}
let total = files
.iter()
.try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len()))
.ok_or("build pack expanded size overflow")?;
if files.len() > MAX_FILES || total > MAX_EXPANDED_BYTES {
return Err("build pack exceeds file or byte budget".into());
}
let mut names = BTreeSet::new();
for (path, _) in files {
crate::workers::valid_path(path)?;
if path != "dever-build.json"
&& !path.starts_with("rootfs/usr/")
&& !(self.ecosystem == Ecosystem::Pip && path.starts_with("runtime/include/"))
&& !(self.ecosystem == Ecosystem::Npm
&& (path.starts_with("npm/include/node/")
|| path.starts_with("npm/frontend/node_modules/")))
{
return Err(format!(
"build pack path '{path}' is outside its fixed tool layout"
));
}
if !names.insert(path.as_str()) {
return Err("build pack repeats a path".into());
}
}
let mut executable_names = BTreeSet::new();
for path in &self.executables {
if !names.contains(path.as_str())
|| !executable_names.insert(path.as_str())
|| !(path.starts_with("rootfs/usr/")
|| self.ecosystem == Ecosystem::Npm
&& path.starts_with("npm/frontend/node_modules/"))
{
return Err(
"build executable is absent, repeated, or outside its tool root".into(),
);
}
}
for name in ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] {
if !executable_names.contains(format!("rootfs/usr/bin/{name}").as_str()) {
return Err(format!("build pack is missing required tool '{name}'"));
}
}
Ok(())
}
}
pub fn unpack(bytes: &[u8], descriptor: &Descriptor) -> Result<(Manifest, PackFiles), String> {
if bytes.len() > MAX_PACK_BYTES || crate::libs::sha256(bytes) != descriptor.sha256 {
return Err("build pack exceeds its budget or SHA-256 differs".into());
}
let files = super::super::registry::tar_files(bytes, "tgz", MAX_EXPANDED_BYTES as u64)?;
let metadata = files
.iter()
.find(|(path, _)| path == "dever-build.json")
.ok_or("build pack is missing dever-build.json")?;
let manifest: Manifest = serde_json::from_slice(&metadata.1)
.map_err(|error| format!("invalid build pack manifest: {error}"))?;
manifest.validate(&files, descriptor)?;
Ok((manifest, files))
}
pub fn validate_tools(
manifest: &Manifest,
files: &[(String, Vec<u8>)],
assets: &[(String, Vec<u8>)],
) -> Result<(), String> {
let mut supplied = files
.iter()
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in assets {
supplied.insert(format!("sandbox/{path}"), bytes);
}
crate::workers::python_wheel::native::validate_build_tools(
&manifest.target,
&manifest.executables,
&supplied,
)
}
+186
View File
@@ -0,0 +1,186 @@
//! One lifecycle for every managed compiler and package build hook.
use std::fs::{self, File};
use std::io::Read;
use std::path::Path;
use std::path::PathBuf;
use std::process::{Child, Command, Stdio};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
const DEADLINE: Duration = Duration::from_secs(300);
const MAX_OUTPUT: u64 = 16 * 1024 * 1024;
static NEXT_STAGE: AtomicU64 = AtomicU64::new(0);
mod budget;
pub(crate) struct Stage(pub(crate) PathBuf);
impl Stage {
pub(crate) fn new() -> Result<Self, String> {
for _ in 0..32 {
let path = std::env::temp_dir().join(format!(
"dever-lib-build-{}-{}",
std::process::id(),
NEXT_STAGE.fetch_add(1, Ordering::Relaxed)
));
#[cfg(unix)]
let created = {
use std::os::unix::fs::DirBuilderExt;
let mut builder = fs::DirBuilder::new();
builder.mode(0o700);
builder.create(&path)
};
#[cfg(not(unix))]
let created = fs::create_dir(&path);
match created {
Ok(()) => return Ok(Self(path)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(format!("cannot create private build staging: {error}")),
}
}
Err("cannot allocate private build staging".into())
}
pub(crate) fn write(
&self,
relative: &str,
bytes: &[u8],
executable: bool,
) -> Result<PathBuf, String> {
crate::workers::valid_path(relative)?;
let path = self.0.join(relative);
fs::create_dir_all(path.parent().expect("validated relative path"))
.map_err(|error| format!("cannot create build input directory: {error}"))?;
// This tree is private and never a writable child mount. A second write
// must not silently replace a previously verified input.
use std::io::Write;
let mut file = fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.map_err(|error| format!("cannot stage build input: {error}"))?;
file.write_all(bytes)
.map_err(|error| format!("cannot write build input: {error}"))?;
#[cfg(unix)]
if executable {
use std::os::unix::fs::PermissionsExt;
file.set_permissions(fs::Permissions::from_mode(0o700))
.map_err(|error| format!("cannot mark build tool executable: {error}"))?;
}
Ok(path)
}
pub(crate) fn run(
&self,
command: Command,
operation: &str,
writable: &Path,
) -> Result<Vec<u8>, String> {
let status = self.execute(command, operation, Some(writable))?;
if !status.success() {
return Err(format!(
"{operation} failed: {}",
String::from_utf8_lossy(&self.diagnostics()?)
));
}
fs::read(self.0.join("tool-stdout"))
.map_err(|error| format!("cannot read {operation} output: {error}"))
}
pub(crate) fn probe(&self, command: Command) -> Result<bool, String> {
match self
.execute(command, "isolated Node addon probe", None)?
.code()
{
Some(0) => Ok(true),
Some(10) => Ok(false),
_ => Err(format!(
"Node addon probe failed outside its loader check: {}",
String::from_utf8_lossy(&self.diagnostics()?)
)),
}
}
fn execute(
&self,
mut command: Command,
operation: &str,
writable: Option<&Path>,
) -> Result<std::process::ExitStatus, String> {
// Captures are outside every child-writable grant. Namespace teardown
// reaps descendants when the owned bootstrap is killed on any exit path.
let stdout = self.0.join("tool-stdout");
let stderr = self.0.join("tool-stderr");
let child = command
.stdin(Stdio::null())
.stdout(Stdio::from(
File::create(&stdout).map_err(|error| error.to_string())?,
))
.stderr(Stdio::from(
File::create(&stderr).map_err(|error| error.to_string())?,
))
.spawn()
.map_err(|error| format!("cannot execute {operation}: {error}"))?;
let mut child = OwnedChild(child);
let deadline = Instant::now() + DEADLINE;
let mut next_scratch_check = Instant::now();
let within_budget = || -> Result<(), String> {
for path in [&stdout, &stderr] {
if fs::metadata(path).map_err(|error| error.to_string())?.len() > MAX_OUTPUT {
return Err(format!("{operation} output exceeds byte limit"));
}
}
Ok(())
};
let status = loop {
within_budget()?;
if Instant::now() >= next_scratch_check {
if let Some(path) = writable {
budget::check(path)?;
}
next_scratch_check = Instant::now() + Duration::from_millis(250);
}
if let Some(status) = child
.0
.try_wait()
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
{
break status;
}
if Instant::now() >= deadline {
return Err(format!("{operation} exceeded its five-minute deadline"));
}
std::thread::sleep(Duration::from_millis(25));
};
within_budget()?;
if let Some(path) = writable {
budget::check(path)?;
}
Ok(status)
}
pub(crate) fn diagnostics(&self) -> Result<Vec<u8>, String> {
let mut bytes = Vec::new();
for name in ["tool-stdout", "tool-stderr"] {
File::open(self.0.join(name))
.and_then(|file| file.take(MAX_OUTPUT).read_to_end(&mut bytes))
.map_err(|error| format!("cannot read build diagnostic: {error}"))?;
}
Ok(bytes)
}
}
impl Drop for Stage {
fn drop(&mut self) {
let _ = fs::remove_dir_all(&self.0);
}
}
struct OwnedChild(Child);
impl Drop for OwnedChild {
fn drop(&mut self) {
let _ = self.0.kill();
let _ = self.0.wait();
}
}
@@ -0,0 +1,68 @@
//! Periodic scratch accounting is separate from immutable runtime/tool inputs.
use std::fs;
use std::path::Path;
const MAX_BYTES: u64 = 512 * 1024 * 1024;
const MAX_ENTRIES: usize = 65_536;
const MAX_DEPTH: usize = 128;
#[derive(Default)]
struct Budget {
bytes: u64,
entries: usize,
}
pub(super) fn check(root: &Path) -> Result<(), String> {
visit(root, 0, &mut Budget::default()).map_err(|error| format!("build scratch budget: {error}"))
}
#[cfg(unix)]
fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> {
use std::os::fd::AsRawFd;
use std::os::unix::fs::OpenOptionsExt;
if depth > MAX_DEPTH {
return Err("directory depth exceeds limit".into());
}
// An open directory pins the inode throughout enumeration. Child-created
// symlinks and rename races cannot redirect accounting outside this tree.
let directory = match fs::OpenOptions::new()
.read(true)
.custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::DIRECTORY).bits() as i32)
.open(path)
{
Ok(file) => file,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(format!("cannot pin scratch directory: {error}")),
};
let pinned = format!("/proc/self/fd/{}", directory.as_raw_fd());
for entry in fs::read_dir(pinned).map_err(|error| error.to_string())? {
let entry = entry.map_err(|error| error.to_string())?;
let metadata = match entry.metadata() {
Ok(metadata) => metadata,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
Err(error) => return Err(error.to_string()),
};
budget.entries += 1;
if budget.entries > MAX_ENTRIES {
return Err("entry count exceeds limit".into());
}
if metadata.is_dir() {
visit(&entry.path(), depth + 1, budget)?;
} else {
budget.bytes = budget
.bytes
.checked_add(metadata.len())
.ok_or("byte count overflow")?;
if budget.bytes > MAX_BYTES {
return Err("bytes exceed 512 MiB limit".into());
}
}
}
Ok(())
}
#[cfg(not(unix))]
fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> {
Err("managed build scratch accounting requires its Linux sandbox".into())
}
+119
View File
@@ -0,0 +1,119 @@
"""Fixed PEP 517 frontend, executed only inside the managed build sandbox."""
import importlib
import json
import os
from pathlib import Path
import re
import shutil
import sys
import tomllib
def build_system(source):
project = source / "pyproject.toml"
config = tomllib.loads(project.read_text("utf-8")) if project.exists() else {}
system = config.get("build-system")
if system is None:
system = {"requires": ["setuptools>=40.8.0"], "build-backend": "setuptools.build_meta:__legacy__"}
if not isinstance(system, dict) or not isinstance(system.get("requires"), list):
raise ValueError("build-system.requires must be an array")
requires = system["requires"]
if len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
raise ValueError("invalid static build requirements")
backend = system.get("build-backend", "setuptools.build_meta:__legacy__")
identifier = r"[A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*"
if not isinstance(backend, str) or not re.fullmatch(identifier + "(?::" + identifier + ")?", backend):
raise ValueError("invalid build-backend entry point")
paths = system.get("backend-path", [])
if not isinstance(paths, list) or len(paths) > 64:
raise ValueError("invalid backend-path")
for name in paths:
if not isinstance(name, str) or not name or Path(name).is_absolute() or "\\" in name:
raise ValueError("backend-path must be source-relative")
resolved = (source / name).resolve()
if not resolved.is_relative_to(source.resolve()) or not resolved.is_dir():
raise ValueError("backend-path escapes the source tree")
return {"requires": requires, "backend": backend, "backend_path": paths}
def backend_object(source, system):
paths = [(source / path).resolve() for path in system["backend_path"]]
sys.path[:0] = [str(path) for path in paths]
module_name, _, object_name = system["backend"].partition(":")
module = importlib.import_module(module_name)
if paths:
origin = Path(module.__file__).resolve()
if not any(origin.is_relative_to(path) for path in paths):
raise ValueError("in-tree backend was loaded outside backend-path")
backend = module
for part in object_name.split(".") if object_name else []:
backend = getattr(backend, part)
return backend
def leaf(value, suffix):
if not isinstance(value, str) or Path(value).name != value or not value.endswith(suffix):
raise ValueError("build hook returned an invalid output name")
return value
def execute(request, source, output):
system = build_system(source)
if request["phase"] == "inspect":
return system
if system != request["system"]:
raise ValueError("source build configuration changed between hooks")
backend = backend_object(source, system)
settings = request["config_settings"]
if request["phase"] == "requires":
hook = getattr(backend, "get_requires_for_build_wheel", None)
requires = hook(settings) if hook else []
if not isinstance(requires, list) or len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
raise ValueError("invalid dynamic build requirements")
return {"requires": requires}
if request["phase"] == "metadata":
metadata = output / "metadata"
metadata.mkdir()
hook = getattr(backend, "prepare_metadata_for_build_wheel", None)
prepared = metadata / leaf(hook(str(metadata), settings), ".dist-info") if hook else None
if prepared is not None and (not prepared.is_dir() or prepared.is_symlink()):
raise ValueError("metadata hook did not create its declared directory")
return {"metadata": str(prepared.relative_to(output)) if prepared else None}
prepared = None
if request.get("metadata") is not None:
shutil.copytree("/worker/prepared", output / "prepared")
prepared = output / "prepared" / request["metadata"]
if not prepared.resolve().is_relative_to((output / "prepared").resolve()):
raise ValueError("prepared metadata path escapes its snapshot")
wheels = output / "wheels"
wheels.mkdir()
name = leaf(backend.build_wheel(str(wheels), settings, str(prepared) if prepared else None), ".whl")
return {"filename": name}
def main():
request = json.loads(Path("/worker/request.json").read_text("utf-8"))
work = Path("/data/build")
source = work / "source"
shutil.copytree("/worker/source", source)
output = work / "output"
output.mkdir()
(work / "tmp").mkdir()
# These values are part of the fixed frontend, never project/host settings.
os.environ.update({
"PATH": "/worker/runtime/bin:/usr/bin",
"CC": "/usr/bin/cc", "CXX": "/usr/bin/c++",
"AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib",
"LDSHARED": "/usr/bin/cc -shared",
"CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.",
"CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"],
"TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0",
})
os.chdir(source)
result = execute(request, source, output)
(output / "result.json").write_text(json.dumps(result, sort_keys=True), "utf-8")
if __name__ == "__main__":
main()
+564
View File
@@ -0,0 +1,564 @@
//! PEP 517 source candidates enter the normal dependency solver as real wheels.
use std::collections::{BTreeMap, BTreeSet};
use std::ffi::OsString;
use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
use crate::libs::{
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
sha256,
};
use crate::workers::{python_wheel::Wheel, runtime};
use serde_json::{Value, json};
struct Environment {
files: BTreeMap<String, (Vec<u8>, bool)>,
assets: Vec<(String, Vec<u8>)>,
executable: String,
version: String,
extension_suffixes: Vec<String>,
descriptor: pack::Descriptor,
}
struct Active<'a> {
session: &'a Session<'a>,
key: String,
}
impl Drop for Active<'_> {
fn drop(&mut self) {
self.session.active.borrow_mut().remove(&self.key);
}
}
impl Session<'_> {
pub(crate) fn python(
&self,
resolver: &RegistryResolver<'_>,
spec: &LibSpec,
runtime: &RegistryRuntime,
filename: &str,
source: &[u8],
) -> Result<Product, String> {
let key = format!(
"{}@{}:{}:{}",
spec.distribution_name(),
spec.version,
sha256(source),
runtime.pack.sha256
);
if let Some(product) = self.products.borrow().get(&key) {
return Ok(product.clone());
}
if self.active.borrow().len() >= 16 || !self.active.borrow_mut().insert(key.clone()) {
return Err(format!(
"cyclic or overly deep Python build requirement for {}",
spec.key()
));
}
let _active = Active {
session: self,
key: key.clone(),
};
let sources = source_files(filename, source)?;
let environment = Environment::new(self, runtime)?;
let empty = LockFile::new(vec![])?;
let inspected =
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
let system: PythonSystem = serde_json::from_value(inspected.response)
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
let mut requirements = system.requires.clone();
let mut dependencies = resolver.build_requirements(&requirements, runtime)?;
let mut stable = None;
let mut discovered = BTreeSet::new();
for _ in 0..8 {
let result = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"requires","system":system}),
None,
)?;
let dynamic: Vec<String> = serde_json::from_value(
result
.response
.get("requires")
.cloned()
.ok_or("build backend omitted requirements")?,
)
.map_err(|error| format!("invalid dynamic Python build requirements: {error}"))?;
discovered.extend(dynamic);
requirements = system.requires.iter().chain(&discovered).cloned().collect();
let next = resolver.build_requirements(&requirements, runtime)?;
if next == dependencies {
stable = Some(discovered.iter().cloned().collect());
break;
}
dependencies = next;
}
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
let metadata = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"metadata","system":system}),
None,
)?;
let result = environment.run(
resolver,
&dependencies,
&sources,
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
Some(&metadata.outputs),
)?;
let filename = result
.response
.get("filename")
.and_then(Value::as_str)
.ok_or("build backend omitted wheel filename")?;
crate::workers::valid_path(filename)?;
if filename.contains('/') || !filename.ends_with(".whl") {
return Err("build backend returned invalid wheel filename".into());
}
let bytes = result
.outputs
.get(&format!("wheels/{filename}"))
.ok_or("build backend did not produce its declared wheel")?
.clone();
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
compare_metadata(prefix, &metadata.outputs, &bytes)?;
}
let output = artifact(
runtime,
format!(
"lib/pip/{}/{}/archive.whl",
sha256(spec.distribution_name().as_bytes()),
spec.version
),
&bytes,
);
let source_artifact = artifact(
runtime,
format!("build/source/{}/source", sha256(source)),
source,
);
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
return Err("build artifact cache identity mismatch".into());
}
}
let base: LibSpec = format!("pip:{}@{}", spec.distribution_name(), spec.version).parse()?;
let receipt = Receipt {
ecosystem: Ecosystem::Pip,
roots: vec![base],
sources: vec![source_artifact],
graph: None,
runtime: runtime.pack.clone(),
auxiliary_runtime: None,
native_entries: BTreeSet::new(),
native_rejections: BTreeMap::new(),
python_markers: runtime.python_markers.clone(),
tools: environment.descriptor,
frontend: super::frontend(&Ecosystem::Pip)?,
python: Some(system),
dynamic_requires: dynamic,
dependencies: Box::new(dependencies),
config_settings: BTreeMap::new(),
output,
};
let product = Product {
bytes,
filename: filename.into(),
receipt,
};
self.remember(key, product.clone())?;
Ok(product)
}
}
fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArtifact {
LockedArtifact {
target: runtime.target.clone(),
path,
bytes: bytes.len() as u64,
sha256: sha256(bytes),
}
}
struct Source {
files: Vec<(String, Vec<u8>)>,
executables: BTreeSet<String>,
}
fn source_files(filename: &str, bytes: &[u8]) -> Result<Source, String> {
let extension = if filename.ends_with(".tar.gz") {
"tgz"
} else if filename.ends_with(".zip") {
"zip"
} else {
return Err("unsupported Python source archive format".into());
};
let files = archive_files(extension, bytes)?;
let executables = crate::libs::registry::archive_executables(extension, bytes)?;
let root = files
.first()
.and_then(|(path, _)| path.split_once('/'))
.map(|(root, _)| format!("{root}/"))
.ok_or("Python source archive needs one root directory")?;
let files = files
.into_iter()
.map(|(path, bytes)| {
let path = path
.strip_prefix(&root)
.ok_or("Python source archive contains multiple roots")?
.to_owned();
crate::workers::valid_path(&path)?;
Ok((path, bytes))
})
.collect::<Result<Vec<_>, String>>()?;
let executables = executables
.into_iter()
.map(|path| {
path.strip_prefix(&root)
.map(str::to_owned)
.ok_or("source executable escapes archive root".into())
})
.collect::<Result<_, String>>()?;
Ok(Source { files, executables })
}
struct ResultFiles {
response: Value,
outputs: BTreeMap<String, Vec<u8>>,
}
impl Environment {
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
super::require_execution_target(&requested.target, "Python source build")?;
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
if runtime.pack != requested.pack || sha256(&bytes) != requested.pack.sha256 {
return Err("build Python runtime differs from resolver identity".into());
}
let runtime_files = archive_files("tgz", &bytes)?;
let manifest = runtime::validate(&runtime_files, &Ecosystem::Pip, &requested.target)?;
if manifest.python_wheel_tags != requested.python_wheel_tags
|| manifest.python_markers != requested.python_markers
{
return Err("build runtime policy differs from signed descriptor".into());
}
let executable = manifest
.executable
.clone()
.ok_or("build Python executable missing")?;
let executable = format!("runtime/{executable}");
let mut files = runtime_files
.into_iter()
.map(|(path, bytes)| {
let executable = manifest.executable(&path);
(format!("runtime/{path}"), (bytes, executable))
})
.collect::<BTreeMap<_, _>>();
let (descriptor, bytes) = session.inputs.tools(Ecosystem::Pip)?;
descriptor.validate(requested, None)?;
let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?;
let assets = session.inputs.assets()?;
dever_sandbox::validate_assets(&assets)?;
pack::validate_tools(&tools, &tool_files, &assets)?;
for (path, bytes) in tool_files {
let executable = tools.executables.contains(&path);
if files.insert(path, (bytes, executable)).is_some() {
return Err("build tools overlap runtime inputs".into());
}
}
let version = requested
.python_markers
.as_ref()
.ok_or("build Python markers absent")?
.python_version()
.to_string();
Ok(Self {
files,
assets,
executable,
version,
extension_suffixes: manifest.python_extension_suffixes,
descriptor,
})
}
fn run(
&self,
resolver: &RegistryResolver<'_>,
dependencies: &LockFile,
source: &Source,
mut request: Value,
prepared: Option<&BTreeMap<String, Vec<u8>>>,
) -> Result<ResultFiles, String> {
let stage = Stage::new()?;
let input = stage.0.join("input");
for (path, (bytes, executable)) in &self.files {
stage.write(&format!("input/{path}"), bytes, *executable)?;
}
for (path, bytes) in &self.assets {
stage.write(
&format!("sandbox/{path}"),
bytes,
dever_sandbox::asset_executable(path),
)?;
}
for (path, bytes) in &source.files {
stage.write(
&format!("input/source/{path}"),
bytes,
source.executables.contains(path),
)?;
}
if let Some(prepared) = prepared {
for (path, bytes) in prepared {
stage.write(&format!("input/prepared/{path}"), bytes, false)?;
}
}
let mut installed = BTreeMap::<String, Vec<u8>>::new();
let runtime = resolver
.runtimes
.get(&Ecosystem::Pip)
.ok_or("build dependency runtime is missing")?;
let mut distributions = BTreeSet::new();
for lib in &dependencies.libs {
super::validate_python_policy(
dependencies,
lib,
runtime
.python_markers
.as_ref()
.ok_or("Python build markers absent")?,
)?;
if lib.runtime != runtime.pack || lib.artifacts.len() != 1 {
return Err(
"build dependency does not use the exact Python runtime and wheel".into(),
);
}
let artifact = &lib.artifacts[0];
let bytes =
resolver
.store
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
wheel.validate_dependencies(
lib,
runtime
.python_markers
.as_ref()
.ok_or("Python build markers absent")?,
)?;
if !distributions.insert((
lib.spec.distribution_name().to_owned(),
lib.spec.version.clone(),
)) {
continue;
}
for file in wheel.install(&self.executable, &self.version)? {
if self.files.contains_key(&file.path)
|| installed
.insert(file.path.clone(), file.bytes.clone())
.is_some()
{
return Err("build dependency installation has conflicting file owners".into());
}
stage.write(
&format!("input/{}", file.path),
&file.bytes,
file.executable,
)?;
}
}
let mut supplied = self
.files
.iter()
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
.collect::<BTreeMap<_, _>>();
for (path, bytes) in &self.assets {
supplied.insert(format!("sandbox/{path}"), bytes);
}
for (path, bytes) in &installed {
supplied.insert(path.clone(), bytes);
}
crate::workers::python_wheel::validate_native(
&runtime.target,
&self.extension_suffixes,
&self.executable,
&supplied,
&installed.keys().cloned().collect(),
)?;
request["python_version"] = json!(self.version);
request["config_settings"] = json!({});
stage.write(
"input/request.json",
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
false,
)?;
stage.write("input/frontend.py", include_bytes!("python.py"), false)?;
let work = stage.0.join("work");
fs::create_dir(&work).map_err(|error| error.to_string())?;
let grants = [dever_sandbox::Grant {
source: work.clone(),
destination: "/data/build".into(),
writable: true,
}];
let arguments = [
OsString::from("-I"),
OsString::from("-B"),
input.join("frontend.py").into_os_string(),
];
let executable = input.join(&self.executable);
let assets = stage.0.join("sandbox");
let launch = dever_sandbox::Launch {
assets: &assets,
worker: &input,
executable: &executable,
arguments: &arguments,
working_directory: &input,
capabilities: dever_sandbox::Capabilities {
network: false,
process: true,
},
grants: &grants,
};
stage.run(
launch.command_for_build(&input.join("rootfs"))?,
"isolated Python build hook",
&work,
)?;
if let Some(session) = resolver.build {
session.inputs.diagnostic(&stage.diagnostics()?);
}
let outputs = output_files(&work.join("output"))?;
let response = serde_json::from_slice(
outputs
.get("result.json")
.ok_or("build hook produced no result")?,
)
.map_err(|error| format!("invalid build hook response: {error}"))?;
Ok(ResultFiles { response, outputs })
}
}
pub(super) fn output_files(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
let mut files = BTreeMap::new();
let mut pending = vec![root.to_owned()];
let mut remaining = crate::workers::MAX_TREE_BYTES;
let mut entries = 0usize;
#[cfg(unix)]
let mut links = BTreeMap::<(u64, u64), (u64, u64)>::new();
while let Some(directory) = pending.pop() {
if fs::symlink_metadata(&directory)
.map_err(|error| error.to_string())?
.file_type()
.is_symlink()
{
return Err("build output directory is a symbolic link".into());
}
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
entries += 1;
if entries > crate::workers::MAX_TREE_FILES {
return Err("build output exceeds file budget".into());
}
let entry = entry.map_err(|error| error.to_string())?;
let kind = entry.file_type().map_err(|error| error.to_string())?;
if kind.is_dir() {
pending.push(entry.path());
continue;
}
if !kind.is_file() {
return Err("build output contains a link or special file".into());
}
let mut options = fs::OpenOptions::new();
options.read(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
}
let file = options
.open(entry.path())
.map_err(|error| error.to_string())?;
let metadata = file.metadata().map_err(|error| error.to_string())?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let aliases = links
.entry((metadata.dev(), metadata.ino()))
.or_insert((metadata.nlink(), 0));
if aliases.0 != metadata.nlink() {
return Err("build output link count changed during collection".into());
}
aliases.1 += 1;
}
if !metadata.is_file() || metadata.len() > remaining as u64 {
return Err("build output exceeds byte budget".into());
}
let mut bytes = Vec::new();
file.take(remaining as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|error| error.to_string())?;
remaining = remaining
.checked_sub(bytes.len())
.ok_or("build output exceeds byte budget")?;
let path = entry.path();
let name = path
.strip_prefix(root)
.map_err(|error| error.to_string())?
.to_str()
.ok_or("build output path is not UTF-8")?
.to_owned();
crate::workers::valid_path(&name)?;
files.insert(name, bytes);
}
}
// node-gyp's COPY rule may hard-link two output names. Namespace teardown
// precedes this read: accept aliases only when every link is accounted for
// inside the output root, then publish independent immutable byte entries.
#[cfg(unix)]
if links
.values()
.any(|(expected, observed)| expected != observed)
{
return Err("build output has a hard-link alias outside its output root".into());
}
Ok(files)
}
fn compare_metadata(
prefix: &str,
outputs: &BTreeMap<String, Vec<u8>>,
wheel: &[u8],
) -> Result<(), String> {
crate::workers::valid_path(prefix)?;
let distribution = PathBuf::from(prefix)
.file_name()
.and_then(|name| name.to_str())
.ok_or("invalid prepared metadata directory")?
.to_owned();
let wheel = archive_files("whl", wheel)?
.into_iter()
.collect::<BTreeMap<_, _>>();
let prefix = format!("{prefix}/");
let mut count = 0;
for (path, bytes) in outputs {
if let Some(relative) = path.strip_prefix(&prefix) {
count += 1;
if wheel.get(&format!("{distribution}/{relative}")) != Some(bytes) {
return Err(
"built wheel metadata differs from prepare_metadata_for_build_wheel".into(),
);
}
}
}
if count == 0 {
return Err("metadata hook produced an empty directory".into());
}
Ok(())
}
+80
View File
@@ -0,0 +1,80 @@
//! Serialize dependency mutations from the first project read to publication.
//! The guard file is stable: replacing dever.lock must not replace its mutex.
use std::fs::{self, File, Metadata, OpenOptions};
use std::path::Path;
pub(crate) struct ProjectMutation {
_file: File,
}
impl ProjectMutation {
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
if !matches!(command, "add" | "update" | "remove") {
return Ok(None);
}
let directory = fs::symlink_metadata(root)
.map_err(|error| format!("cannot inspect project mutation directory: {error}"))?;
validate(&directory, true)?;
let path = root.join(".dever-project.lock");
if let Ok(metadata) = fs::symlink_metadata(&path) {
validate(&metadata, false)?;
}
let mut options = OpenOptions::new();
options.read(true).write(true).create(true).truncate(false);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options
.mode(0o600)
.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
}
let file = options
.open(&path)
.map_err(|error| format!("cannot open project mutation lock: {error}"))?;
let opened = file.metadata().map_err(|error| error.to_string())?;
validate(&opened, false)?;
let current = fs::symlink_metadata(&path).map_err(|error| error.to_string())?;
validate(&current, false)?;
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
if opened.dev() != current.dev() || opened.ino() != current.ino() {
return Err("project mutation lock changed while opening".into());
}
}
fs2::FileExt::try_lock_exclusive(&file).map_err(|error| {
if error.kind() == std::io::ErrorKind::WouldBlock {
"project dependency mutation is busy; retry after the current command finishes"
.into()
} else {
format!("cannot acquire project mutation lock: {error}")
}
})?;
Ok(Some(Self { _file: file }))
}
}
fn validate(metadata: &Metadata, directory: bool) -> Result<(), String> {
if metadata.file_type().is_symlink()
|| (directory && !metadata.is_dir())
|| (!directory && !metadata.is_file())
{
return Err(
"project mutation path must be a real directory or regular lock file, without symlinks"
.into(),
);
}
#[cfg(unix)]
{
use std::os::unix::fs::MetadataExt;
let forbidden = if directory { 0o022 } else { 0o077 };
if metadata.uid() != rustix::process::geteuid().as_raw()
|| metadata.mode() & forbidden != 0
|| (!directory && metadata.nlink() != 1)
{
return Err("project mutation directory must be owned and not group/other writable; lock must be owned, private and unlinked elsewhere".into());
}
}
Ok(())
}
File diff suppressed because it is too large. Load diff
+995
View File
@@ -0,0 +1,995 @@
//! npm package instances follow Node's ancestor lookup, not one global version
//! per name. Archives remain content-addressed and shared between instances.
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use node_semver::{Range, Version};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use super::{Ecosystem, LibSpec, LockFile, LockedLib, RegistryRuntime, archive_files};
mod resolve;
pub(super) use resolve::resolve;
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Kind {
Dependency,
Optional,
Peer,
OptionalPeer,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Omission {
Target,
Unavailable,
Conflict,
OptionalPeerAbsent,
BuildFailure,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Edge {
pub name: String,
pub requirement: String,
pub kind: Kind,
pub target: Option<String>,
pub omission: Option<Omission>,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Source {
pub archive: LibSpec,
pub prefix: String,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Instance {
pub path: String,
pub spec: LibSpec,
pub source: Source,
/// Exact bundled layout of a root archive, including hoisted transitives.
pub bundled: BTreeMap<String, LibSpec>,
pub edges: Vec<Edge>,
}
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Environment {
pub roots: Vec<LibSpec>,
pub instances: Vec<Instance>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub build: Option<String>,
}
impl Environment {
/// Explicit single-package graph for author fixtures with no dependencies.
/// Materialization still checks package.json against this graph.
pub fn single(spec: LibSpec) -> Self {
Self {
build: None,
roots: vec![spec.clone()],
instances: vec![Instance {
path: child("", &spec.name),
spec: spec.clone(),
source: Source {
archive: spec,
prefix: "package/".into(),
},
bundled: BTreeMap::new(),
edges: Vec::new(),
}],
}
}
}
pub fn roots(specs: &[LibSpec]) -> Vec<LibSpec> {
let mut roots = specs
.iter()
.filter(|spec| spec.ecosystem == Ecosystem::Npm)
.cloned()
.collect::<Vec<_>>();
roots.sort();
roots.dedup();
roots
}
pub fn environment<'a>(
lock: &'a LockFile,
requested: &[LibSpec],
) -> Result<Option<&'a Environment>, String> {
let requested = roots(requested);
if requested.is_empty() {
return Ok(None);
}
lock.npm
.iter()
.find(|env| env.roots == requested)
.map(Some)
.ok_or("npm Worker roots have no locked installation environment".into())
}
pub fn archive_roots(lock: &LockFile, requested: &[LibSpec]) -> Result<BTreeSet<LibSpec>, String> {
let requested = roots(requested);
if requested.is_empty() {
return Ok(BTreeSet::new());
}
if let Some(environment) = lock
.npm
.iter()
.find(|environment| environment.roots == requested)
{
return Ok(environment
.instances
.iter()
.map(|node| node.source.archive.clone())
.collect());
}
// Built-in metadata fixtures are not registry packages and have no tree.
let fixtures = super::super::FixtureRegistry::builtin();
if requested
.iter()
.all(|spec| fixtures.package(spec).is_some())
{
return Ok(requested.into_iter().collect());
}
Err("npm roots have no exact locked installation environment".into())
}
pub fn retain(lock: &LockFile, groups: &[Vec<LibSpec>]) -> Result<Vec<Environment>, String> {
let mut environments = Vec::new();
for group in groups {
let requested = roots(group);
if requested.is_empty() {
continue;
}
let original = lock
.npm
.iter()
.find(|env| env.roots == requested)
.or_else(|| {
lock.npm
.iter()
.find(|env| requested.iter().all(|root| env.roots.contains(root)))
});
let Some(original) = original else {
archive_roots(lock, group)?;
continue;
};
let mut queue = requested
.iter()
.map(|spec| child("", &spec.name))
.collect::<VecDeque<_>>();
let mut paths = BTreeSet::new();
while let Some(path) = queue.pop_front() {
if !paths.insert(path.clone()) {
continue;
}
let node = original
.instances
.iter()
.find(|node| node.path == path)
.ok_or("npm retained graph lacks a dependency instance")?;
queue.extend(node.edges.iter().filter_map(|edge| edge.target.clone()));
}
environments.push(Environment {
build: original.build.clone(),
roots: requested,
instances: original
.instances
.iter()
.filter(|node| paths.contains(&node.path))
.cloned()
.collect(),
});
}
normalize(&mut environments)?;
Ok(environments)
}
pub(super) fn child(parent: &str, name: &str) -> String {
if parent.is_empty() {
format!("node_modules/{name}")
} else {
format!("{parent}/node_modules/{name}")
}
}
pub(super) fn parent(path: &str) -> &str {
path.rsplit_once("/node_modules/")
.map_or("", |(parent, _)| parent)
}
fn validate_path(path: &str) -> Result<(), String> {
if path.len() > 4096 {
return Err("npm instance path exceeds its bound".into());
}
let path = path
.strip_prefix("node_modules/")
.ok_or("invalid npm instance path")?;
for name in path.split("/node_modules/") {
super::super::validate_name(&Ecosystem::Npm, name)?;
}
Ok(())
}
pub(super) fn visible<'a>(
instances: &'a BTreeMap<String, Instance>,
consumer: &str,
name: &str,
peer: bool,
) -> Option<&'a Instance> {
let mut scope = if peer { parent(consumer) } else { consumer };
loop {
if let Some(found) = instances.get(&child(scope, name)) {
return Some(found);
}
if scope.is_empty() {
return None;
}
scope = parent(scope);
}
}
#[derive(Clone, Debug)]
pub(super) struct Metadata {
pub spec: LibSpec,
pub edges: Vec<Edge>,
pub bundles: BTreeSet<String>,
pub document: Value,
}
fn requirements(document: &Value, field: &str) -> Result<BTreeMap<String, String>, String> {
let Some(value) = document.get(field) else {
return Ok(BTreeMap::new());
};
value
.as_object()
.ok_or_else(|| format!("npm {field} must be an object"))?
.iter()
.map(|(name, value)| {
super::super::validate_name(&Ecosystem::Npm, name)?;
let rule = value.as_str().ok_or("npm dependency range must be text")?;
rule.parse::<Range>()
.map_err(|error| format!("invalid npm range for {name}: {error}"))?;
Ok((name.clone(), rule.into()))
})
.collect()
}
impl Metadata {
pub(super) fn parse(document: Value) -> Result<Self, String> {
let spec: LibSpec = format!(
"npm:{}@{}",
super::field(&document, "name")?,
super::field(&document, "version")?
)
.parse()?;
let mut regular = requirements(&document, "dependencies")?;
let optional = requirements(&document, "optionalDependencies")?;
regular.retain(|name, _| !optional.contains_key(name));
let mut peers = requirements(&document, "peerDependencies")?;
let peer_meta = document
.get("peerDependenciesMeta")
.map(|value| {
value
.as_object()
.ok_or("npm peerDependenciesMeta must be an object")
})
.transpose()?;
if let Some(meta) = peer_meta {
for (name, value) in meta {
if !peers.contains_key(name)
|| !value.is_object()
|| value.get("optional").is_some_and(|flag| !flag.is_boolean())
{
return Err("invalid npm optional peer metadata".into());
}
}
}
// npm has one outgoing edge per name: optional > production > peer.
peers.retain(|name, _| !regular.contains_key(name) && !optional.contains_key(name));
let mut edges = Vec::new();
for (entries, kind) in [
(regular.clone(), Kind::Dependency),
(optional.clone(), Kind::Optional),
(peers, Kind::Peer),
] {
for (name, requirement) in entries {
let kind = if kind == Kind::Peer
&& peer_meta
.and_then(|meta| meta.get(&name))
.and_then(|value| value.get("optional"))
.and_then(Value::as_bool)
== Some(true)
{
Kind::OptionalPeer
} else {
kind.clone()
};
edges.push(Edge {
name,
requirement,
kind,
target: None,
omission: None,
});
}
}
edges.sort();
let bundles = |value: &Value| -> Result<BTreeSet<String>, String> {
match value {
Value::Bool(true) => Ok(regular.keys().chain(optional.keys()).cloned().collect()),
Value::Bool(false) => Ok(BTreeSet::new()),
Value::Array(values) => values
.iter()
.map(|value| {
let name = value
.as_str()
.ok_or("npm bundled dependency name must be text")?;
super::super::validate_name(&Ecosystem::Npm, name)?;
if !regular.contains_key(name) && !optional.contains_key(name) {
return Err(
"npm bundled dependency is not a declared dependency".into()
);
}
Ok(name.to_owned())
})
.collect(),
_ => Err("npm bundleDependencies must be an array or boolean".into()),
}
};
let first = document
.get("bundleDependencies")
.map(bundles)
.transpose()?;
let second = document
.get("bundledDependencies")
.map(bundles)
.transpose()?;
if first.is_some() && second.is_some() && first != second {
return Err("npm bundled dependency aliases disagree".into());
}
Ok(Self {
spec,
edges,
bundles: first.or(second).unwrap_or_default(),
document,
})
}
pub(super) fn eligible(&self, runtime: &RegistryRuntime) -> Result<bool, String> {
let (os, cpu) = runtime
.target
.split_once('-')
.ok_or("npm runtime target lacks platform identity")?;
let os = match os {
"windows" => "win32",
"macos" => "darwin",
other => other,
};
let cpu = match cpu {
"x86_64" => "x64",
"aarch64" => "arm64",
other => other,
};
for (field, selected) in [
("os", Some(os)),
("cpu", Some(cpu)),
("libc", runtime.npm_libc.as_deref()),
] {
if field == "libc" && os != "linux" {
continue;
}
let Some(value) = self.document.get(field) else {
continue;
};
let values = value
.as_array()
.ok_or_else(|| format!("npm {field} must be an array"))?;
let values = values
.iter()
.map(|value| value.as_str().ok_or("npm platform selector must be text"))
.collect::<Result<Vec<_>, _>>()?;
let Some(selected) = selected else {
return Ok(false);
};
if values
.iter()
.any(|value| value.strip_prefix('!') == Some(selected))
|| (values.iter().any(|value| !value.starts_with('!'))
&& !values.contains(&selected)
&& !values.contains(&"any"))
{
return Ok(false);
}
}
Ok(true)
}
}
pub(super) struct Archive {
pub files: Vec<(String, Vec<u8>)>,
pub packages: BTreeMap<String, Metadata>,
}
impl Archive {
pub(super) fn bundled(&self) -> BTreeMap<String, LibSpec> {
self.packages
.iter()
.filter(|(prefix, _)| prefix.as_str() != "package/")
.map(|(prefix, metadata)| {
(
prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/')
.into(),
metadata.spec.clone(),
)
})
.collect()
}
pub(super) fn parse(spec: &LibSpec, bytes: &[u8]) -> Result<Self, String> {
let files = archive_files("tgz", bytes)?;
let mut packages = BTreeMap::<String, Metadata>::new();
for (path, bytes) in &files {
let relative = path
.strip_prefix("package/")
.ok_or("npm archive must have a package/ root")?;
if relative == "package.json" || relative.ends_with("/package.json") {
let prefix = path.strip_suffix("package.json").unwrap();
if prefix != "package/" {
let nested = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
if validate_path(nested).is_err() {
continue;
}
}
let document = super::parse_json(bytes)?;
let metadata = Metadata::parse(document)?;
if prefix == "package/" {
if metadata.spec != *spec {
return Err(
"npm package.json identity differs from its registry archive".into(),
);
}
} else if !prefix.ends_with(&format!("node_modules/{}/", metadata.spec.name)) {
return Err("npm bundled package identity differs from its path".into());
}
packages.insert(prefix.into(), metadata);
}
}
let root = packages
.get("package/")
.ok_or("npm archive lacks package.json")?;
for name in &root.bundles {
if !packages.contains_key(&format!("package/node_modules/{name}/")) {
return Err("npm declared bundled dependency is absent from its archive".into());
}
}
for (path, _) in &files {
if !packages.contains_key(&source_owner(path)?) {
return Err("npm archive has unowned node_modules content".into());
}
}
let nodes = packages
.iter()
.filter(|(prefix, _)| prefix.as_str() != "package/")
.map(|(prefix, metadata)| {
let path = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/')
.to_owned();
(
path.clone(),
Instance {
path,
spec: metadata.spec.clone(),
source: Source {
archive: spec.clone(),
prefix: prefix.clone(),
},
bundled: BTreeMap::new(),
edges: Vec::new(),
},
)
})
.collect::<BTreeMap<_, _>>();
let mut pending = root
.bundles
.iter()
.map(|name| child("", name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = nodes
.get(&path)
.ok_or("npm bundle references a missing package")?;
for edge in &packages[&node.source.prefix].edges {
let peer = matches!(edge.kind, Kind::Peer | Kind::OptionalPeer);
if let Some(found) = visible(&nodes, &path, &edge.name, peer) {
if !Version::from_str(&found.spec.version)
.map_err(|error| error.to_string())?
.satisfies(
&edge
.requirement
.parse::<Range>()
.map_err(|error| error.to_string())?,
)
{
return Err("npm bundled dependency has an incompatible version".into());
}
pending.push_back(found.path.clone());
} else if edge.kind == Kind::Dependency
&& edge.name != spec.name
&& !root.edges.iter().any(|root| root.name == edge.name)
{
return Err("npm bundled dependency closure is incomplete".into());
}
}
}
if reached.len() != nodes.len() {
return Err("npm archive contains undeclared or unreachable bundled packages".into());
}
Ok(Self { files, packages })
}
}
fn source_owner(path: &str) -> Result<String, String> {
let Some((parent, tail)) = path.rsplit_once("/node_modules/") else {
return Ok("package/".into());
};
let mut parts = tail.split('/');
let first = parts.next().ok_or("npm archive has invalid package path")?;
let name = if first.starts_with('@') {
format!(
"{first}/{}",
parts
.next()
.ok_or("npm scoped package path is incomplete")?
)
} else {
first.into()
};
let prefix = format!("{parent}/node_modules/{name}");
validate_path(
prefix
.strip_prefix("package/")
.ok_or("npm bundled archive has invalid root")?,
)?;
Ok(format!("{prefix}/"))
}
fn validate_source_owner(environment: &Environment, node: &Instance) -> Result<(), String> {
for owner in &environment.instances {
for (relative, spec) in &owner.bundled {
if node.path == format!("{}/{relative}", owner.path)
&& (node.spec != *spec
|| node.source.archive != owner.source.archive
|| node.source.prefix != format!("package/{relative}/"))
{
return Err(
"npm bundled instance is not owned by its parent archive installation".into(),
);
}
}
}
if node.source.prefix == "package/" {
for (relative, spec) in &node.bundled {
validate_path(relative)?;
if spec.ecosystem != Ecosystem::Npm
|| !relative.ends_with(&format!("node_modules/{}", spec.name))
{
return Err("invalid npm bundled layout identity".into());
}
super::super::validate_name(&Ecosystem::Npm, &spec.name)?;
super::super::validate_version(&Ecosystem::Npm, &spec.version)?;
}
return if node.source.archive == node.spec {
Ok(())
} else {
Err("npm root archive identity differs from its instance".into())
};
}
let relative = node
.source
.prefix
.strip_prefix("package/")
.and_then(|prefix| prefix.strip_suffix('/'))
.ok_or("invalid npm bundled source prefix")?;
validate_path(relative)?;
if !node.bundled.is_empty()
|| !environment.instances.iter().any(|owner| {
owner.source.prefix == "package/"
&& owner.source.archive == node.source.archive
&& owner.spec == node.source.archive
&& node.path == format!("{}/{relative}", owner.path)
&& owner.bundled.get(relative) == Some(&node.spec)
})
{
return Err("npm bundled instance is not owned by its parent archive installation".into());
}
Ok(())
}
/// Validate immutable manifests and plan borrowed files before cloning any
/// installation output. Reused archives must not multiply past the tree budget.
pub fn files<'a>(
environment: &Environment,
lock: &LockFile,
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
) -> Result<Vec<(String, Vec<u8>)>, String> {
source_files(environment, lock, archive, false)
}
pub fn build_files<'a>(
environment: &Environment,
lock: &LockFile,
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
) -> Result<Vec<(String, Vec<u8>)>, String> {
source_files(environment, lock, archive, true)
}
pub(crate) fn build_executables(
environment: &Environment,
archives: &BTreeMap<LibSpec, Vec<u8>>,
) -> Result<BTreeSet<String>, String> {
let modes = archives
.iter()
.map(|(spec, bytes)| Ok((spec, super::archive_executables("tgz", bytes)?)))
.collect::<Result<BTreeMap<_, _>, String>>()?;
let mut installed = BTreeSet::new();
for node in &environment.instances {
for path in modes
.get(&node.source.archive)
.ok_or("npm source modes missing")?
{
if source_owner(path)? == node.source.prefix {
let relative = path
.strip_prefix(&node.source.prefix)
.ok_or("npm source mode outside owner")?;
installed.insert(format!("{}/{relative}", node.path));
}
}
}
Ok(installed)
}
pub(crate) fn installed_owner(path: &str) -> Result<String, String> {
source_owner(&format!("package/{path}"))?
.strip_prefix("package/")
.map(|owner| owner.trim_end_matches('/').into())
.ok_or("npm output is outside its installation".into())
}
fn source_files<'a>(
environment: &Environment,
lock: &LockFile,
mut archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
preparing: bool,
) -> Result<Vec<(String, Vec<u8>)>, String> {
let mut archives = BTreeMap::new();
let mut archive_bytes = 0usize;
for node in &environment.instances {
validate_source_owner(environment, node)?;
if archives.contains_key(&node.source.archive) {
continue;
}
let lib = lock
.libs
.iter()
.find(|lib| lib.spec == node.source.archive)
.ok_or("npm graph source archive is missing")?;
let parsed = Archive::parse(&lib.spec, archive(lib)?)?;
for (_, bytes) in &parsed.files {
archive_bytes = archive_bytes
.checked_add(bytes.len())
.ok_or("npm archive size overflow")?;
if archive_bytes > crate::workers::MAX_TREE_BYTES {
return Err("npm archive cache exceeds its tree byte budget".into());
}
}
archives.insert(lib.spec.clone(), parsed);
}
let mut files = BTreeMap::new();
let mut installed_bytes = 0usize;
for node in &environment.instances {
let source = &archives[&node.source.archive];
if node.source.prefix == "package/" && node.bundled != source.bundled() {
return Err("npm bundled layout differs from its parent archive".into());
}
let metadata = source
.packages
.get(&node.source.prefix)
.ok_or("npm instance source package is absent")?;
if metadata.spec != node.spec
|| metadata.edges.len() != node.edges.len()
|| metadata.edges.iter().any(|expected| {
!node.edges.iter().any(|edge| {
edge.name == expected.name
&& edge.requirement == expected.requirement
&& edge.kind == expected.kind
})
})
{
return Err("npm locked dependency graph differs from package.json".into());
}
if !preparing
&& (metadata.document.get("gypfile").and_then(Value::as_bool) == Some(true)
|| metadata
.document
.get("scripts")
.and_then(Value::as_object)
.is_some_and(|scripts| {
["preinstall", "install", "postinstall"]
.iter()
.any(|name| scripts.contains_key(*name))
}))
{
return Err(
"managed npm Worker rejects lifecycle scripts and native build hooks".into(),
);
}
for (path, bytes) in &source.files {
if source_owner(path)? != node.source.prefix {
continue;
}
let relative = path
.strip_prefix(&node.source.prefix)
.ok_or("npm file is outside its instance source")?;
if !preparing && (relative.ends_with(".node") || relative == "binding.gyp") {
return Err("managed npm Worker rejects native package content".into());
}
let installed = format!("{}/{relative}", node.path);
installed_bytes = installed_bytes
.checked_add(bytes.len())
.ok_or("npm installed size overflow")?;
if installed_bytes > crate::workers::MAX_TREE_BYTES
|| files.len() >= crate::workers::MAX_TREE_FILES
{
return Err("npm installation exceeds its tree file/byte budget".into());
}
if files.insert(installed, bytes).is_some() {
return Err("npm installed files have conflicting owners".into());
}
}
}
Ok(files
.into_iter()
.map(|(path, bytes)| (path, bytes.clone()))
.collect())
}
pub fn normalize(environments: &mut Vec<Environment>) -> Result<(), String> {
for env in environments.iter_mut() {
env.roots.sort();
env.roots.dedup();
for node in &mut env.instances {
node.edges.sort();
}
env.instances
.sort_by(|left, right| left.path.cmp(&right.path));
}
environments.sort_by(|left, right| left.roots.cmp(&right.roots));
for pair in environments.windows(2) {
if pair[0].roots == pair[1].roots && pair[0] != pair[1] {
return Err("npm roots have inconsistent installation environments".into());
}
}
environments.dedup();
Ok(())
}
pub fn requires_build(files: &[(String, Vec<u8>)]) -> Result<bool, String> {
for (path, bytes) in files {
if path.ends_with(".node") || path.ends_with("/binding.gyp") {
return Ok(true);
}
if path.ends_with("/package.json") {
let metadata: Value = serde_json::from_slice(bytes)
.map_err(|error| format!("invalid npm installation metadata: {error}"))?;
if metadata.get("gypfile").and_then(Value::as_bool) == Some(true)
|| metadata
.get("scripts")
.and_then(Value::as_object)
.is_some_and(|scripts| {
["preinstall", "install", "postinstall"]
.iter()
.any(|name| scripts.contains_key(*name))
})
{
return Ok(true);
}
}
}
Ok(false)
}
/// Lifecycle output may add generated files, but cannot rewrite dependency
/// identities, create undeclared package instances, or escape the locked tree.
pub fn validate_built_files(
environment: &Environment,
files: &[(String, Vec<u8>)],
) -> Result<(), String> {
let nodes = environment
.instances
.iter()
.map(|node| (node.path.as_str(), node))
.collect::<BTreeMap<_, _>>();
let mut manifests = BTreeSet::new();
let mut names = BTreeSet::new();
let mut total = 0usize;
for (path, bytes) in files {
crate::workers::valid_path(path)?;
let owner = source_owner(&format!("package/{path}"))?;
let owner = owner
.strip_prefix("package/")
.ok_or("npm output is outside its installation")?
.trim_end_matches('/');
let node = nodes
.get(owner)
.ok_or("npm build output introduces an unlocked instance")?;
if !names.insert(path) {
return Err("npm build output repeats a file".into());
}
total = total
.checked_add(bytes.len())
.ok_or("npm output byte overflow")?;
if total > crate::workers::MAX_TREE_BYTES || names.len() > crate::workers::MAX_TREE_FILES {
return Err("npm output exceeds installation budget".into());
}
if path == &format!("{owner}/package.json") {
let metadata =
Metadata::parse(serde_json::from_slice(bytes).map_err(|error| error.to_string())?)?;
if metadata.spec != node.spec
|| metadata.edges.len() != node.edges.len()
|| metadata.edges.iter().any(|expected| {
!node.edges.iter().any(|edge| {
edge.name == expected.name
&& edge.requirement == expected.requirement
&& edge.kind == expected.kind
})
})
{
return Err("npm lifecycle changed locked package identity or dependencies".into());
}
manifests.insert(owner.to_owned());
}
}
if manifests.len() != nodes.len() {
return Err("npm lifecycle removed a locked package manifest".into());
}
Ok(())
}
pub fn doctor(lock: &LockFile) -> Result<(), String> {
for env in &lock.npm {
if env.roots.is_empty()
|| env.roots != roots(&env.roots)
|| env.instances.len() > super::MAX_PACKAGES
{
return Err("invalid npm installation environment roots or size".into());
}
let nodes = env
.instances
.iter()
.map(|node| (node.path.clone(), node.clone()))
.collect::<BTreeMap<_, _>>();
if nodes.len() != env.instances.len() {
return Err("duplicate npm instance path".into());
}
let mut pending = VecDeque::new();
for root in &env.roots {
let path = child("", &root.name);
if nodes.get(&path).map(|node| &node.spec) != Some(root) {
return Err("npm root is not installed at its exact path".into());
}
pending.push_back(path);
}
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let node = nodes
.get(&path)
.ok_or("npm edge references a missing instance")?;
validate_source_owner(env, node)?;
validate_path(&node.path)?;
super::super::validate_name(&Ecosystem::Npm, &node.spec.name)?;
super::super::validate_version(&Ecosystem::Npm, &node.spec.version)?;
if node.spec.ecosystem != Ecosystem::Npm
|| !node
.path
.ends_with(&format!("node_modules/{}", node.spec.name))
{
return Err("npm instance identity differs from its path".into());
}
if !lock
.libs
.iter()
.any(|lib| lib.spec == node.source.archive && lib.spec.ecosystem == Ecosystem::Npm)
{
return Err("npm instance source archive is not locked".into());
}
if node.source.prefix != "package/" {
validate_path(
node.source
.prefix
.strip_prefix("package/")
.and_then(|path| path.strip_suffix('/'))
.ok_or("invalid npm bundled source prefix")?,
)?;
} else if node.source.archive != node.spec {
return Err("npm root archive identity differs from its instance".into());
}
let mut unique = BTreeSet::new();
for edge in &node.edges {
if !unique.insert((&edge.name, &edge.kind)) {
return Err("duplicate npm dependency edge".into());
}
let range = edge
.requirement
.parse::<Range>()
.map_err(|error| error.to_string())?;
let actual = visible(
&nodes,
&path,
&edge.name,
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
);
match (&edge.target, &edge.omission) {
(Some(target), None)
if actual.is_some_and(|found| {
found.path == *target
&& Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&range))
}) =>
{
pending.push_back(target.clone());
}
(None, Some(reason))
if actual.is_none()
&& (edge.kind == Kind::Optional
&& *reason != Omission::OptionalPeerAbsent
|| edge.kind == Kind::OptionalPeer
&& matches!(
reason,
Omission::OptionalPeerAbsent | Omission::BuildFailure
))
&& (*reason != Omission::BuildFailure || env.build.is_some()) => {}
_ => {
return Err(
"npm edge differs from Node lookup or its required range".into()
);
}
}
}
}
if reached.len() != nodes.len() {
return Err("npm environment contains unreachable package instances".into());
}
}
for worker in &lock.workers {
if worker.ecosystem == "npm" {
environment(lock, &worker.libs)?;
}
}
Ok(())
}
use std::str::FromStr;
@@ -0,0 +1,436 @@
use super::*;
use crate::libs::registry::{MAX_PACKAGES, MAX_RESOLUTION_ATTEMPTS, RegistryResolver, SearchError};
#[derive(Clone)]
struct Request {
consumer: Option<String>,
edge: Edge,
}
#[derive(Clone, Default)]
struct State {
nodes: BTreeMap<String, Instance>,
pending: VecDeque<Request>,
}
struct Catalog<'a, 'r> {
resolver: &'a RegistryResolver<'r>,
runtime: &'a RegistryRuntime,
indexes: BTreeMap<String, Vec<String>>,
metadata: BTreeMap<(String, String), Value>,
archives: BTreeMap<LibSpec, (LockedLib, Archive)>,
attempts: usize,
archive_bytes: usize,
}
pub(crate) fn resolve(
resolver: &RegistryResolver<'_>,
requested: &[LibSpec],
runtime: &RegistryRuntime,
) -> Result<(Vec<LockedLib>, Environment), String> {
let roots = roots(requested);
let mut state = State::default();
for spec in &roots {
state.pending.push_back(Request {
consumer: None,
edge: Edge {
name: spec.name.clone(),
requirement: format!("={}", spec.version),
kind: Kind::Dependency,
target: None,
omission: None,
},
});
}
let mut catalog = Catalog {
resolver,
runtime,
indexes: BTreeMap::new(),
metadata: BTreeMap::new(),
archives: BTreeMap::new(),
attempts: 0,
archive_bytes: 0,
};
let mut state = catalog.search(state).map_err(SearchError::message)?;
let mut pending = roots
.iter()
.map(|root| child("", &root.name))
.collect::<VecDeque<_>>();
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
pending.extend(
state.nodes[&path]
.edges
.iter()
.filter_map(|edge| edge.target.clone()),
);
}
state.nodes.retain(|path, _| reached.contains(path));
let used = state
.nodes
.values()
.map(|node| node.source.archive.clone())
.collect::<BTreeSet<_>>();
let libraries = used
.into_iter()
.map(|spec| {
catalog
.archives
.remove(&spec)
.expect("resolved npm archive")
.0
})
.collect();
Ok((
libraries,
Environment {
build: None,
roots,
instances: state.nodes.into_values().collect(),
},
))
}
impl Catalog<'_, '_> {
fn archive(&mut self, spec: &LibSpec) -> Result<(), SearchError> {
if self.archives.contains_key(spec) {
return Ok(());
}
let document = self
.resolver
.registry_metadata(
Ecosystem::Npm,
&spec.name,
&spec.version,
&mut self.metadata,
)
.map_err(SearchError::Fatal)?;
let bytes = self
.resolver
.npm_archive(spec, &document)
.map_err(SearchError::Fatal)?;
let archive = Archive::parse(spec, &bytes).map_err(SearchError::Fatal)?;
self.archive_bytes = self
.archive_bytes
.checked_add(
archive
.files
.iter()
.map(|(_, bytes)| bytes.len())
.sum::<usize>(),
)
.ok_or_else(|| SearchError::Fatal("npm archive cache size overflow".into()))?;
if self.archive_bytes > 256 * 1024 * 1024 || self.archives.len() >= MAX_PACKAGES {
return Err(SearchError::Fatal(
"npm archive cache exceeds its 256 MiB/package budget".into(),
));
}
let lib = self
.resolver
.lock_archive(
spec,
Vec::new(),
self.runtime,
"tgz",
&bytes,
"npm-instances-v1",
)
.map_err(SearchError::Fatal)?;
self.archives.insert(spec.clone(), (lib, archive));
Ok(())
}
fn metadata(&self, node: &Instance) -> &Metadata {
&self.archives[&node.source.archive].1.packages[&node.source.prefix]
}
fn bind(
state: &mut State,
request: &Request,
target: Option<String>,
omission: Option<Omission>,
) {
if let Some(consumer) = &request.consumer {
let edge = state
.nodes
.get_mut(consumer)
.expect("pending npm consumer")
.edges
.iter_mut()
.find(|edge| edge.name == request.edge.name && edge.kind == request.edge.kind)
.expect("pending npm edge");
edge.target = target;
edge.omission = omission;
}
}
fn add(&self, state: &mut State, path: &str, spec: &LibSpec) -> Result<(), SearchError> {
for (prefix, metadata) in &self.archives[spec].1.packages {
let relative = prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
let location = if relative.is_empty() {
path.into()
} else {
format!("{path}/{relative}")
};
let instance = Instance {
path: location.clone(),
spec: metadata.spec.clone(),
source: Source {
archive: spec.clone(),
prefix: prefix.clone(),
},
bundled: if relative.is_empty() {
self.archives[spec].1.bundled()
} else {
BTreeMap::new()
},
edges: metadata.edges.clone(),
};
if state.nodes.insert(location.clone(), instance).is_some() {
return Err(SearchError::Conflict(
"npm package installation paths collide".into(),
));
}
for edge in &metadata.edges {
if edge.kind != Kind::OptionalPeer {
state.pending.push_back(Request {
consumer: Some(location.clone()),
edge: edge.clone(),
});
}
}
}
Ok(())
}
fn prune_bundled_orphans(&self, state: &mut State, root: &str) {
let mut pending = VecDeque::from([root.to_owned()]);
let mut reached = BTreeSet::new();
while let Some(path) = pending.pop_front() {
if !reached.insert(path.clone()) {
continue;
}
let Some(node) = state.nodes.get(&path) else {
continue;
};
for edge in &self.metadata(node).edges {
if let Some(found) = visible(
&state.nodes,
&path,
&edge.name,
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
) {
pending.push_back(found.path.clone());
}
}
}
let prefix = format!("{root}/");
state.nodes.retain(|path, node| {
node.source.prefix == "package/" || !path.starts_with(&prefix) || reached.contains(path)
});
}
fn search(&mut self, mut state: State) -> Result<State, SearchError> {
self.attempts += 1;
if self.attempts > MAX_RESOLUTION_ATTEMPTS
|| state.nodes.len() > MAX_PACKAGES
|| state.nodes.keys().any(|path| path.len() > 4096)
{
return Err(SearchError::Fatal(
"npm instance graph exceeds its resolution budget".into(),
));
}
while let Some(request) = state.pending.pop_front() {
if request
.consumer
.as_ref()
.is_some_and(|consumer| !state.nodes.contains_key(consumer))
{
continue;
}
let consumer = request.consumer.as_deref().unwrap_or("");
let peer = request.edge.kind == Kind::Peer;
let scope = if peer { parent(consumer) } else { consumer };
let location = child(scope, &request.edge.name);
let rule = request
.edge
.requirement
.parse::<Range>()
.map_err(|error| SearchError::Fatal(error.to_string()))?;
let existing = visible(&state.nodes, consumer, &request.edge.name, peer).cloned();
if let Some(found) = &existing {
let eligible = self
.metadata(found)
.eligible(self.runtime)
.map_err(SearchError::Fatal)?;
if !eligible
&& request.edge.kind == Kind::Optional
&& found.source.prefix != "package/"
{
let prefix = format!("{}/", found.path);
state
.nodes
.retain(|path, _| path != &found.path && !path.starts_with(&prefix));
let relative = found
.source
.prefix
.strip_prefix("package/")
.unwrap()
.trim_end_matches('/');
let owner = found
.path
.strip_suffix(relative)
.unwrap()
.trim_end_matches('/');
self.prune_bundled_orphans(&mut state, owner);
Self::bind(&mut state, &request, None, Some(Omission::Target));
continue;
}
// Ordinary dependencies get a private nested slot. Reuse an
// ancestor only to close a real package cycle, never a sibling's
// independently resolved peer context.
let bundled = state.nodes.get(consumer).is_some_and(|node| {
node.source.prefix != "package/" && node.source.archive == found.source.archive
});
let reusable = peer
|| bundled
|| found.path == location
|| consumer == found.path
|| consumer.starts_with(&format!("{}/", found.path));
if reusable
&& eligible
&& Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&rule))
{
Self::bind(&mut state, &request, Some(found.path.clone()), None);
continue;
}
if found.path == location {
return Err(SearchError::Conflict(format!(
"npm dependency/peer conflict for {} at {location}",
request.edge.name
)));
}
}
if let Some(node) = request
.consumer
.as_ref()
.and_then(|consumer| state.nodes.get(consumer))
&& node.source.prefix != "package/"
&& !peer
{
if request.edge.kind == Kind::Optional && existing.is_none() {
Self::bind(&mut state, &request, None, Some(Omission::Unavailable));
continue;
}
return Err(SearchError::Fatal(
"npm bundled dependency closure is incomplete".into(),
));
}
let versions = if let Some(versions) = self.indexes.get(&request.edge.name) {
versions.clone()
} else {
let versions = self
.resolver
.registry_versions(Ecosystem::Npm, &request.edge.name)
.map_err(SearchError::Fatal)?;
self.indexes
.insert(request.edge.name.clone(), versions.clone());
versions
};
let mut reason = Omission::Unavailable;
for version in versions.into_iter().filter(|version| {
Version::from_str(version).is_ok_and(|version| version.satisfies(&rule))
}) {
self.attempts += 1;
if self.attempts > MAX_RESOLUTION_ATTEMPTS {
return Err(SearchError::Fatal(
"npm resolution exceeds its candidate budget".into(),
));
}
let spec = LibSpec {
ecosystem: Ecosystem::Npm,
name: request.edge.name.clone(),
version,
};
self.archive(&spec)?;
if !self.archives[&spec].1.packages["package/"]
.eligible(self.runtime)
.map_err(SearchError::Fatal)?
{
reason = Omission::Target;
continue;
}
reason = Omission::Conflict;
let mut candidate = state.clone();
self.add(&mut candidate, &location, &spec)?;
Self::bind(&mut candidate, &request, Some(location.clone()), None);
match self.search(candidate) {
Ok(resolved) => return Ok(resolved),
Err(SearchError::Conflict(_)) => {}
Err(fatal) => return Err(fatal),
}
}
if request.edge.kind == Kind::Optional && existing.is_none() {
Self::bind(&mut state, &request, None, Some(reason));
continue;
}
return Err(SearchError::Conflict(format!(
"no npm version satisfies {} at {location}",
request.edge.name
)));
}
let edges = state
.nodes
.values()
.flat_map(|node| {
node.edges.iter().map(|edge| Request {
consumer: Some(node.path.clone()),
edge: edge.clone(),
})
})
.collect::<Vec<_>>();
for request in edges {
let found = visible(
&state.nodes,
request.consumer.as_deref().unwrap(),
&request.edge.name,
matches!(request.edge.kind, Kind::Peer | Kind::OptionalPeer),
);
if let Some(found) = found {
let range = request
.edge
.requirement
.parse::<Range>()
.map_err(|error| SearchError::Fatal(error.to_string()))?;
if !Version::from_str(&found.spec.version)
.is_ok_and(|version| version.satisfies(&range))
{
return Err(SearchError::Conflict(
"npm dependency or peer conflicts with final Node lookup".into(),
));
}
let target = found.path.clone();
Self::bind(&mut state, &request, Some(target), None);
} else if request.edge.kind == Kind::OptionalPeer {
Self::bind(
&mut state,
&request,
None,
Some(Omission::OptionalPeerAbsent),
);
} else if request.edge.kind != Kind::Optional || request.edge.omission.is_none() {
return Err(SearchError::Conflict(
"npm required dependency is absent from final Node lookup".into(),
));
}
}
Ok(state)
}
}
+524
View File
@@ -0,0 +1,524 @@
//! Authenticated Go checksum database receipts. Trust comes from the verifier
//! supplied by the caller, never from a key stored in a project's lock file.
//! Formats follow golang.org/x/mod/sumdb/{note,tlog,dirhash} and RFC 6962.
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Mutex, MutexGuard};
use base64::{Engine, engine::general_purpose::STANDARD};
use ring::signature::{ED25519, UnparsedPublicKey};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use super::{RegistryTransport, go_path, zip_entries};
mod merkle;
pub const OFFICIAL_VERIFIER: &str =
"sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8";
const MAX_NOTE_BYTES: usize = 16 * 1024;
const MAX_RECORDS: usize = 1024;
const MAX_EVIDENCE_BYTES: usize = 16 * 1024 * 1024;
type Hash = [u8; 32];
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct Tree {
size: u64,
hash: Hash,
}
fn encode_hash(hash: Hash) -> String {
STANDARD.encode(hash)
}
fn decode_hash(value: &str) -> Result<Hash, String> {
STANDARD
.decode(value)
.map_err(|_| "sumdb hash is not base64")?
.try_into()
.map_err(|_| "sumdb hash must contain 32 bytes".into())
}
/// Explicit verifier injection supports registry protocol tests and author
/// tooling. Released project commands always select `official()`.
#[derive(Clone)]
pub struct Verifier {
name: String,
key_hash: [u8; 4],
key: [u8; 32],
}
impl Verifier {
pub fn official() -> Self {
Self::new(OFFICIAL_VERIFIER).expect("fixed official sumdb verifier")
}
pub fn new(encoded: &str) -> Result<Self, String> {
// '+' also occurs inside base64; only the first two separators delimit.
let (name, rest) = encoded.split_once('+').ok_or("invalid sumdb verifier")?;
let (key_hash, encoded_key) = rest.split_once('+').ok_or("invalid sumdb verifier")?;
if name.is_empty() || name.chars().any(char::is_whitespace) || key_hash.len() != 8 {
return Err("invalid sumdb verifier identity".into());
}
let expected = u32::from_str_radix(key_hash, 16)
.map_err(|_| "invalid sumdb key hash")?
.to_be_bytes();
let key = STANDARD
.decode(encoded_key)
.map_err(|_| "invalid sumdb verifier encoding")?;
if key.len() != 33 || key[0] != 1 {
return Err("sumdb requires an Ed25519 verifier".into());
}
let mut hash = Sha256::new();
hash.update(name.as_bytes());
hash.update(b"\n");
hash.update(&key);
if hash.finalize()[..4] != expected {
return Err("sumdb verifier key hash mismatch".into());
}
Ok(Self {
name: name.into(),
key_hash: expected,
key: key[1..].try_into().expect("validated Ed25519 key length"),
})
}
fn checkpoint(&self, note: &str) -> Result<Tree, String> {
if note.len() > MAX_NOTE_BYTES || note.bytes().any(|byte| byte < 32 && byte != b'\n') {
return Err("invalid sumdb checkpoint note".into());
}
let (text, signatures) = note
.rsplit_once("\n\n")
.ok_or("sumdb checkpoint lacks signatures")?;
let message = format!("{text}\n");
if !signatures.ends_with('\n') {
return Err("unterminated sumdb signature".into());
}
let mut verified = false;
for (index, line) in signatures.lines().enumerate() {
if index >= 100 {
return Err("sumdb checkpoint has too many signatures".into());
}
let (name, encoded) = line
.strip_prefix("— ")
.and_then(|line| line.split_once(' '))
.ok_or("malformed sumdb signature")?;
let signature = STANDARD
.decode(encoded)
.map_err(|_| "invalid sumdb signature encoding")?;
if name.is_empty() || name.chars().any(char::is_whitespace) || signature.len() < 5 {
return Err("malformed sumdb signature identity".into());
}
if name == self.name && signature[..4] == self.key_hash {
UnparsedPublicKey::new(&ED25519, self.key)
.verify(message.as_bytes(), &signature[4..])
.map_err(|_| "sumdb checkpoint signature verification failed")?;
verified = true;
}
}
if !verified {
return Err("sumdb checkpoint has no trusted signature".into());
}
let mut lines = text.lines();
if lines.next() != Some("go.sum database tree") {
return Err("unsupported sumdb checkpoint format".into());
}
let size_text = lines.next().ok_or("sumdb checkpoint lacks tree size")?;
let size = decimal(size_text)?;
let hash = decode_hash(lines.next().ok_or("sumdb checkpoint lacks root hash")?)?;
let empty: Hash = Sha256::digest([]).into();
if size == 0 && hash != empty {
return Err("invalid sumdb empty root".into());
}
Ok(Tree { size, hash })
}
}
fn decimal(value: &str) -> Result<u64, String> {
let number = value
.parse::<u64>()
.map_err(|_| "invalid sumdb record/tree number")?;
if number > i64::MAX as u64 || value != number.to_string() {
return Err("noncanonical sumdb record/tree number".into());
}
Ok(number)
}
struct Lookup<'a> {
id: u64,
text: &'a str,
checkpoint: &'a str,
}
fn lookup(value: &str) -> Result<Lookup<'_>, String> {
if value.len() > MAX_NOTE_BYTES || value.bytes().any(|byte| byte < 32 && byte != b'\n') {
return Err("invalid sumdb lookup text".into());
}
let (id, rest) = value
.split_once('\n')
.ok_or("sumdb lookup lacks record number")?;
let end = rest
.find("\n\n")
.ok_or("sumdb lookup lacks record terminator")?;
Ok(Lookup {
id: decimal(id)?,
text: &rest[..end + 1],
checkpoint: &rest[end + 2..],
})
}
fn hashes(record: &str, module: &str, version: &str) -> Result<(String, String), String> {
let version = format!("v{version}");
let mod_version = format!("{version}/go.mod");
let mut values = BTreeMap::new();
for line in record.lines() {
let parts = line.split(' ').collect::<Vec<_>>();
if parts.len() != 3
|| parts[0] != module
|| (parts[1] != version && parts[1] != mod_version)
{
return Err("sumdb lookup record has a different module/version identity".into());
}
decode_hash(
parts[2]
.strip_prefix("h1:")
.ok_or("sumdb record requires h1 hashes")?,
)?;
if values.insert(parts[1], parts[2]).is_some() {
return Err("sumdb lookup repeats a module checksum".into());
}
}
Ok((
values
.get(mod_version.as_str())
.ok_or("sumdb lookup lacks go.mod hash")?
.to_string(),
values
.get(version.as_str())
.ok_or("sumdb lookup lacks module ZIP hash")?
.to_string(),
))
}
fn dirhash(files: &[(String, Vec<u8>)]) -> Result<String, String> {
let mut files = files.iter().collect::<Vec<_>>();
files.sort_by(|left, right| left.0.cmp(&right.0));
let mut seen = BTreeSet::new();
let mut hash = Sha256::new();
for (name, bytes) in files {
if name.contains('\n') || !seen.insert(name) {
return Err("invalid or duplicate Go dirhash filename".into());
}
hash.update(format!("{} {name}\n", crate::libs::sha256(bytes)));
}
Ok(format!("h1:{}", STANDARD.encode(hash.finalize())))
}
pub fn mod_hash(bytes: &[u8]) -> String {
// The Go protocol hashes a synthetic directory containing only go.mod.
dirhash(&[("go.mod".into(), bytes.to_vec())]).expect("fixed dirhash filename")
}
pub fn zip_hash(module: &str, version: &str, bytes: &[u8]) -> Result<String, String> {
let files = zip_entries(bytes)?;
let prefix = format!("{module}@v{version}/");
if files.iter().any(|(path, _)| !path.starts_with(&prefix)) {
return Err("Go module archive has a mismatched module root".into());
}
dirhash(&files)
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Record {
pub module: String,
pub version: String,
pub go_mod: String,
pub lookup: String,
pub inclusion: Vec<String>,
pub consistency: Vec<String>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Evidence {
pub previous: Option<String>,
pub checkpoint: String,
pub consistency: Vec<String>,
pub records: Vec<Record>,
}
#[derive(Debug)]
pub struct Verified {
pub checkpoint: String,
checksums: BTreeMap<(String, String), String>,
}
impl Verified {
pub fn verify_zip(&self, module: &str, version: &str, bytes: &[u8]) -> Result<(), String> {
let expected = self
.checksums
.get(&(module.into(), version.into()))
.ok_or("Go module lacks authenticated sumdb evidence")?;
if zip_hash(module, version, bytes)? != *expected {
return Err("Go module ZIP does not match authenticated sumdb h1".into());
}
Ok(())
}
pub fn contains(&self, module: &str, version: &str) -> bool {
self.checksums
.contains_key(&(module.into(), version.into()))
}
}
impl Evidence {
pub fn verify(&self, verifier: &Verifier) -> Result<Verified, String> {
if self.records.len() > MAX_RECORDS
|| serde_json::to_vec(self).map_err(|e| e.to_string())?.len() > MAX_EVIDENCE_BYTES
{
return Err("sumdb evidence exceeds its size budget".into());
}
let mut current = verifier.checkpoint(&self.checkpoint)?;
if let Some(previous) = &self.previous {
let older = verifier.checkpoint(previous)?;
if older.size > current.size {
return Err("sumdb checkpoint rollback rejected".into());
}
merkle::check_consistency(&self.consistency, older, current)?;
} else if !self.consistency.is_empty() {
return Err("sumdb first checkpoint has an unexpected consistency proof".into());
}
let mut latest = self.checkpoint.clone();
let mut checksums = BTreeMap::new();
for record in &self.records {
let entry = lookup(&record.lookup)?;
let tree = verifier.checkpoint(entry.checkpoint)?;
merkle::check_record(&record.inclusion, tree, entry.id, entry.text.as_bytes())?;
let (older, newer) = if tree.size < current.size {
(tree, current)
} else {
(current, tree)
};
merkle::check_consistency(&record.consistency, older, newer)?;
let (mod_h1, zip_h1) = hashes(entry.text, &record.module, &record.version)?;
if mod_hash(record.go_mod.as_bytes()) != mod_h1 {
return Err("Go go.mod does not match authenticated sumdb h1".into());
}
if checksums
.insert((record.module.clone(), record.version.clone()), zip_h1)
.is_some()
{
return Err("sumdb evidence repeats a module/version".into());
}
if tree.size > current.size {
current = tree;
latest = entry.checkpoint.into();
}
}
Ok(Verified {
checkpoint: latest,
checksums,
})
}
}
/// One explicit resolution sequence owns the checkpoint. The project lock is
/// its persistent anchor; deleting that lock intentionally starts first trust.
pub struct ChecksumDatabase {
verifier: Verifier,
latest: Mutex<Option<String>>,
}
impl ChecksumDatabase {
pub fn verifier(&self) -> &Verifier {
&self.verifier
}
pub fn new(verifier: Verifier, previous: Option<String>) -> Result<Self, String> {
if let Some(note) = &previous {
verifier.checkpoint(note)?;
}
Ok(Self {
verifier,
latest: Mutex::new(previous),
})
}
pub fn official(previous: Option<String>) -> Result<Self, String> {
Self::new(Verifier::official(), previous)
}
pub(super) fn begin<'a>(
&'a self,
transport: &'a dyn RegistryTransport,
) -> Result<Session<'a>, String> {
let guard = self
.latest
.lock()
.map_err(|_| "sumdb checkpoint state poisoned")?;
let checkpoint = String::from_utf8(transport.get_sumdb("/latest", MAX_NOTE_BYTES)?)
.map_err(|_| "sumdb checkpoint is not UTF-8")?;
let current = self.verifier.checkpoint(&checkpoint)?;
let mut tiles = merkle::Tiles::new(transport);
let consistency = if let Some(previous) = &*guard {
let older = self.verifier.checkpoint(previous)?;
if older.size > current.size {
return Err("sumdb checkpoint rollback rejected".into());
}
tiles.consistency(older, current)?
} else {
Vec::new()
};
let evidence = Evidence {
previous: guard.clone(),
checkpoint: checkpoint.clone(),
consistency,
records: Vec::new(),
};
Ok(Session {
verifier: &self.verifier,
transport,
guard,
tiles,
current,
latest: checkpoint,
evidence,
})
}
}
pub(super) struct Session<'a> {
verifier: &'a Verifier,
transport: &'a dyn RegistryTransport,
guard: MutexGuard<'a, Option<String>>,
tiles: merkle::Tiles<'a>,
current: Tree,
latest: String,
evidence: Evidence,
}
impl Session<'_> {
pub(super) fn verify_mod(
&mut self,
module: &str,
version: &str,
source: &[u8],
) -> Result<(), String> {
if let Some(record) = self
.evidence
.records
.iter()
.find(|record| record.module == module && record.version == version)
{
if record.go_mod.as_bytes() != source {
return Err("Go go.mod changed during resolution".into());
}
return Ok(());
}
if self.evidence.records.len() >= MAX_RECORDS {
return Err("sumdb resolution exceeds record limit".into());
}
let path = format!("/lookup/{}@v{}", go_path(module), go_path(version));
let response = String::from_utf8(self.transport.get_sumdb(&path, MAX_NOTE_BYTES)?)
.map_err(|_| "sumdb lookup is not UTF-8")?;
let entry = lookup(&response)?;
let tree = self.verifier.checkpoint(entry.checkpoint)?;
let inclusion = self.tiles.inclusion(tree.size, entry.id)?;
merkle::check_record(&inclusion, tree, entry.id, entry.text.as_bytes())?;
let (older, newer) = if tree.size < self.current.size {
(tree, self.current)
} else {
(self.current, tree)
};
let consistency = self.tiles.consistency(older, newer)?;
let (expected, _) = hashes(entry.text, module, version)?;
if mod_hash(source) != expected {
return Err("Go go.mod does not match authenticated sumdb h1".into());
}
if tree.size > self.current.size {
self.current = tree;
self.latest = entry.checkpoint.into();
}
let record = Record {
module: module.into(),
version: version.into(),
go_mod: String::from_utf8(source.to_vec()).map_err(|_| "Go go.mod is not UTF-8")?,
lookup: response,
inclusion,
consistency,
};
self.evidence.records.push(record);
if serde_json::to_vec(&self.evidence)
.map_err(|e| e.to_string())?
.len()
> MAX_EVIDENCE_BYTES
{
return Err("sumdb evidence exceeds its size budget".into());
}
Ok(())
}
pub(super) fn verify_zip(
&self,
module: &str,
version: &str,
bytes: &[u8],
) -> Result<(), String> {
let record = self
.evidence
.records
.iter()
.find(|record| record.module == module && record.version == version)
.ok_or("Go module ZIP has no authenticated metadata")?;
let (_, expected) = hashes(lookup(&record.lookup)?.text, module, version)?;
if zip_hash(module, version, bytes)? != expected {
return Err("Go module ZIP does not match authenticated sumdb h1".into());
}
Ok(())
}
pub(super) fn finish(mut self) -> Evidence {
*self.guard = Some(self.latest);
self.evidence
}
}
pub fn verify_chain(evidence: &[Evidence], verifier: &Verifier) -> Result<Verified, String> {
let mut latest = None;
let mut checksums = BTreeMap::new();
if evidence.len() > MAX_RECORDS {
return Err("sumdb resolution history exceeds its limit".into());
}
let mut bytes = 0usize;
for receipt in evidence {
bytes = bytes
.checked_add(
serde_json::to_vec(receipt)
.map_err(|error| error.to_string())?
.len(),
)
.ok_or("sumdb resolution history size overflow")?;
if bytes > MAX_EVIDENCE_BYTES {
return Err("sumdb resolution history exceeds its size budget".into());
}
if let Some(previous) = &latest
&& receipt.previous.as_ref() != Some(previous)
{
return Err("sumdb resolution checkpoints are not connected".into());
}
let verified = receipt.verify(verifier)?;
for (identity, hash) in verified.checksums {
if let Some(old) = checksums.insert(identity, hash.clone())
&& old != hash
{
return Err("sumdb resolutions disagree about module content".into());
}
}
latest = Some(verified.checkpoint);
}
Ok(Verified {
checkpoint: latest.unwrap_or_default(),
checksums,
})
}
@@ -0,0 +1,260 @@
//! RFC 6962 tree proofs and Go's height-eight tile addressing.
//! Protocol reference: golang.org/x/mod/sumdb/tlog (BSD-3-Clause).
use std::collections::BTreeMap;
use sha2::{Digest, Sha256};
use super::{Hash, Tree, decode_hash, encode_hash};
use crate::libs::RegistryTransport;
pub(super) fn leaf(bytes: &[u8]) -> Hash {
let mut hash = Sha256::new();
hash.update([0]);
hash.update(bytes);
hash.finalize().into()
}
fn node(left: Hash, right: Hash) -> Hash {
let mut hash = Sha256::new();
hash.update([1]);
hash.update(left);
hash.update(right);
hash.finalize().into()
}
fn split(size: u64) -> u64 {
1 << (63 - (size - 1).leading_zeros())
}
pub(super) fn check_record(
proof: &[String],
tree: Tree,
id: u64,
bytes: &[u8],
) -> Result<(), String> {
if id >= tree.size || proof.len() > 63 {
return Err("sumdb record proof has invalid bounds".into());
}
fn root(proof: &[String], size: u64, id: u64, value: Hash) -> Result<Hash, String> {
if size == 1 {
return if proof.is_empty() {
Ok(value)
} else {
Err("sumdb inclusion proof has surplus hashes".into())
};
}
let (last, rest) = proof
.split_last()
.ok_or("sumdb inclusion proof is incomplete")?;
let sibling = decode_hash(last)?;
let left = split(size);
if id < left {
Ok(node(root(rest, left, id, value)?, sibling))
} else {
Ok(node(sibling, root(rest, size - left, id - left, value)?))
}
}
if root(proof, tree.size, id, leaf(bytes))? != tree.hash {
return Err("sumdb inclusion proof does not match signed checkpoint".into());
}
Ok(())
}
pub(super) fn check_consistency(proof: &[String], older: Tree, newer: Tree) -> Result<(), String> {
if older.size > newer.size || proof.len() > 63 {
return Err("sumdb consistency proof has invalid bounds".into());
}
if older.size == 0 {
let empty: Hash = Sha256::digest([]).into();
return if proof.is_empty() && older.hash == empty {
Ok(())
} else {
Err("sumdb empty checkpoint is invalid".into())
};
}
fn roots(
proof: &[String],
lo: u64,
size: u64,
old_size: u64,
old_hash: Hash,
) -> Result<(Hash, Hash), String> {
if old_size == size {
return if lo == 0 && proof.is_empty() {
Ok((old_hash, old_hash))
} else if lo != 0 && proof.len() == 1 {
let hash = decode_hash(&proof[0])?;
Ok((hash, hash))
} else {
Err("sumdb consistency proof has surplus or missing hashes".into())
};
}
let (last, rest) = proof
.split_last()
.ok_or("sumdb consistency proof is incomplete")?;
let sibling = decode_hash(last)?;
let left = split(size);
if old_size <= left {
let (old, new) = roots(rest, lo, left, old_size, old_hash)?;
Ok((old, node(new, sibling)))
} else {
let (old, new) = roots(rest, lo + left, size - left, old_size - left, old_hash)?;
Ok((node(sibling, old), node(sibling, new)))
}
}
let (old, new) = roots(proof, 0, newer.size, older.size, older.hash)?;
if old != older.hash || new != newer.hash {
return Err("sumdb consistency proof detects a conflicting history".into());
}
Ok(())
}
/// Fetched tiles are only inputs to proofs; no byte is trusted merely because
/// it is cached. Persisted Evidence contains only completely verified proofs.
pub(super) struct Tiles<'a> {
transport: &'a dyn RegistryTransport,
cache: BTreeMap<String, Vec<u8>>,
}
impl<'a> Tiles<'a> {
pub(super) fn new(transport: &'a dyn RegistryTransport) -> Self {
Self {
transport,
cache: BTreeMap::new(),
}
}
fn subtree(&mut self, lo: u64, size: u64, tree_size: u64) -> Result<Hash, String> {
if size.is_power_of_two() {
let level = size.trailing_zeros();
let tile_level = level / 8;
let tile_scale = tile_level * 8;
let count = 1u64 << (level % 8);
let first = lo >> tile_scale;
let index = first / 256;
let offset = first % 256;
let width = (tree_size >> tile_scale)
.checked_sub(index * 256)
.ok_or("sumdb tile lies outside checkpoint")?
.min(256);
if offset + count > width {
return Err("sumdb tile does not cover requested subtree".into());
}
let mut chunks = vec![format!("{:03}", index % 1000)];
let mut prefix = index / 1000;
while prefix > 0 {
chunks.push(format!("x{:03}", prefix % 1000));
prefix /= 1000;
}
chunks.reverse();
let full = format!("/tile/8/{tile_level}/{}", chunks.join("/"));
let path = if width == 256 {
full.clone()
} else {
format!("{full}.p/{width}")
};
if !self.cache.contains_key(&path) {
if self.cache.len() >= 4096 {
return Err("sumdb tile cache exceeds its 32 MiB budget".into());
}
let bytes = match self.transport.get_sumdb(&path, 8192) {
Ok(bytes) => bytes,
Err(_) if width < 256 => {
let mut bytes = self.transport.get_sumdb(&full, 8192)?;
if bytes.len() != 8192 {
return Err("sumdb full tile has wrong length".into());
}
bytes.truncate(width as usize * 32);
bytes
}
Err(error) => return Err(error),
};
if bytes.len() != width as usize * 32 {
return Err("sumdb tile has wrong length".into());
}
self.cache.insert(path.clone(), bytes);
}
fn fold(bytes: &[u8]) -> Hash {
if bytes.len() == 32 {
return bytes.try_into().expect("one tile hash");
}
let (left, right) = bytes.split_at(bytes.len() / 2);
node(fold(left), fold(right))
}
return Ok(fold(
&self.cache[&path][offset as usize * 32..(offset + count) as usize * 32],
));
}
let left = split(size);
Ok(node(
self.subtree(lo, left, tree_size)?,
self.subtree(lo + left, size - left, tree_size)?,
))
}
pub(super) fn inclusion(&mut self, size: u64, id: u64) -> Result<Vec<String>, String> {
fn walk(
tiles: &mut Tiles<'_>,
lo: u64,
size: u64,
id: u64,
total: u64,
proof: &mut Vec<String>,
) -> Result<(), String> {
if size == 1 {
return Ok(());
}
let left = split(size);
let sibling = if id < lo + left {
walk(tiles, lo, left, id, total, proof)?;
tiles.subtree(lo + left, size - left, total)?
} else {
walk(tiles, lo + left, size - left, id, total, proof)?;
tiles.subtree(lo, left, total)?
};
proof.push(encode_hash(sibling));
Ok(())
}
if id >= size {
return Err("sumdb record lies outside signed checkpoint".into());
}
let mut proof = Vec::new();
walk(self, 0, size, id, size, &mut proof)?;
Ok(proof)
}
pub(super) fn consistency(&mut self, older: Tree, newer: Tree) -> Result<Vec<String>, String> {
fn walk(
tiles: &mut Tiles<'_>,
lo: u64,
size: u64,
old_size: u64,
total: u64,
proof: &mut Vec<String>,
) -> Result<(), String> {
if old_size == size {
if lo != 0 {
proof.push(encode_hash(tiles.subtree(lo, size, total)?));
}
return Ok(());
}
let left = split(size);
let sibling = if old_size <= left {
walk(tiles, lo, left, old_size, total, proof)?;
tiles.subtree(lo + left, size - left, total)?
} else {
walk(tiles, lo + left, size - left, old_size - left, total, proof)?;
tiles.subtree(lo, left, total)?
};
proof.push(encode_hash(sibling));
Ok(())
}
let mut proof = Vec::new();
if older.size != 0 {
walk(self, 0, newer.size, older.size, newer.size, &mut proof)?;
}
check_consistency(&proof, older, newer)?;
Ok(proof)
}
}