feat: publish Dever core with project scaffolding and versioned AI skills
This commit is contained in:
1 parent
70fa73cdc2
commit
ac458ffa7f
960 files changed
+221319
-1163
No files matched your search
@@ -0,0 +1,526 @@
|
||||
//! Explicit Lib preparation: shared owned staging and bounded tool execution.
|
||||
|
||||
mod npm;
|
||||
pub mod pack;
|
||||
pub(crate) mod process;
|
||||
mod python;
|
||||
|
||||
use super::{Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryRuntime, RuntimePack};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::cell::RefCell;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
fn require_execution_target(target: &str, operation: &str) -> Result<(), String> {
|
||||
if target != crate::toolchain::BuildTarget::host()?.platform() {
|
||||
return Err(format!(
|
||||
"{operation} for {target} requires prepared target output and a complete build receipt; prepare the signed target inputs in a matching isolated execution environment before the offline build"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Production supplies only authenticated release resources. Explicit author
|
||||
/// tests inject the same immutable inputs without creating a host-tool fallback.
|
||||
pub trait Inputs {
|
||||
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String>;
|
||||
fn tools(&self, ecosystem: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String>;
|
||||
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String>;
|
||||
/// Author tools may display bounded hook output without retaining temporary
|
||||
/// staging trees or putting diagnostics in reproducible build identities.
|
||||
fn diagnostic(&self, _output: &[u8]) {}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PythonSystem {
|
||||
pub requires: Vec<String>,
|
||||
pub backend: String,
|
||||
pub backend_path: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NativeRejection {
|
||||
Format,
|
||||
Target,
|
||||
Closure,
|
||||
Unloadable,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Receipt {
|
||||
pub ecosystem: Ecosystem,
|
||||
pub roots: Vec<LibSpec>,
|
||||
pub sources: Vec<LockedArtifact>,
|
||||
pub graph: Option<super::npm::Environment>,
|
||||
pub runtime: RuntimePack,
|
||||
pub auxiliary_runtime: Option<RuntimePack>,
|
||||
pub native_entries: BTreeSet<String>,
|
||||
pub native_rejections: BTreeMap<String, NativeRejection>,
|
||||
pub python_markers: Option<pep508_rs::MarkerEnvironment>,
|
||||
pub tools: pack::Descriptor,
|
||||
pub frontend: String,
|
||||
pub python: Option<PythonSystem>,
|
||||
pub dynamic_requires: Vec<String>,
|
||||
pub dependencies: Box<LockFile>,
|
||||
pub config_settings: BTreeMap<String, serde_json::Value>,
|
||||
pub output: LockedArtifact,
|
||||
}
|
||||
|
||||
impl Receipt {
|
||||
pub fn identity(&self) -> Result<String, String> {
|
||||
serde_json::to_vec(self)
|
||||
.map(|bytes| super::sha256(&bytes))
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct Product {
|
||||
pub bytes: Vec<u8>,
|
||||
pub filename: String,
|
||||
pub receipt: Receipt,
|
||||
}
|
||||
|
||||
pub struct Session<'a> {
|
||||
inputs: &'a dyn Inputs,
|
||||
active: RefCell<BTreeSet<String>>,
|
||||
products: RefCell<BTreeMap<String, Product>>,
|
||||
}
|
||||
|
||||
impl<'a> Session<'a> {
|
||||
pub fn new(inputs: &'a dyn Inputs) -> Self {
|
||||
Self {
|
||||
inputs,
|
||||
active: RefCell::new(BTreeSet::new()),
|
||||
products: RefCell::new(BTreeMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn receipts(&self, identities: &BTreeSet<String>) -> Result<Vec<Receipt>, String> {
|
||||
let receipts = self
|
||||
.products
|
||||
.borrow()
|
||||
.values()
|
||||
.map(|product| Ok((product.receipt.identity()?, product.receipt.clone())))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
identities
|
||||
.iter()
|
||||
.map(|identity| {
|
||||
receipts
|
||||
.get(identity)
|
||||
.cloned()
|
||||
.ok_or("prepared build receipt is missing".into())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn remember(&self, key: String, product: Product) -> Result<(), String> {
|
||||
let total = self
|
||||
.products
|
||||
.borrow()
|
||||
.values()
|
||||
.try_fold(product.bytes.len(), |total, cached| {
|
||||
total.checked_add(cached.bytes.len())
|
||||
})
|
||||
.ok_or("build candidate cache byte overflow")?;
|
||||
if total > crate::workers::MAX_TREE_BYTES {
|
||||
return Err("build candidate cache exceeds byte budget".into());
|
||||
}
|
||||
self.products.borrow_mut().insert(key, product);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn normalize(receipts: &mut Vec<Receipt>) -> Result<(), String> {
|
||||
let mut indexed = BTreeMap::new();
|
||||
for receipt in receipts.drain(..) {
|
||||
indexed.insert(receipt.identity()?, receipt);
|
||||
}
|
||||
*receipts = indexed.into_values().collect();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn npm_outputs(lock: &LockFile) -> impl Iterator<Item = &Receipt> {
|
||||
lock.builds
|
||||
.iter()
|
||||
.filter(|receipt| receipt.ecosystem == Ecosystem::Npm)
|
||||
}
|
||||
|
||||
pub(crate) fn npm_receipt<'a>(
|
||||
lock: &'a LockFile,
|
||||
environment: &super::npm::Environment,
|
||||
) -> Result<Option<&'a Receipt>, String> {
|
||||
let Some(identity) = &environment.build else {
|
||||
return Ok(None);
|
||||
};
|
||||
lock.builds
|
||||
.iter()
|
||||
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
|
||||
.map(Some)
|
||||
.ok_or("npm installation receipt is missing".into())
|
||||
}
|
||||
|
||||
/// Offline consumption never invokes a frontend. Removing roots projects the
|
||||
/// already built graph and output; it does not rerun lifecycle side effects.
|
||||
pub fn npm_installation(
|
||||
lock: &LockFile,
|
||||
environment: &super::npm::Environment,
|
||||
bytes: &[u8],
|
||||
) -> Result<Vec<(String, Vec<u8>, bool)>, String> {
|
||||
let receipt = npm_receipt(lock, environment)?.ok_or("npm installation has no build receipt")?;
|
||||
if bytes.len() as u64 != receipt.output.bytes || super::sha256(bytes) != receipt.output.sha256 {
|
||||
return Err("npm built installation differs from its locked artifact".into());
|
||||
}
|
||||
let graph = receipt
|
||||
.graph
|
||||
.as_ref()
|
||||
.ok_or("npm receipt lacks its exact graph")?;
|
||||
let files = super::archive_files("tgz", bytes)?;
|
||||
super::npm::validate_built_files(graph, &files)?;
|
||||
let candidates = files
|
||||
.iter()
|
||||
.filter(|(path, _)| path.ends_with(".node"))
|
||||
.map(|(path, _)| path.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let accounted = receipt
|
||||
.native_entries
|
||||
.iter()
|
||||
.chain(receipt.native_rejections.keys())
|
||||
.cloned()
|
||||
.collect::<BTreeSet<_>>();
|
||||
if candidates != accounted
|
||||
|| receipt
|
||||
.native_rejections
|
||||
.keys()
|
||||
.any(|path| receipt.native_entries.contains(path))
|
||||
{
|
||||
return Err("npm native admission does not cover its exact addon files".into());
|
||||
}
|
||||
for path in &receipt.native_entries {
|
||||
if !files.iter().any(|(name, bytes)| {
|
||||
name == path && path.ends_with(".node") && bytes.starts_with(b"\x7fELF")
|
||||
}) {
|
||||
return Err("npm locked native entry point is absent from output".into());
|
||||
}
|
||||
}
|
||||
let executable = super::registry::archive_executables("tgz", bytes)?;
|
||||
let retained = environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| node.path.as_str())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let mut output = Vec::new();
|
||||
for (path, bytes) in files {
|
||||
if path.ends_with(".node") && !receipt.native_entries.contains(&path) {
|
||||
continue;
|
||||
}
|
||||
if retained.contains(super::npm::installed_owner(&path)?.as_str()) {
|
||||
let mode = executable.contains(&path);
|
||||
output.push((path, bytes, mode));
|
||||
}
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
doctor_at(lock, 0)
|
||||
}
|
||||
|
||||
fn doctor_at(lock: &LockFile, depth: usize) -> Result<(), String> {
|
||||
if depth > 16 {
|
||||
return Err("build dependency depth exceeds limit".into());
|
||||
}
|
||||
let records = lock
|
||||
.builds
|
||||
.iter()
|
||||
.map(|receipt| Ok((receipt.identity()?, receipt)))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
if records.len() != lock.builds.len() {
|
||||
return Err("duplicate build receipt".into());
|
||||
}
|
||||
let mut used = BTreeSet::new();
|
||||
for lib in &lock.libs {
|
||||
if let Some(identity) = &lib.build {
|
||||
let receipt = records
|
||||
.get(identity)
|
||||
.ok_or("Python Lib has no exact build receipt")?;
|
||||
if lib.spec.ecosystem != Ecosystem::Pip
|
||||
|| receipt.ecosystem != Ecosystem::Pip
|
||||
|| receipt.runtime != lib.runtime
|
||||
|| !lib.artifacts.contains(&receipt.output)
|
||||
|| receipt.roots.len() != 1
|
||||
|| receipt.roots[0].distribution_name() != lib.spec.distribution_name()
|
||||
|| receipt.roots[0].version != lib.spec.version
|
||||
{
|
||||
return Err(
|
||||
"Python Lib build receipt differs from locked artifact identity".into(),
|
||||
);
|
||||
}
|
||||
used.insert(identity.clone());
|
||||
}
|
||||
}
|
||||
for environment in &lock.npm {
|
||||
if let Some(identity) = &environment.build {
|
||||
let receipt = records
|
||||
.get(identity)
|
||||
.ok_or("npm environment has no exact build receipt")?;
|
||||
let graph = receipt
|
||||
.graph
|
||||
.as_ref()
|
||||
.ok_or("npm build receipt lacks its graph")?;
|
||||
if receipt.ecosystem != Ecosystem::Npm
|
||||
|| receipt.roots != graph.roots
|
||||
|| graph.build.is_some()
|
||||
|| !environment
|
||||
.roots
|
||||
.iter()
|
||||
.all(|root| graph.roots.contains(root))
|
||||
|| !environment
|
||||
.instances
|
||||
.iter()
|
||||
.all(|node| graph.instances.contains(node))
|
||||
|| !environment.instances.iter().all(|node| {
|
||||
lock.libs.iter().any(|lib| {
|
||||
lib.spec == node.source.archive
|
||||
&& lib.runtime == receipt.runtime
|
||||
&& lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.all(|artifact| receipt.sources.contains(artifact))
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm installation build receipt differs from locked graph".into());
|
||||
}
|
||||
used.insert(identity.clone());
|
||||
}
|
||||
}
|
||||
if used.len() != records.len() {
|
||||
return Err("lock contains an unreferenced build receipt".into());
|
||||
}
|
||||
for receipt in records.values() {
|
||||
super::validate_runtime(&receipt.runtime)?;
|
||||
if receipt.tools.runtime_sha256 != receipt.runtime.sha256
|
||||
|| receipt.tools.ecosystem != receipt.ecosystem
|
||||
|| receipt.sources.is_empty()
|
||||
|| receipt.output.target != receipt.tools.target
|
||||
|| receipt.roots.is_empty()
|
||||
|| receipt.frontend != frontend(&receipt.ecosystem)?
|
||||
{
|
||||
return Err("build receipt has inconsistent tools, frontend, or target".into());
|
||||
}
|
||||
for artifact in receipt.sources.iter().chain([&receipt.output]) {
|
||||
super::validate_artifact(artifact)?;
|
||||
if artifact.target != receipt.tools.target {
|
||||
return Err("build artifact target differs from its tools".into());
|
||||
}
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Pip
|
||||
&& (receipt.python.is_none()
|
||||
|| receipt.graph.is_some()
|
||||
|| receipt.auxiliary_runtime.is_some())
|
||||
{
|
||||
return Err("Python build receipt lacks its backend configuration".into());
|
||||
}
|
||||
if receipt.ecosystem == Ecosystem::Pip
|
||||
&& (!receipt.native_entries.is_empty() || !receipt.native_rejections.is_empty())
|
||||
{
|
||||
return Err("Python receipt cannot declare Node native entry points".into());
|
||||
}
|
||||
for path in receipt
|
||||
.native_entries
|
||||
.iter()
|
||||
.chain(receipt.native_rejections.keys())
|
||||
{
|
||||
crate::workers::valid_path(path)?;
|
||||
}
|
||||
let policy = RegistryRuntime {
|
||||
pack: receipt.runtime.clone(),
|
||||
target: receipt.tools.target.clone(),
|
||||
python_markers: receipt.python_markers.clone(),
|
||||
python_wheel_tags: vec![],
|
||||
npm_libc: None,
|
||||
};
|
||||
let auxiliary = receipt
|
||||
.auxiliary_runtime
|
||||
.as_ref()
|
||||
.map(|runtime| RegistryRuntime {
|
||||
pack: runtime.clone(),
|
||||
target: receipt.tools.target.clone(),
|
||||
python_markers: None,
|
||||
python_wheel_tags: vec![],
|
||||
npm_libc: None,
|
||||
});
|
||||
if let Some(runtime) = &receipt.auxiliary_runtime {
|
||||
super::validate_runtime(runtime)?;
|
||||
}
|
||||
receipt.tools.validate(&policy, auxiliary.as_ref())?;
|
||||
if receipt.ecosystem == Ecosystem::Npm
|
||||
&& (receipt.python.is_some()
|
||||
|| receipt.python_markers.is_some()
|
||||
|| !receipt.dynamic_requires.is_empty()
|
||||
|| !receipt.dependencies.libs.is_empty()
|
||||
|| receipt.graph.is_none())
|
||||
{
|
||||
return Err("npm build receipt contains Python backend state".into());
|
||||
}
|
||||
if !receipt.config_settings.is_empty() {
|
||||
return Err("unsupported source-build configuration settings".into());
|
||||
}
|
||||
if let Some(system) = &receipt.python {
|
||||
validate_requirements(receipt, system)?;
|
||||
}
|
||||
if !receipt.dependencies.workers.is_empty()
|
||||
|| !receipt.dependencies.packages.is_empty()
|
||||
|| receipt
|
||||
.dependencies
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.ecosystem != Ecosystem::Pip)
|
||||
{
|
||||
return Err("build dependency closure is not a separate Python environment".into());
|
||||
}
|
||||
doctor_at(&receipt.dependencies, depth + 1)?;
|
||||
// Full graph checks belong to the ordinary lock owner; recursion is
|
||||
// bounded above before invoking that owner on a dependency lock.
|
||||
let mut dependencies = (*receipt.dependencies).clone();
|
||||
dependencies.builds.clear();
|
||||
for lib in &mut dependencies.libs {
|
||||
lib.build = None;
|
||||
}
|
||||
super::doctor(&dependencies)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_requirements(receipt: &Receipt, system: &PythonSystem) -> Result<(), String> {
|
||||
let identifier = |path: &str| {
|
||||
!path.is_empty()
|
||||
&& path.split('.').all(|part| {
|
||||
let mut bytes = part.bytes();
|
||||
bytes
|
||||
.next()
|
||||
.is_some_and(|byte| byte.is_ascii_alphabetic() || byte == b'_')
|
||||
&& bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
|
||||
})
|
||||
};
|
||||
let (module, function) = system
|
||||
.backend
|
||||
.split_once(':')
|
||||
.map(|(module, function)| (module, Some(function)))
|
||||
.unwrap_or((&system.backend, None));
|
||||
if !identifier(module)
|
||||
|| function.is_some_and(|function| !identifier(function))
|
||||
|| system.requires.len() > 1024
|
||||
|| receipt.dynamic_requires.len() > 1024
|
||||
|| system.backend_path.len() > 64
|
||||
{
|
||||
return Err("build receipt has an invalid backend or requirement set".into());
|
||||
}
|
||||
for path in &system.backend_path {
|
||||
if path.is_empty() || path.starts_with('/') || path.contains(['\\', '\0']) {
|
||||
return Err("invalid backend-path in receipt".into());
|
||||
}
|
||||
let mut depth = 0usize;
|
||||
for part in path.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
depth = depth.checked_sub(1).ok_or("backend-path escapes source")?;
|
||||
}
|
||||
_ => depth += 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
let markers = receipt
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build receipt lacks marker policy")?;
|
||||
let requirements = system
|
||||
.requires
|
||||
.iter()
|
||||
.chain(&receipt.dynamic_requires)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let document = serde_json::json!({"info":{"requires_dist":requirements}});
|
||||
let expected = super::python_dependencies(&document, markers, &[])?;
|
||||
let libraries = receipt
|
||||
.dependencies
|
||||
.libs
|
||||
.iter()
|
||||
.map(|lib| (lib.spec.name.as_str(), lib))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut remaining = Vec::new();
|
||||
for (name, rule) in expected {
|
||||
let lib = libraries
|
||||
.get(name.as_str())
|
||||
.ok_or("Python build receipt omits a required dependency")?;
|
||||
let version = lib
|
||||
.spec
|
||||
.version
|
||||
.parse::<pep440_rs::Version>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
if !rule.contains(&version) {
|
||||
return Err("Python build dependency does not satisfy recorded requirements".into());
|
||||
}
|
||||
remaining.push(lib.spec.name.as_str());
|
||||
}
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(name) = remaining.pop() {
|
||||
if !reached.insert(name) {
|
||||
continue;
|
||||
}
|
||||
let lib = libraries
|
||||
.get(name)
|
||||
.ok_or("build dependency closure is incomplete")?;
|
||||
if lib.runtime != receipt.runtime
|
||||
|| lib
|
||||
.artifacts
|
||||
.iter()
|
||||
.any(|artifact| artifact.target != receipt.tools.target)
|
||||
{
|
||||
return Err("build dependency runtime or target differs from its receipt".into());
|
||||
}
|
||||
remaining.extend(lib.dependencies.iter().map(|edge| edge.spec.name.as_str()));
|
||||
}
|
||||
if reached.len() != libraries.len() {
|
||||
return Err("build dependency lock contains unrelated libraries".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Offline consumers bind recorded marker evaluation to the immutable runtime
|
||||
/// manifest they have just verified, never to a lock's self-declared policy.
|
||||
pub fn validate_python_policy(
|
||||
lock: &LockFile,
|
||||
lib: &super::LockedLib,
|
||||
markers: &pep508_rs::MarkerEnvironment,
|
||||
) -> Result<(), String> {
|
||||
if let Some(identity) = &lib.build {
|
||||
let receipt = lock
|
||||
.builds
|
||||
.iter()
|
||||
.find(|receipt| receipt.identity().as_ref() == Ok(identity))
|
||||
.ok_or("Python build receipt missing")?;
|
||||
if receipt.python_markers.as_ref() != Some(markers) {
|
||||
return Err("Python build marker policy differs from signed runtime".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn frontend(ecosystem: &Ecosystem) -> Result<String, String> {
|
||||
match ecosystem {
|
||||
Ecosystem::Pip => Ok(super::sha256(include_bytes!("build/python.py"))),
|
||||
Ecosystem::Npm => Ok(super::sha256(
|
||||
&[
|
||||
include_bytes!("build/npm.js").as_slice(),
|
||||
include_bytes!("build/npm/probe.cjs").as_slice(),
|
||||
]
|
||||
.concat(),
|
||||
)),
|
||||
_ => Err("source-build frontend is not available for this ecosystem".into()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Fixed registry-package lifecycle frontend. Runs only in the build sandbox.
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const runScript = require('/worker/npm/frontend/node_modules/npm/node_modules/@npmcli/run-script');
|
||||
const binLinks = require('/worker/npm/frontend/node_modules/npm/node_modules/bin-links');
|
||||
|
||||
async function main() {
|
||||
const request = JSON.parse(fs.readFileSync('/worker/request.json', 'utf8'));
|
||||
const root = '/data/build/output/install';
|
||||
fs.mkdirSync('/data/build/output');
|
||||
fs.cpSync('/worker/install', root, { recursive: true, dereference: false, errorOnExist: true });
|
||||
fs.mkdirSync('/data/build/tmp');
|
||||
fs.mkdirSync('/data/build/home');
|
||||
Object.assign(process.env, {
|
||||
PATH: '/worker/runtime/bin:/usr/bin',
|
||||
CC: '/usr/bin/cc', CXX: '/usr/bin/c++', AR: '/usr/bin/ar',
|
||||
PYTHON: request.python,
|
||||
npm_config_python: request.python,
|
||||
npm_config_nodedir: '/worker/npm',
|
||||
npm_config_node_gyp: '/worker/npm/frontend/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js',
|
||||
npm_config_build_from_source: 'true',
|
||||
npm_config_cache: '/data/build/npm-cache',
|
||||
TMPDIR: '/data/build/tmp', SOURCE_DATE_EPOCH: '0',
|
||||
HOME: '/data/build/home',
|
||||
});
|
||||
const nodes = new Map(request.instances.map(node => [node.path, node]));
|
||||
for (const node of request.instances) {
|
||||
const directory = path.join(root, node.path);
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(directory, 'package.json'), 'utf8'));
|
||||
await binLinks({ path: directory, pkg, global: false, top: false, force: false });
|
||||
}
|
||||
const visited = new Set();
|
||||
const required = new Set(request.required);
|
||||
const failed = new Set();
|
||||
const nativeEntries = new Set();
|
||||
async function install(name) {
|
||||
if (visited.has(name)) return;
|
||||
visited.add(name);
|
||||
const node = nodes.get(name);
|
||||
try {
|
||||
for (const edge of node.edges) {
|
||||
if (edge.target && nodes.has(edge.target)) {
|
||||
await install(edge.target);
|
||||
if (failed.has(edge.target) && ['dependency', 'peer'].includes(edge.kind)) {
|
||||
throw new Error('required lifecycle dependency failed: ' + edge.target);
|
||||
}
|
||||
}
|
||||
}
|
||||
const directory = path.join(root, name);
|
||||
for (const event of ['preinstall', 'install', 'postinstall']) {
|
||||
await runScript({ event, path: directory, stdio: 'inherit', scriptShell: '/bin/sh' });
|
||||
}
|
||||
const native = path.join(directory, 'build', 'Release');
|
||||
if (fs.existsSync(native)) {
|
||||
for (const file of fs.readdirSync(native)) {
|
||||
if (file.endsWith('.node')) {
|
||||
const addon = path.join(native, file);
|
||||
require(addon);
|
||||
nativeEntries.add(path.relative(root, addon));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (required.has(name)) throw error;
|
||||
process.stderr.write('optional npm lifecycle failed: ' + name + ': ' + String(error.message || error) + '\n');
|
||||
failed.add(name);
|
||||
}
|
||||
}
|
||||
for (const node of request.instances) await install(node.path);
|
||||
let changed = true;
|
||||
while (changed) {
|
||||
changed = false;
|
||||
for (const node of request.instances) {
|
||||
if (!failed.has(node.path) && node.edges.some(edge => ['dependency', 'peer'].includes(edge.kind) && failed.has(edge.target))) {
|
||||
if (required.has(node.path)) throw new Error('required lifecycle dependency failed: ' + node.path);
|
||||
failed.add(node.path);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
const retained = new Set();
|
||||
function retain(name) {
|
||||
if (retained.has(name) || failed.has(name)) return;
|
||||
retained.add(name);
|
||||
for (const edge of nodes.get(name).edges) {
|
||||
if (edge.target) retain(edge.target);
|
||||
}
|
||||
}
|
||||
for (const node of request.roots) retain(node);
|
||||
for (const node of request.instances) {
|
||||
if (!retained.has(node.path)) fs.rmSync(path.join(root, node.path), { recursive: true, force: true });
|
||||
}
|
||||
// Bin links are a build-time PATH input. The immutable Worker tree stores
|
||||
// package files only; it never persists symlinks created by installer hooks.
|
||||
for (const node of request.instances) {
|
||||
fs.rmSync(path.join(root, node.path, 'node_modules', '.bin'), { recursive: true, force: true });
|
||||
fs.rmSync(path.join(root, node.path, 'build', 'node_gyp_bins'), { recursive: true, force: true });
|
||||
}
|
||||
fs.rmSync(path.join(root, 'node_modules', '.bin'), { recursive: true, force: true });
|
||||
fs.writeFileSync('/data/build/output/result.json', JSON.stringify({ failed: [...failed].sort(), native: [...nativeEntries].filter(file => retained.has(file.split('/build/Release/')[0])).sort() }));
|
||||
}
|
||||
|
||||
main().catch(error => { process.stderr.write(String(error.stack || error) + '\n'); process.exitCode = 1; });
|
||||
@@ -0,0 +1,230 @@
|
||||
//! Registry lifecycle scripts use the signed npm frontend and exact instance graph.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
use std::io::Cursor;
|
||||
|
||||
use super::{Product, Receipt, Session};
|
||||
use crate::libs::{
|
||||
Ecosystem, LockFile, LockedArtifact, LockedLib, RegistryResolver, RegistryRuntime, npm, sha256,
|
||||
};
|
||||
|
||||
mod environment;
|
||||
use environment::{Environment, Output};
|
||||
|
||||
impl Session<'_> {
|
||||
pub(crate) fn npm(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
libraries: &mut Vec<LockedLib>,
|
||||
graph: &mut npm::Environment,
|
||||
runtime: &RegistryRuntime,
|
||||
) -> Result<(), String> {
|
||||
let mut lock = LockFile::new(libraries.clone())?;
|
||||
lock.npm.push(graph.clone());
|
||||
let archives = libraries
|
||||
.iter()
|
||||
.map(|lib| {
|
||||
let artifact = lib.artifacts.first().ok_or("npm source archive missing")?;
|
||||
Ok((
|
||||
lib.spec.clone(),
|
||||
resolver.store.verify_exact(
|
||||
&artifact.sha256,
|
||||
artifact.bytes,
|
||||
&artifact.target,
|
||||
)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let files = npm::build_files(graph, &lock, |lib| {
|
||||
archives
|
||||
.get(&lib.spec)
|
||||
.map(Vec::as_slice)
|
||||
.ok_or("npm source archive missing".into())
|
||||
})?;
|
||||
if !npm::requires_build(&files)? {
|
||||
return Ok(());
|
||||
}
|
||||
let mut sources = libraries
|
||||
.iter()
|
||||
.flat_map(|lib| lib.artifacts.clone())
|
||||
.collect::<Vec<_>>();
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
let key = format!(
|
||||
"npm:{}:{}:{}",
|
||||
sha256(&serde_json::to_vec(graph).map_err(|error| error.to_string())?),
|
||||
runtime.pack.sha256,
|
||||
sha256(&serde_json::to_vec(&sources).map_err(|error| error.to_string())?)
|
||||
);
|
||||
if let Some(product) = self.products.borrow().get(&key) {
|
||||
*graph = product
|
||||
.receipt
|
||||
.graph
|
||||
.clone()
|
||||
.ok_or("cached npm graph missing")?;
|
||||
graph.build = Some(product.receipt.identity()?);
|
||||
retain_archives(libraries, graph);
|
||||
return Ok(());
|
||||
}
|
||||
let executables = npm::build_executables(graph, &archives)?;
|
||||
let environment = Environment::new(self.inputs, runtime)?;
|
||||
let output = environment.run(self.inputs, graph, &files, &executables)?;
|
||||
prune_failed(graph, &output.failed)?;
|
||||
retain_archives(libraries, graph);
|
||||
npm::validate_built_files(graph, &output.files)?;
|
||||
let bytes = archive(&output)?;
|
||||
let digest = resolver.store.publish(&bytes, &runtime.target)?;
|
||||
if digest != sha256(&bytes) {
|
||||
return Err("built npm cache identity mismatch".into());
|
||||
}
|
||||
let output_artifact = LockedArtifact {
|
||||
target: runtime.target.clone(),
|
||||
path: format!("build/npm/{digest}/installation.tgz"),
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: digest,
|
||||
};
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
roots: graph.roots.clone(),
|
||||
sources,
|
||||
graph: Some(graph.clone()),
|
||||
runtime: runtime.pack.clone(),
|
||||
auxiliary_runtime: Some(environment.python_runtime),
|
||||
native_entries: output.native,
|
||||
native_rejections: output.rejections,
|
||||
python_markers: None,
|
||||
tools: environment.descriptor,
|
||||
frontend: super::frontend(&Ecosystem::Npm)?,
|
||||
python: None,
|
||||
dynamic_requires: vec![],
|
||||
dependencies: Box::new(LockFile::new(vec![])?),
|
||||
config_settings: BTreeMap::new(),
|
||||
output: output_artifact,
|
||||
};
|
||||
graph.build = Some(receipt.identity()?);
|
||||
self.remember(
|
||||
key,
|
||||
Product {
|
||||
bytes,
|
||||
filename: "installation.tgz".into(),
|
||||
receipt,
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn archive(output: &Output) -> Result<Vec<u8>, String> {
|
||||
let encoder = flate2::GzBuilder::new()
|
||||
.mtime(0)
|
||||
.operating_system(255)
|
||||
.write(Vec::new(), flate2::Compression::default());
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for (path, bytes) in &output.files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(if output.executables.contains(path) {
|
||||
0o755
|
||||
} else {
|
||||
0o644
|
||||
});
|
||||
header.set_uid(0);
|
||||
header.set_gid(0);
|
||||
header.set_mtime(0);
|
||||
header.set_cksum();
|
||||
archive
|
||||
.append_data(&mut header, path, Cursor::new(bytes))
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
archive
|
||||
.into_inner()
|
||||
.and_then(|encoder| encoder.finish())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn retain_archives(libraries: &mut Vec<LockedLib>, graph: &npm::Environment) {
|
||||
let used = graph
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| &node.source.archive)
|
||||
.collect::<BTreeSet<_>>();
|
||||
libraries.retain(|lib| used.contains(&lib.spec));
|
||||
}
|
||||
|
||||
fn required_nodes(graph: &npm::Environment) -> Result<BTreeSet<String>, String> {
|
||||
reachable(graph, true)
|
||||
}
|
||||
|
||||
fn reachable(graph: &npm::Environment, mandatory_only: bool) -> Result<BTreeSet<String>, String> {
|
||||
let mut remaining = graph
|
||||
.roots
|
||||
.iter()
|
||||
.map(|root| format!("node_modules/{}", root.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = remaining.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = graph
|
||||
.instances
|
||||
.iter()
|
||||
.find(|node| node.path == path)
|
||||
.ok_or("npm reachable instance missing")?;
|
||||
remaining.extend(
|
||||
node.edges
|
||||
.iter()
|
||||
.filter(|edge| {
|
||||
!mandatory_only || matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
|
||||
})
|
||||
.filter_map(|edge| edge.target.clone()),
|
||||
);
|
||||
}
|
||||
Ok(reached)
|
||||
}
|
||||
|
||||
fn prune_failed(graph: &mut npm::Environment, failed: &BTreeSet<String>) -> Result<(), String> {
|
||||
if failed
|
||||
.iter()
|
||||
.any(|path| !graph.instances.iter().any(|node| &node.path == path))
|
||||
{
|
||||
return Err("npm build reported an unknown failed instance".into());
|
||||
}
|
||||
let mut failed = failed.clone();
|
||||
loop {
|
||||
let before = failed.len();
|
||||
for node in &graph.instances {
|
||||
if node.edges.iter().any(|edge| {
|
||||
matches!(edge.kind, npm::Kind::Dependency | npm::Kind::Peer)
|
||||
&& edge
|
||||
.target
|
||||
.as_ref()
|
||||
.is_some_and(|path| failed.contains(path))
|
||||
}) {
|
||||
failed.insert(node.path.clone());
|
||||
}
|
||||
}
|
||||
if before == failed.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if !required_nodes(graph)?.is_disjoint(&failed) {
|
||||
return Err("required npm lifecycle failed".into());
|
||||
}
|
||||
graph.instances.retain(|node| !failed.contains(&node.path));
|
||||
for node in &mut graph.instances {
|
||||
for edge in &mut node.edges {
|
||||
if edge
|
||||
.target
|
||||
.as_ref()
|
||||
.is_some_and(|path| failed.contains(path))
|
||||
{
|
||||
edge.target = None;
|
||||
edge.omission = Some(npm::Omission::BuildFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
let reached = reachable(graph, false)?;
|
||||
graph.instances.retain(|node| reached.contains(&node.path));
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
//! One immutable Node/Python/tool environment for registry lifecycle execution.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs;
|
||||
|
||||
use crate::libs::build::NativeRejection;
|
||||
use crate::libs::build::{Inputs, pack, process::Stage};
|
||||
use crate::libs::{Ecosystem, RegistryRuntime, RuntimePack, archive_files, npm, sha256};
|
||||
use crate::workers::runtime;
|
||||
|
||||
pub(super) struct Environment {
|
||||
files: BTreeMap<String, (Vec<u8>, bool)>,
|
||||
assets: Vec<(String, Vec<u8>)>,
|
||||
node: String,
|
||||
node_paths: BTreeSet<String>,
|
||||
python: String,
|
||||
pub python_runtime: RuntimePack,
|
||||
pub descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
pub(super) struct Output {
|
||||
pub files: Vec<(String, Vec<u8>)>,
|
||||
pub executables: BTreeSet<String>,
|
||||
pub failed: BTreeSet<String>,
|
||||
pub native: BTreeSet<String>,
|
||||
pub rejections: BTreeMap<String, NativeRejection>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ResultMetadata {
|
||||
failed: BTreeSet<String>,
|
||||
native: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
pub fn new(inputs: &dyn Inputs, runtime: &RegistryRuntime) -> Result<Self, String> {
|
||||
crate::libs::build::require_execution_target(&runtime.target, "npm lifecycle build")?;
|
||||
let (node, bytes) = inputs.runtime(Ecosystem::Npm)?;
|
||||
if node.pack != runtime.pack || sha256(&bytes) != runtime.pack.sha256 {
|
||||
return Err("npm build runtime differs from resolver".into());
|
||||
}
|
||||
let node_files = archive_files("tgz", &bytes)?;
|
||||
let node_manifest = runtime::validate(&node_files, &Ecosystem::Npm, &runtime.target)?;
|
||||
let node_paths = node_files
|
||||
.iter()
|
||||
.map(|(path, _)| format!("runtime/{path}"))
|
||||
.collect();
|
||||
let node = format!(
|
||||
"runtime/{}",
|
||||
node_manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.ok_or("Node executable missing")?
|
||||
);
|
||||
let (python, bytes) = inputs.runtime(Ecosystem::Pip)?;
|
||||
if sha256(&bytes) != python.pack.sha256 {
|
||||
return Err("auxiliary Python runtime hash mismatch".into());
|
||||
}
|
||||
let python_files = archive_files("tgz", &bytes)?;
|
||||
let python_manifest = runtime::validate(&python_files, &Ecosystem::Pip, &runtime.target)?;
|
||||
let python_executable = format!(
|
||||
"runtime/{}",
|
||||
python_manifest
|
||||
.executable
|
||||
.as_deref()
|
||||
.ok_or("build Python executable missing")?
|
||||
);
|
||||
let (descriptor, bytes) = inputs.tools(Ecosystem::Npm)?;
|
||||
descriptor.validate(runtime, Some(&python))?;
|
||||
let (manifest, tools) = pack::unpack(&bytes, &descriptor)?;
|
||||
let assets = inputs.assets()?;
|
||||
dever_sandbox::validate_assets(&assets)?;
|
||||
pack::validate_tools(&manifest, &tools, &assets)?;
|
||||
let mut files = BTreeMap::new();
|
||||
for (path, bytes) in node_files {
|
||||
let executable = node_manifest.executable(&path);
|
||||
files.insert(format!("runtime/{path}"), (bytes, executable));
|
||||
}
|
||||
for (path, bytes) in python_files {
|
||||
if path == "dever-runtime.json" {
|
||||
continue;
|
||||
}
|
||||
let executable = python_manifest.executable(&path);
|
||||
let path = format!("runtime/{path}");
|
||||
if let Some((existing, _)) = files.get(&path) {
|
||||
if existing != &bytes {
|
||||
return Err("Node and auxiliary Python runtime paths conflict".into());
|
||||
}
|
||||
} else {
|
||||
files.insert(path, (bytes, executable));
|
||||
}
|
||||
}
|
||||
for (path, bytes) in tools {
|
||||
let executable = manifest.executables.contains(&path);
|
||||
if files.insert(path, (bytes, executable)).is_some() {
|
||||
return Err("npm tools overlap runtimes".into());
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
files,
|
||||
assets,
|
||||
node,
|
||||
node_paths,
|
||||
python: python_executable,
|
||||
python_runtime: python.pack,
|
||||
descriptor,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(
|
||||
&self,
|
||||
inputs: &dyn Inputs,
|
||||
graph: &npm::Environment,
|
||||
files: &[(String, Vec<u8>)],
|
||||
executables: &BTreeSet<String>,
|
||||
) -> Result<Output, String> {
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
stage.write(&format!("input/{path}"), bytes, *executable)?;
|
||||
}
|
||||
for (path, bytes) in &self.assets {
|
||||
stage.write(
|
||||
&format!("sandbox/{path}"),
|
||||
bytes,
|
||||
dever_sandbox::asset_executable(path),
|
||||
)?;
|
||||
}
|
||||
for (path, bytes) in files {
|
||||
stage.write(
|
||||
&format!("input/install/{path}"),
|
||||
bytes,
|
||||
executables.contains(path),
|
||||
)?;
|
||||
}
|
||||
let request = serde_json::json!({
|
||||
"instances":graph.instances, "required":super::required_nodes(graph)?,
|
||||
"roots":graph.roots.iter().map(|root|format!("node_modules/{}",root.name)).collect::<Vec<_>>(),
|
||||
"python":format!("/worker/{}", self.python),
|
||||
});
|
||||
stage.write(
|
||||
"input/request.json",
|
||||
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
)?;
|
||||
stage.write("input/frontend.js", include_bytes!("../npm.js"), false)?;
|
||||
let work = stage.0.join("work");
|
||||
fs::create_dir(&work).map_err(|error| error.to_string())?;
|
||||
let grants = [dever_sandbox::Grant {
|
||||
source: work.clone(),
|
||||
destination: "/data/build".into(),
|
||||
writable: true,
|
||||
}];
|
||||
let arguments = [input.join("frontend.js").into_os_string()];
|
||||
let executable = input.join(&self.node);
|
||||
let assets = stage.0.join("sandbox");
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities {
|
||||
network: false,
|
||||
process: true,
|
||||
},
|
||||
grants: &grants,
|
||||
};
|
||||
stage.run(
|
||||
launch.command_for_build(&input.join("rootfs"))?,
|
||||
"isolated npm lifecycle",
|
||||
&work,
|
||||
)?;
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
let mut outputs = super::super::python::output_files(&work.join("output"))?;
|
||||
let response: ResultMetadata = serde_json::from_slice(
|
||||
&outputs
|
||||
.remove("result.json")
|
||||
.ok_or("npm build result missing")?,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut files = Vec::new();
|
||||
let mut executables = BTreeSet::new();
|
||||
for (path, bytes) in outputs {
|
||||
let path = path
|
||||
.strip_prefix("install/")
|
||||
.ok_or("npm lifecycle output is outside its installation")?
|
||||
.to_owned();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if fs::symlink_metadata(work.join("output/install").join(&path))
|
||||
.map_err(|error| error.to_string())?
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o111
|
||||
!= 0
|
||||
{
|
||||
executables.insert(path.clone());
|
||||
}
|
||||
}
|
||||
files.push((path, bytes));
|
||||
}
|
||||
let mut output = Output {
|
||||
files,
|
||||
executables,
|
||||
failed: response.failed,
|
||||
native: response.native,
|
||||
rejections: BTreeMap::new(),
|
||||
};
|
||||
// Release the large compiler tree before staging the much smaller
|
||||
// runtime-only native admission sandbox.
|
||||
drop(stage);
|
||||
self.admit_native(inputs, &mut output)?;
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn admit_native(&self, inputs: &dyn Inputs, output: &mut Output) -> Result<(), String> {
|
||||
let required = std::mem::take(&mut output.native);
|
||||
let candidates = output
|
||||
.files
|
||||
.iter()
|
||||
.filter(|(path, _)| path.ends_with(".node"))
|
||||
.collect::<Vec<_>>();
|
||||
if candidates.is_empty() {
|
||||
return if required.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("npm generated native entry point is missing".into())
|
||||
};
|
||||
}
|
||||
let machine = match self.descriptor.target.as_str() {
|
||||
"linux-x86_64" => goblin::elf::header::EM_X86_64,
|
||||
"linux-aarch64" => goblin::elf::header::EM_AARCH64,
|
||||
_ => return Err("Node native admission target is unsupported".into()),
|
||||
};
|
||||
let mut files = self
|
||||
.files
|
||||
.iter()
|
||||
.filter(|(path, _)| self.node_paths.contains(*path))
|
||||
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
files.extend(
|
||||
self.assets
|
||||
.iter()
|
||||
.map(|(path, bytes)| (format!("sandbox/{path}"), bytes.as_slice())),
|
||||
);
|
||||
files.extend(
|
||||
output
|
||||
.files
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.clone(), bytes.as_slice())),
|
||||
);
|
||||
let stage = Stage::new()?;
|
||||
for (path, bytes) in &files {
|
||||
let relative = if path.starts_with("sandbox/") {
|
||||
path.clone()
|
||||
} else {
|
||||
format!("input/{path}")
|
||||
};
|
||||
let executable = path == &self.node
|
||||
|| path
|
||||
.strip_prefix("sandbox/")
|
||||
.is_some_and(dever_sandbox::asset_executable);
|
||||
stage.write(&relative, bytes, executable)?;
|
||||
}
|
||||
stage.write("input/probe.cjs", include_bytes!("probe.cjs"), false)?;
|
||||
let input = stage.0.join("input");
|
||||
let executable = input.join(&self.node);
|
||||
let assets = stage.0.join("sandbox");
|
||||
for (path, bytes) in candidates {
|
||||
let rejection = match goblin::elf::Elf::parse(bytes) {
|
||||
Err(_) => Some(NativeRejection::Format),
|
||||
Ok(elf) if !elf.is_64 || !elf.little_endian || elf.header.e_machine != machine => {
|
||||
Some(NativeRejection::Target)
|
||||
}
|
||||
Ok(_) => {
|
||||
if let Err(error) = crate::workers::python_wheel::validate_native(
|
||||
&self.descriptor.target,
|
||||
&[],
|
||||
&self.node,
|
||||
&files,
|
||||
&BTreeSet::from([path.clone()]),
|
||||
) {
|
||||
inputs.diagnostic(
|
||||
format!("npm addon '{path}' has no packaged loader closure: {error}\n")
|
||||
.as_bytes(),
|
||||
);
|
||||
Some(NativeRejection::Closure)
|
||||
} else {
|
||||
let arguments = [
|
||||
input.join("probe.cjs").into_os_string(),
|
||||
input.join(path).into_os_string(),
|
||||
];
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities::default(),
|
||||
grants: &[],
|
||||
};
|
||||
let admitted = stage.probe(launch.command()?)?;
|
||||
inputs.diagnostic(&stage.diagnostics()?);
|
||||
if admitted {
|
||||
None
|
||||
} else {
|
||||
Some(NativeRejection::Unloadable)
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Some(reason) = rejection {
|
||||
output.rejections.insert(path.clone(), reason);
|
||||
} else {
|
||||
output.native.insert(path.clone());
|
||||
}
|
||||
}
|
||||
if !required.is_subset(&output.native) {
|
||||
return Err("source-built npm addon did not pass runtime-only native admission".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
// No package entry modules are executed. Probe exactly one immutable addon in
|
||||
// an isolated runtime-only process, so addon globals cannot affect its peers.
|
||||
try {
|
||||
require(process.argv[2]);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
process.stderr.write(String(error.stack || error) + '\n');
|
||||
process.exit(10);
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
//! Signed, target-specific source-build tools. Never sourced from host PATH.
|
||||
|
||||
use crate::libs::{Ecosystem, RegistryRuntime};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
pub const MAX_PACK_BYTES: usize = 256 * 1024 * 1024;
|
||||
pub const MAX_EXPANDED_BYTES: usize = 512 * 1024 * 1024;
|
||||
pub const MAX_FILES: usize = 65_536;
|
||||
|
||||
pub type PackFiles = Vec<(String, Vec<u8>)>;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Manifest {
|
||||
pub format: String,
|
||||
pub ecosystem: Ecosystem,
|
||||
pub target: String,
|
||||
pub runtime_sha256: String,
|
||||
pub python_runtime_sha256: Option<String>,
|
||||
pub executables: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Descriptor {
|
||||
pub format: String,
|
||||
pub ecosystem: Ecosystem,
|
||||
pub target: String,
|
||||
pub runtime_sha256: String,
|
||||
pub python_runtime_sha256: Option<String>,
|
||||
pub sha256: String,
|
||||
}
|
||||
|
||||
fn hash(value: &str) -> Result<(), String> {
|
||||
if value.len() != 64
|
||||
|| !value
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
{
|
||||
return Err("build pack identity must be canonical SHA-256".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Descriptor {
|
||||
pub fn validate(
|
||||
&self,
|
||||
runtime: &RegistryRuntime,
|
||||
python: Option<&RegistryRuntime>,
|
||||
) -> Result<(), String> {
|
||||
if self.format != "dever-registry-build-v1"
|
||||
|| self.ecosystem == Ecosystem::Go
|
||||
|| self.target != runtime.target
|
||||
|| self.runtime_sha256 != runtime.pack.sha256
|
||||
{
|
||||
return Err("signed build pack does not match its runtime and target".into());
|
||||
}
|
||||
hash(&self.sha256)?;
|
||||
hash(&self.runtime_sha256)?;
|
||||
match (&self.ecosystem, &self.python_runtime_sha256, python) {
|
||||
(Ecosystem::Pip, None, _) => Ok(()),
|
||||
(Ecosystem::Npm, Some(identity), Some(python))
|
||||
if identity == &python.pack.sha256 && python.target == self.target =>
|
||||
{
|
||||
hash(identity)
|
||||
}
|
||||
_ => Err("build pack has no exact auxiliary Python runtime binding".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Manifest {
|
||||
pub fn validate(
|
||||
&self,
|
||||
files: &[(String, Vec<u8>)],
|
||||
descriptor: &Descriptor,
|
||||
) -> Result<(), String> {
|
||||
if self.format != "dever-lib-build-v1"
|
||||
|| self.ecosystem != descriptor.ecosystem
|
||||
|| self.target != descriptor.target
|
||||
|| self.runtime_sha256 != descriptor.runtime_sha256
|
||||
|| self.python_runtime_sha256 != descriptor.python_runtime_sha256
|
||||
{
|
||||
return Err("build pack manifest differs from its signed descriptor".into());
|
||||
}
|
||||
let total = files
|
||||
.iter()
|
||||
.try_fold(0usize, |total, (_, bytes)| total.checked_add(bytes.len()))
|
||||
.ok_or("build pack expanded size overflow")?;
|
||||
if files.len() > MAX_FILES || total > MAX_EXPANDED_BYTES {
|
||||
return Err("build pack exceeds file or byte budget".into());
|
||||
}
|
||||
let mut names = BTreeSet::new();
|
||||
for (path, _) in files {
|
||||
crate::workers::valid_path(path)?;
|
||||
if path != "dever-build.json"
|
||||
&& !path.starts_with("rootfs/usr/")
|
||||
&& !(self.ecosystem == Ecosystem::Pip && path.starts_with("runtime/include/"))
|
||||
&& !(self.ecosystem == Ecosystem::Npm
|
||||
&& (path.starts_with("npm/include/node/")
|
||||
|| path.starts_with("npm/frontend/node_modules/")))
|
||||
{
|
||||
return Err(format!(
|
||||
"build pack path '{path}' is outside its fixed tool layout"
|
||||
));
|
||||
}
|
||||
if !names.insert(path.as_str()) {
|
||||
return Err("build pack repeats a path".into());
|
||||
}
|
||||
}
|
||||
let mut executable_names = BTreeSet::new();
|
||||
for path in &self.executables {
|
||||
if !names.contains(path.as_str())
|
||||
|| !executable_names.insert(path.as_str())
|
||||
|| !(path.starts_with("rootfs/usr/")
|
||||
|| self.ecosystem == Ecosystem::Npm
|
||||
&& path.starts_with("npm/frontend/node_modules/"))
|
||||
{
|
||||
return Err(
|
||||
"build executable is absent, repeated, or outside its tool root".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
for name in ["sh", "cc", "c++", "as", "ld", "ar", "ranlib", "make"] {
|
||||
if !executable_names.contains(format!("rootfs/usr/bin/{name}").as_str()) {
|
||||
return Err(format!("build pack is missing required tool '{name}'"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn unpack(bytes: &[u8], descriptor: &Descriptor) -> Result<(Manifest, PackFiles), String> {
|
||||
if bytes.len() > MAX_PACK_BYTES || crate::libs::sha256(bytes) != descriptor.sha256 {
|
||||
return Err("build pack exceeds its budget or SHA-256 differs".into());
|
||||
}
|
||||
let files = super::super::registry::tar_files(bytes, "tgz", MAX_EXPANDED_BYTES as u64)?;
|
||||
let metadata = files
|
||||
.iter()
|
||||
.find(|(path, _)| path == "dever-build.json")
|
||||
.ok_or("build pack is missing dever-build.json")?;
|
||||
let manifest: Manifest = serde_json::from_slice(&metadata.1)
|
||||
.map_err(|error| format!("invalid build pack manifest: {error}"))?;
|
||||
manifest.validate(&files, descriptor)?;
|
||||
Ok((manifest, files))
|
||||
}
|
||||
|
||||
pub fn validate_tools(
|
||||
manifest: &Manifest,
|
||||
files: &[(String, Vec<u8>)],
|
||||
assets: &[(String, Vec<u8>)],
|
||||
) -> Result<(), String> {
|
||||
let mut supplied = files
|
||||
.iter()
|
||||
.map(|(path, bytes)| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (path, bytes) in assets {
|
||||
supplied.insert(format!("sandbox/{path}"), bytes);
|
||||
}
|
||||
crate::workers::python_wheel::native::validate_build_tools(
|
||||
&manifest.target,
|
||||
&manifest.executables,
|
||||
&supplied,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//! One lifecycle for every managed compiler and package build hook.
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const DEADLINE: Duration = Duration::from_secs(300);
|
||||
const MAX_OUTPUT: u64 = 16 * 1024 * 1024;
|
||||
static NEXT_STAGE: AtomicU64 = AtomicU64::new(0);
|
||||
mod budget;
|
||||
|
||||
pub(crate) struct Stage(pub(crate) PathBuf);
|
||||
|
||||
impl Stage {
|
||||
pub(crate) fn new() -> Result<Self, String> {
|
||||
for _ in 0..32 {
|
||||
let path = std::env::temp_dir().join(format!(
|
||||
"dever-lib-build-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_STAGE.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
#[cfg(unix)]
|
||||
let created = {
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
let mut builder = fs::DirBuilder::new();
|
||||
builder.mode(0o700);
|
||||
builder.create(&path)
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let created = fs::create_dir(&path);
|
||||
match created {
|
||||
Ok(()) => return Ok(Self(path)),
|
||||
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(error) => return Err(format!("cannot create private build staging: {error}")),
|
||||
}
|
||||
}
|
||||
Err("cannot allocate private build staging".into())
|
||||
}
|
||||
|
||||
pub(crate) fn write(
|
||||
&self,
|
||||
relative: &str,
|
||||
bytes: &[u8],
|
||||
executable: bool,
|
||||
) -> Result<PathBuf, String> {
|
||||
crate::workers::valid_path(relative)?;
|
||||
let path = self.0.join(relative);
|
||||
fs::create_dir_all(path.parent().expect("validated relative path"))
|
||||
.map_err(|error| format!("cannot create build input directory: {error}"))?;
|
||||
// This tree is private and never a writable child mount. A second write
|
||||
// must not silently replace a previously verified input.
|
||||
use std::io::Write;
|
||||
let mut file = fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot stage build input: {error}"))?;
|
||||
file.write_all(bytes)
|
||||
.map_err(|error| format!("cannot write build input: {error}"))?;
|
||||
#[cfg(unix)]
|
||||
if executable {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
file.set_permissions(fs::Permissions::from_mode(0o700))
|
||||
.map_err(|error| format!("cannot mark build tool executable: {error}"))?;
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
pub(crate) fn run(
|
||||
&self,
|
||||
command: Command,
|
||||
operation: &str,
|
||||
writable: &Path,
|
||||
) -> Result<Vec<u8>, String> {
|
||||
let status = self.execute(command, operation, Some(writable))?;
|
||||
if !status.success() {
|
||||
return Err(format!(
|
||||
"{operation} failed: {}",
|
||||
String::from_utf8_lossy(&self.diagnostics()?)
|
||||
));
|
||||
}
|
||||
fs::read(self.0.join("tool-stdout"))
|
||||
.map_err(|error| format!("cannot read {operation} output: {error}"))
|
||||
}
|
||||
|
||||
pub(crate) fn probe(&self, command: Command) -> Result<bool, String> {
|
||||
match self
|
||||
.execute(command, "isolated Node addon probe", None)?
|
||||
.code()
|
||||
{
|
||||
Some(0) => Ok(true),
|
||||
Some(10) => Ok(false),
|
||||
_ => Err(format!(
|
||||
"Node addon probe failed outside its loader check: {}",
|
||||
String::from_utf8_lossy(&self.diagnostics()?)
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn execute(
|
||||
&self,
|
||||
mut command: Command,
|
||||
operation: &str,
|
||||
writable: Option<&Path>,
|
||||
) -> Result<std::process::ExitStatus, String> {
|
||||
// Captures are outside every child-writable grant. Namespace teardown
|
||||
// reaps descendants when the owned bootstrap is killed on any exit path.
|
||||
let stdout = self.0.join("tool-stdout");
|
||||
let stderr = self.0.join("tool-stderr");
|
||||
let child = command
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(
|
||||
File::create(&stdout).map_err(|error| error.to_string())?,
|
||||
))
|
||||
.stderr(Stdio::from(
|
||||
File::create(&stderr).map_err(|error| error.to_string())?,
|
||||
))
|
||||
.spawn()
|
||||
.map_err(|error| format!("cannot execute {operation}: {error}"))?;
|
||||
let mut child = OwnedChild(child);
|
||||
let deadline = Instant::now() + DEADLINE;
|
||||
let mut next_scratch_check = Instant::now();
|
||||
let within_budget = || -> Result<(), String> {
|
||||
for path in [&stdout, &stderr] {
|
||||
if fs::metadata(path).map_err(|error| error.to_string())?.len() > MAX_OUTPUT {
|
||||
return Err(format!("{operation} output exceeds byte limit"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let status = loop {
|
||||
within_budget()?;
|
||||
if Instant::now() >= next_scratch_check {
|
||||
if let Some(path) = writable {
|
||||
budget::check(path)?;
|
||||
}
|
||||
next_scratch_check = Instant::now() + Duration::from_millis(250);
|
||||
}
|
||||
if let Some(status) = child
|
||||
.0
|
||||
.try_wait()
|
||||
.map_err(|error| format!("cannot wait for {operation}: {error}"))?
|
||||
{
|
||||
break status;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!("{operation} exceeded its five-minute deadline"));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(25));
|
||||
};
|
||||
within_budget()?;
|
||||
if let Some(path) = writable {
|
||||
budget::check(path)?;
|
||||
}
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
pub(crate) fn diagnostics(&self) -> Result<Vec<u8>, String> {
|
||||
let mut bytes = Vec::new();
|
||||
for name in ["tool-stdout", "tool-stderr"] {
|
||||
File::open(self.0.join(name))
|
||||
.and_then(|file| file.take(MAX_OUTPUT).read_to_end(&mut bytes))
|
||||
.map_err(|error| format!("cannot read build diagnostic: {error}"))?;
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Stage {
|
||||
fn drop(&mut self) {
|
||||
let _ = fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct OwnedChild(Child);
|
||||
|
||||
impl Drop for OwnedChild {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
//! Periodic scratch accounting is separate from immutable runtime/tool inputs.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
const MAX_BYTES: u64 = 512 * 1024 * 1024;
|
||||
const MAX_ENTRIES: usize = 65_536;
|
||||
const MAX_DEPTH: usize = 128;
|
||||
|
||||
#[derive(Default)]
|
||||
struct Budget {
|
||||
bytes: u64,
|
||||
entries: usize,
|
||||
}
|
||||
|
||||
pub(super) fn check(root: &Path) -> Result<(), String> {
|
||||
visit(root, 0, &mut Budget::default()).map_err(|error| format!("build scratch budget: {error}"))
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn visit(path: &Path, depth: usize, budget: &mut Budget) -> Result<(), String> {
|
||||
use std::os::fd::AsRawFd;
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
if depth > MAX_DEPTH {
|
||||
return Err("directory depth exceeds limit".into());
|
||||
}
|
||||
// An open directory pins the inode throughout enumeration. Child-created
|
||||
// symlinks and rename races cannot redirect accounting outside this tree.
|
||||
let directory = match fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.custom_flags((rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::DIRECTORY).bits() as i32)
|
||||
.open(path)
|
||||
{
|
||||
Ok(file) => file,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(format!("cannot pin scratch directory: {error}")),
|
||||
};
|
||||
let pinned = format!("/proc/self/fd/{}", directory.as_raw_fd());
|
||||
for entry in fs::read_dir(pinned).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let metadata = match entry.metadata() {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error.to_string()),
|
||||
};
|
||||
budget.entries += 1;
|
||||
if budget.entries > MAX_ENTRIES {
|
||||
return Err("entry count exceeds limit".into());
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
visit(&entry.path(), depth + 1, budget)?;
|
||||
} else {
|
||||
budget.bytes = budget
|
||||
.bytes
|
||||
.checked_add(metadata.len())
|
||||
.ok_or("byte count overflow")?;
|
||||
if budget.bytes > MAX_BYTES {
|
||||
return Err("bytes exceed 512 MiB limit".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn visit(_path: &Path, _depth: usize, _budget: &mut Budget) -> Result<(), String> {
|
||||
Err("managed build scratch accounting requires its Linux sandbox".into())
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Fixed PEP 517 frontend, executed only inside the managed build sandbox."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
|
||||
def build_system(source):
|
||||
project = source / "pyproject.toml"
|
||||
config = tomllib.loads(project.read_text("utf-8")) if project.exists() else {}
|
||||
system = config.get("build-system")
|
||||
if system is None:
|
||||
system = {"requires": ["setuptools>=40.8.0"], "build-backend": "setuptools.build_meta:__legacy__"}
|
||||
if not isinstance(system, dict) or not isinstance(system.get("requires"), list):
|
||||
raise ValueError("build-system.requires must be an array")
|
||||
requires = system["requires"]
|
||||
if len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
|
||||
raise ValueError("invalid static build requirements")
|
||||
backend = system.get("build-backend", "setuptools.build_meta:__legacy__")
|
||||
identifier = r"[A-Za-z_][A-Za-z_0-9]*(?:\.[A-Za-z_][A-Za-z_0-9]*)*"
|
||||
if not isinstance(backend, str) or not re.fullmatch(identifier + "(?::" + identifier + ")?", backend):
|
||||
raise ValueError("invalid build-backend entry point")
|
||||
paths = system.get("backend-path", [])
|
||||
if not isinstance(paths, list) or len(paths) > 64:
|
||||
raise ValueError("invalid backend-path")
|
||||
for name in paths:
|
||||
if not isinstance(name, str) or not name or Path(name).is_absolute() or "\\" in name:
|
||||
raise ValueError("backend-path must be source-relative")
|
||||
resolved = (source / name).resolve()
|
||||
if not resolved.is_relative_to(source.resolve()) or not resolved.is_dir():
|
||||
raise ValueError("backend-path escapes the source tree")
|
||||
return {"requires": requires, "backend": backend, "backend_path": paths}
|
||||
|
||||
|
||||
def backend_object(source, system):
|
||||
paths = [(source / path).resolve() for path in system["backend_path"]]
|
||||
sys.path[:0] = [str(path) for path in paths]
|
||||
module_name, _, object_name = system["backend"].partition(":")
|
||||
module = importlib.import_module(module_name)
|
||||
if paths:
|
||||
origin = Path(module.__file__).resolve()
|
||||
if not any(origin.is_relative_to(path) for path in paths):
|
||||
raise ValueError("in-tree backend was loaded outside backend-path")
|
||||
backend = module
|
||||
for part in object_name.split(".") if object_name else []:
|
||||
backend = getattr(backend, part)
|
||||
return backend
|
||||
|
||||
|
||||
def leaf(value, suffix):
|
||||
if not isinstance(value, str) or Path(value).name != value or not value.endswith(suffix):
|
||||
raise ValueError("build hook returned an invalid output name")
|
||||
return value
|
||||
|
||||
|
||||
def execute(request, source, output):
|
||||
system = build_system(source)
|
||||
if request["phase"] == "inspect":
|
||||
return system
|
||||
if system != request["system"]:
|
||||
raise ValueError("source build configuration changed between hooks")
|
||||
backend = backend_object(source, system)
|
||||
settings = request["config_settings"]
|
||||
if request["phase"] == "requires":
|
||||
hook = getattr(backend, "get_requires_for_build_wheel", None)
|
||||
requires = hook(settings) if hook else []
|
||||
if not isinstance(requires, list) or len(requires) > 1024 or any(not isinstance(value, str) for value in requires):
|
||||
raise ValueError("invalid dynamic build requirements")
|
||||
return {"requires": requires}
|
||||
if request["phase"] == "metadata":
|
||||
metadata = output / "metadata"
|
||||
metadata.mkdir()
|
||||
hook = getattr(backend, "prepare_metadata_for_build_wheel", None)
|
||||
prepared = metadata / leaf(hook(str(metadata), settings), ".dist-info") if hook else None
|
||||
if prepared is not None and (not prepared.is_dir() or prepared.is_symlink()):
|
||||
raise ValueError("metadata hook did not create its declared directory")
|
||||
return {"metadata": str(prepared.relative_to(output)) if prepared else None}
|
||||
prepared = None
|
||||
if request.get("metadata") is not None:
|
||||
shutil.copytree("/worker/prepared", output / "prepared")
|
||||
prepared = output / "prepared" / request["metadata"]
|
||||
if not prepared.resolve().is_relative_to((output / "prepared").resolve()):
|
||||
raise ValueError("prepared metadata path escapes its snapshot")
|
||||
wheels = output / "wheels"
|
||||
wheels.mkdir()
|
||||
name = leaf(backend.build_wheel(str(wheels), settings, str(prepared) if prepared else None), ".whl")
|
||||
return {"filename": name}
|
||||
|
||||
|
||||
def main():
|
||||
request = json.loads(Path("/worker/request.json").read_text("utf-8"))
|
||||
work = Path("/data/build")
|
||||
source = work / "source"
|
||||
shutil.copytree("/worker/source", source)
|
||||
output = work / "output"
|
||||
output.mkdir()
|
||||
(work / "tmp").mkdir()
|
||||
# These values are part of the fixed frontend, never project/host settings.
|
||||
os.environ.update({
|
||||
"PATH": "/worker/runtime/bin:/usr/bin",
|
||||
"CC": "/usr/bin/cc", "CXX": "/usr/bin/c++",
|
||||
"AR": "/usr/bin/ar", "RANLIB": "/usr/bin/ranlib",
|
||||
"LDSHARED": "/usr/bin/cc -shared",
|
||||
"CFLAGS": "-fPIC -ffile-prefix-map=/data/build/source=.",
|
||||
"CPPFLAGS": "-I/worker/runtime/include/python" + request["python_version"],
|
||||
"TMPDIR": "/data/build/tmp", "SOURCE_DATE_EPOCH": "0",
|
||||
})
|
||||
os.chdir(source)
|
||||
result = execute(request, source, output)
|
||||
(output / "result.json").write_text(json.dumps(result, sort_keys=True), "utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,564 @@
|
||||
//! PEP 517 source candidates enter the normal dependency solver as real wheels.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::ffi::OsString;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::{Product, PythonSystem, Receipt, Session, pack, process::Stage};
|
||||
use crate::libs::{
|
||||
Ecosystem, LibSpec, LockFile, LockedArtifact, RegistryResolver, RegistryRuntime, archive_files,
|
||||
sha256,
|
||||
};
|
||||
use crate::workers::{python_wheel::Wheel, runtime};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Environment {
|
||||
files: BTreeMap<String, (Vec<u8>, bool)>,
|
||||
assets: Vec<(String, Vec<u8>)>,
|
||||
executable: String,
|
||||
version: String,
|
||||
extension_suffixes: Vec<String>,
|
||||
descriptor: pack::Descriptor,
|
||||
}
|
||||
|
||||
struct Active<'a> {
|
||||
session: &'a Session<'a>,
|
||||
key: String,
|
||||
}
|
||||
impl Drop for Active<'_> {
|
||||
fn drop(&mut self) {
|
||||
self.session.active.borrow_mut().remove(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(crate) fn python(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
spec: &LibSpec,
|
||||
runtime: &RegistryRuntime,
|
||||
filename: &str,
|
||||
source: &[u8],
|
||||
) -> Result<Product, String> {
|
||||
let key = format!(
|
||||
"{}@{}:{}:{}",
|
||||
spec.distribution_name(),
|
||||
spec.version,
|
||||
sha256(source),
|
||||
runtime.pack.sha256
|
||||
);
|
||||
if let Some(product) = self.products.borrow().get(&key) {
|
||||
return Ok(product.clone());
|
||||
}
|
||||
if self.active.borrow().len() >= 16 || !self.active.borrow_mut().insert(key.clone()) {
|
||||
return Err(format!(
|
||||
"cyclic or overly deep Python build requirement for {}",
|
||||
spec.key()
|
||||
));
|
||||
}
|
||||
let _active = Active {
|
||||
session: self,
|
||||
key: key.clone(),
|
||||
};
|
||||
let sources = source_files(filename, source)?;
|
||||
let environment = Environment::new(self, runtime)?;
|
||||
let empty = LockFile::new(vec![])?;
|
||||
let inspected =
|
||||
environment.run(resolver, &empty, &sources, json!({"phase":"inspect"}), None)?;
|
||||
let system: PythonSystem = serde_json::from_value(inspected.response)
|
||||
.map_err(|error| format!("invalid Python build configuration: {error}"))?;
|
||||
let mut requirements = system.requires.clone();
|
||||
let mut dependencies = resolver.build_requirements(&requirements, runtime)?;
|
||||
let mut stable = None;
|
||||
let mut discovered = BTreeSet::new();
|
||||
for _ in 0..8 {
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"requires","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let dynamic: Vec<String> = serde_json::from_value(
|
||||
result
|
||||
.response
|
||||
.get("requires")
|
||||
.cloned()
|
||||
.ok_or("build backend omitted requirements")?,
|
||||
)
|
||||
.map_err(|error| format!("invalid dynamic Python build requirements: {error}"))?;
|
||||
discovered.extend(dynamic);
|
||||
requirements = system.requires.iter().chain(&discovered).cloned().collect();
|
||||
let next = resolver.build_requirements(&requirements, runtime)?;
|
||||
if next == dependencies {
|
||||
stable = Some(discovered.iter().cloned().collect());
|
||||
break;
|
||||
}
|
||||
dependencies = next;
|
||||
}
|
||||
let dynamic = stable.ok_or("Python dynamic build requirements did not converge")?;
|
||||
let metadata = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"metadata","system":system}),
|
||||
None,
|
||||
)?;
|
||||
let result = environment.run(
|
||||
resolver,
|
||||
&dependencies,
|
||||
&sources,
|
||||
json!({"phase":"wheel","system":system,"metadata":metadata.response.get("metadata")}),
|
||||
Some(&metadata.outputs),
|
||||
)?;
|
||||
let filename = result
|
||||
.response
|
||||
.get("filename")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("build backend omitted wheel filename")?;
|
||||
crate::workers::valid_path(filename)?;
|
||||
if filename.contains('/') || !filename.ends_with(".whl") {
|
||||
return Err("build backend returned invalid wheel filename".into());
|
||||
}
|
||||
let bytes = result
|
||||
.outputs
|
||||
.get(&format!("wheels/{filename}"))
|
||||
.ok_or("build backend did not produce its declared wheel")?
|
||||
.clone();
|
||||
Wheel::parse(spec, Some(filename), &runtime.python_wheel_tags, &bytes)?;
|
||||
if let Some(prefix) = metadata.response.get("metadata").and_then(Value::as_str) {
|
||||
compare_metadata(prefix, &metadata.outputs, &bytes)?;
|
||||
}
|
||||
let output = artifact(
|
||||
runtime,
|
||||
format!(
|
||||
"lib/pip/{}/{}/archive.whl",
|
||||
sha256(spec.distribution_name().as_bytes()),
|
||||
spec.version
|
||||
),
|
||||
&bytes,
|
||||
);
|
||||
let source_artifact = artifact(
|
||||
runtime,
|
||||
format!("build/source/{}/source", sha256(source)),
|
||||
source,
|
||||
);
|
||||
for (artifact, bytes) in [(&output, bytes.as_slice()), (&source_artifact, source)] {
|
||||
if resolver.store.publish(bytes, &runtime.target)? != artifact.sha256 {
|
||||
return Err("build artifact cache identity mismatch".into());
|
||||
}
|
||||
}
|
||||
let base: LibSpec = format!("pip:{}@{}", spec.distribution_name(), spec.version).parse()?;
|
||||
let receipt = Receipt {
|
||||
ecosystem: Ecosystem::Pip,
|
||||
roots: vec![base],
|
||||
sources: vec![source_artifact],
|
||||
graph: None,
|
||||
runtime: runtime.pack.clone(),
|
||||
auxiliary_runtime: None,
|
||||
native_entries: BTreeSet::new(),
|
||||
native_rejections: BTreeMap::new(),
|
||||
python_markers: runtime.python_markers.clone(),
|
||||
tools: environment.descriptor,
|
||||
frontend: super::frontend(&Ecosystem::Pip)?,
|
||||
python: Some(system),
|
||||
dynamic_requires: dynamic,
|
||||
dependencies: Box::new(dependencies),
|
||||
config_settings: BTreeMap::new(),
|
||||
output,
|
||||
};
|
||||
let product = Product {
|
||||
bytes,
|
||||
filename: filename.into(),
|
||||
receipt,
|
||||
};
|
||||
self.remember(key, product.clone())?;
|
||||
Ok(product)
|
||||
}
|
||||
}
|
||||
|
||||
fn artifact(runtime: &RegistryRuntime, path: String, bytes: &[u8]) -> LockedArtifact {
|
||||
LockedArtifact {
|
||||
target: runtime.target.clone(),
|
||||
path,
|
||||
bytes: bytes.len() as u64,
|
||||
sha256: sha256(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
struct Source {
|
||||
files: Vec<(String, Vec<u8>)>,
|
||||
executables: BTreeSet<String>,
|
||||
}
|
||||
|
||||
fn source_files(filename: &str, bytes: &[u8]) -> Result<Source, String> {
|
||||
let extension = if filename.ends_with(".tar.gz") {
|
||||
"tgz"
|
||||
} else if filename.ends_with(".zip") {
|
||||
"zip"
|
||||
} else {
|
||||
return Err("unsupported Python source archive format".into());
|
||||
};
|
||||
let files = archive_files(extension, bytes)?;
|
||||
let executables = crate::libs::registry::archive_executables(extension, bytes)?;
|
||||
let root = files
|
||||
.first()
|
||||
.and_then(|(path, _)| path.split_once('/'))
|
||||
.map(|(root, _)| format!("{root}/"))
|
||||
.ok_or("Python source archive needs one root directory")?;
|
||||
let files = files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| {
|
||||
let path = path
|
||||
.strip_prefix(&root)
|
||||
.ok_or("Python source archive contains multiple roots")?
|
||||
.to_owned();
|
||||
crate::workers::valid_path(&path)?;
|
||||
Ok((path, bytes))
|
||||
})
|
||||
.collect::<Result<Vec<_>, String>>()?;
|
||||
let executables = executables
|
||||
.into_iter()
|
||||
.map(|path| {
|
||||
path.strip_prefix(&root)
|
||||
.map(str::to_owned)
|
||||
.ok_or("source executable escapes archive root".into())
|
||||
})
|
||||
.collect::<Result<_, String>>()?;
|
||||
Ok(Source { files, executables })
|
||||
}
|
||||
|
||||
struct ResultFiles {
|
||||
response: Value,
|
||||
outputs: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
fn new(session: &Session<'_>, requested: &RegistryRuntime) -> Result<Self, String> {
|
||||
super::require_execution_target(&requested.target, "Python source build")?;
|
||||
let (runtime, bytes) = session.inputs.runtime(Ecosystem::Pip)?;
|
||||
if runtime.pack != requested.pack || sha256(&bytes) != requested.pack.sha256 {
|
||||
return Err("build Python runtime differs from resolver identity".into());
|
||||
}
|
||||
let runtime_files = archive_files("tgz", &bytes)?;
|
||||
let manifest = runtime::validate(&runtime_files, &Ecosystem::Pip, &requested.target)?;
|
||||
if manifest.python_wheel_tags != requested.python_wheel_tags
|
||||
|| manifest.python_markers != requested.python_markers
|
||||
{
|
||||
return Err("build runtime policy differs from signed descriptor".into());
|
||||
}
|
||||
let executable = manifest
|
||||
.executable
|
||||
.clone()
|
||||
.ok_or("build Python executable missing")?;
|
||||
let executable = format!("runtime/{executable}");
|
||||
let mut files = runtime_files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| {
|
||||
let executable = manifest.executable(&path);
|
||||
(format!("runtime/{path}"), (bytes, executable))
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let (descriptor, bytes) = session.inputs.tools(Ecosystem::Pip)?;
|
||||
descriptor.validate(requested, None)?;
|
||||
let (tools, tool_files) = pack::unpack(&bytes, &descriptor)?;
|
||||
let assets = session.inputs.assets()?;
|
||||
dever_sandbox::validate_assets(&assets)?;
|
||||
pack::validate_tools(&tools, &tool_files, &assets)?;
|
||||
for (path, bytes) in tool_files {
|
||||
let executable = tools.executables.contains(&path);
|
||||
if files.insert(path, (bytes, executable)).is_some() {
|
||||
return Err("build tools overlap runtime inputs".into());
|
||||
}
|
||||
}
|
||||
let version = requested
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("build Python markers absent")?
|
||||
.python_version()
|
||||
.to_string();
|
||||
Ok(Self {
|
||||
files,
|
||||
assets,
|
||||
executable,
|
||||
version,
|
||||
extension_suffixes: manifest.python_extension_suffixes,
|
||||
descriptor,
|
||||
})
|
||||
}
|
||||
|
||||
fn run(
|
||||
&self,
|
||||
resolver: &RegistryResolver<'_>,
|
||||
dependencies: &LockFile,
|
||||
source: &Source,
|
||||
mut request: Value,
|
||||
prepared: Option<&BTreeMap<String, Vec<u8>>>,
|
||||
) -> Result<ResultFiles, String> {
|
||||
let stage = Stage::new()?;
|
||||
let input = stage.0.join("input");
|
||||
for (path, (bytes, executable)) in &self.files {
|
||||
stage.write(&format!("input/{path}"), bytes, *executable)?;
|
||||
}
|
||||
for (path, bytes) in &self.assets {
|
||||
stage.write(
|
||||
&format!("sandbox/{path}"),
|
||||
bytes,
|
||||
dever_sandbox::asset_executable(path),
|
||||
)?;
|
||||
}
|
||||
for (path, bytes) in &source.files {
|
||||
stage.write(
|
||||
&format!("input/source/{path}"),
|
||||
bytes,
|
||||
source.executables.contains(path),
|
||||
)?;
|
||||
}
|
||||
if let Some(prepared) = prepared {
|
||||
for (path, bytes) in prepared {
|
||||
stage.write(&format!("input/prepared/{path}"), bytes, false)?;
|
||||
}
|
||||
}
|
||||
let mut installed = BTreeMap::<String, Vec<u8>>::new();
|
||||
let runtime = resolver
|
||||
.runtimes
|
||||
.get(&Ecosystem::Pip)
|
||||
.ok_or("build dependency runtime is missing")?;
|
||||
let mut distributions = BTreeSet::new();
|
||||
for lib in &dependencies.libs {
|
||||
super::validate_python_policy(
|
||||
dependencies,
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build markers absent")?,
|
||||
)?;
|
||||
if lib.runtime != runtime.pack || lib.artifacts.len() != 1 {
|
||||
return Err(
|
||||
"build dependency does not use the exact Python runtime and wheel".into(),
|
||||
);
|
||||
}
|
||||
let artifact = &lib.artifacts[0];
|
||||
let bytes =
|
||||
resolver
|
||||
.store
|
||||
.verify_exact(&artifact.sha256, artifact.bytes, &artifact.target)?;
|
||||
let wheel = Wheel::parse(&lib.spec, None, &runtime.python_wheel_tags, &bytes)?;
|
||||
wheel.validate_dependencies(
|
||||
lib,
|
||||
runtime
|
||||
.python_markers
|
||||
.as_ref()
|
||||
.ok_or("Python build markers absent")?,
|
||||
)?;
|
||||
if !distributions.insert((
|
||||
lib.spec.distribution_name().to_owned(),
|
||||
lib.spec.version.clone(),
|
||||
)) {
|
||||
continue;
|
||||
}
|
||||
for file in wheel.install(&self.executable, &self.version)? {
|
||||
if self.files.contains_key(&file.path)
|
||||
|| installed
|
||||
.insert(file.path.clone(), file.bytes.clone())
|
||||
.is_some()
|
||||
{
|
||||
return Err("build dependency installation has conflicting file owners".into());
|
||||
}
|
||||
stage.write(
|
||||
&format!("input/{}", file.path),
|
||||
&file.bytes,
|
||||
file.executable,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
let mut supplied = self
|
||||
.files
|
||||
.iter()
|
||||
.map(|(path, (bytes, _))| (path.clone(), bytes.as_slice()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
for (path, bytes) in &self.assets {
|
||||
supplied.insert(format!("sandbox/{path}"), bytes);
|
||||
}
|
||||
for (path, bytes) in &installed {
|
||||
supplied.insert(path.clone(), bytes);
|
||||
}
|
||||
crate::workers::python_wheel::validate_native(
|
||||
&runtime.target,
|
||||
&self.extension_suffixes,
|
||||
&self.executable,
|
||||
&supplied,
|
||||
&installed.keys().cloned().collect(),
|
||||
)?;
|
||||
request["python_version"] = json!(self.version);
|
||||
request["config_settings"] = json!({});
|
||||
stage.write(
|
||||
"input/request.json",
|
||||
&serde_json::to_vec(&request).map_err(|error| error.to_string())?,
|
||||
false,
|
||||
)?;
|
||||
stage.write("input/frontend.py", include_bytes!("python.py"), false)?;
|
||||
let work = stage.0.join("work");
|
||||
fs::create_dir(&work).map_err(|error| error.to_string())?;
|
||||
let grants = [dever_sandbox::Grant {
|
||||
source: work.clone(),
|
||||
destination: "/data/build".into(),
|
||||
writable: true,
|
||||
}];
|
||||
let arguments = [
|
||||
OsString::from("-I"),
|
||||
OsString::from("-B"),
|
||||
input.join("frontend.py").into_os_string(),
|
||||
];
|
||||
let executable = input.join(&self.executable);
|
||||
let assets = stage.0.join("sandbox");
|
||||
let launch = dever_sandbox::Launch {
|
||||
assets: &assets,
|
||||
worker: &input,
|
||||
executable: &executable,
|
||||
arguments: &arguments,
|
||||
working_directory: &input,
|
||||
capabilities: dever_sandbox::Capabilities {
|
||||
network: false,
|
||||
process: true,
|
||||
},
|
||||
grants: &grants,
|
||||
};
|
||||
stage.run(
|
||||
launch.command_for_build(&input.join("rootfs"))?,
|
||||
"isolated Python build hook",
|
||||
&work,
|
||||
)?;
|
||||
if let Some(session) = resolver.build {
|
||||
session.inputs.diagnostic(&stage.diagnostics()?);
|
||||
}
|
||||
let outputs = output_files(&work.join("output"))?;
|
||||
let response = serde_json::from_slice(
|
||||
outputs
|
||||
.get("result.json")
|
||||
.ok_or("build hook produced no result")?,
|
||||
)
|
||||
.map_err(|error| format!("invalid build hook response: {error}"))?;
|
||||
Ok(ResultFiles { response, outputs })
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn output_files(root: &Path) -> Result<BTreeMap<String, Vec<u8>>, String> {
|
||||
let mut files = BTreeMap::new();
|
||||
let mut pending = vec![root.to_owned()];
|
||||
let mut remaining = crate::workers::MAX_TREE_BYTES;
|
||||
let mut entries = 0usize;
|
||||
#[cfg(unix)]
|
||||
let mut links = BTreeMap::<(u64, u64), (u64, u64)>::new();
|
||||
while let Some(directory) = pending.pop() {
|
||||
if fs::symlink_metadata(&directory)
|
||||
.map_err(|error| error.to_string())?
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
{
|
||||
return Err("build output directory is a symbolic link".into());
|
||||
}
|
||||
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
entries += 1;
|
||||
if entries > crate::workers::MAX_TREE_FILES {
|
||||
return Err("build output exceeds file budget".into());
|
||||
}
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let kind = entry.file_type().map_err(|error| error.to_string())?;
|
||||
if kind.is_dir() {
|
||||
pending.push(entry.path());
|
||||
continue;
|
||||
}
|
||||
if !kind.is_file() {
|
||||
return Err("build output contains a link or special file".into());
|
||||
}
|
||||
let mut options = fs::OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
|
||||
}
|
||||
let file = options
|
||||
.open(entry.path())
|
||||
.map_err(|error| error.to_string())?;
|
||||
let metadata = file.metadata().map_err(|error| error.to_string())?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let aliases = links
|
||||
.entry((metadata.dev(), metadata.ino()))
|
||||
.or_insert((metadata.nlink(), 0));
|
||||
if aliases.0 != metadata.nlink() {
|
||||
return Err("build output link count changed during collection".into());
|
||||
}
|
||||
aliases.1 += 1;
|
||||
}
|
||||
if !metadata.is_file() || metadata.len() > remaining as u64 {
|
||||
return Err("build output exceeds byte budget".into());
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
file.take(remaining as u64 + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.map_err(|error| error.to_string())?;
|
||||
remaining = remaining
|
||||
.checked_sub(bytes.len())
|
||||
.ok_or("build output exceeds byte budget")?;
|
||||
let path = entry.path();
|
||||
let name = path
|
||||
.strip_prefix(root)
|
||||
.map_err(|error| error.to_string())?
|
||||
.to_str()
|
||||
.ok_or("build output path is not UTF-8")?
|
||||
.to_owned();
|
||||
crate::workers::valid_path(&name)?;
|
||||
files.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
// node-gyp's COPY rule may hard-link two output names. Namespace teardown
|
||||
// precedes this read: accept aliases only when every link is accounted for
|
||||
// inside the output root, then publish independent immutable byte entries.
|
||||
#[cfg(unix)]
|
||||
if links
|
||||
.values()
|
||||
.any(|(expected, observed)| expected != observed)
|
||||
{
|
||||
return Err("build output has a hard-link alias outside its output root".into());
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn compare_metadata(
|
||||
prefix: &str,
|
||||
outputs: &BTreeMap<String, Vec<u8>>,
|
||||
wheel: &[u8],
|
||||
) -> Result<(), String> {
|
||||
crate::workers::valid_path(prefix)?;
|
||||
let distribution = PathBuf::from(prefix)
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or("invalid prepared metadata directory")?
|
||||
.to_owned();
|
||||
let wheel = archive_files("whl", wheel)?
|
||||
.into_iter()
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let prefix = format!("{prefix}/");
|
||||
let mut count = 0;
|
||||
for (path, bytes) in outputs {
|
||||
if let Some(relative) = path.strip_prefix(&prefix) {
|
||||
count += 1;
|
||||
if wheel.get(&format!("{distribution}/{relative}")) != Some(bytes) {
|
||||
return Err(
|
||||
"built wheel metadata differs from prepare_metadata_for_build_wheel".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if count == 0 {
|
||||
return Err("metadata hook produced an empty directory".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
//! Serialize dependency mutations from the first project read to publication.
|
||||
//! The guard file is stable: replacing dever.lock must not replace its mutex.
|
||||
|
||||
use std::fs::{self, File, Metadata, OpenOptions};
|
||||
use std::path::Path;
|
||||
|
||||
pub(crate) struct ProjectMutation {
|
||||
_file: File,
|
||||
}
|
||||
|
||||
impl ProjectMutation {
|
||||
pub(crate) fn for_command(command: &str, root: &Path) -> Result<Option<Self>, String> {
|
||||
if !matches!(command, "add" | "update" | "remove") {
|
||||
return Ok(None);
|
||||
}
|
||||
let directory = fs::symlink_metadata(root)
|
||||
.map_err(|error| format!("cannot inspect project mutation directory: {error}"))?;
|
||||
validate(&directory, true)?;
|
||||
let path = root.join(".dever-project.lock");
|
||||
if let Ok(metadata) = fs::symlink_metadata(&path) {
|
||||
validate(&metadata, false)?;
|
||||
}
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true).write(true).create(true).truncate(false);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options
|
||||
.mode(0o600)
|
||||
.custom_flags(rustix::fs::OFlags::NOFOLLOW.bits() as i32);
|
||||
}
|
||||
let file = options
|
||||
.open(&path)
|
||||
.map_err(|error| format!("cannot open project mutation lock: {error}"))?;
|
||||
let opened = file.metadata().map_err(|error| error.to_string())?;
|
||||
validate(&opened, false)?;
|
||||
let current = fs::symlink_metadata(&path).map_err(|error| error.to_string())?;
|
||||
validate(¤t, false)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
if opened.dev() != current.dev() || opened.ino() != current.ino() {
|
||||
return Err("project mutation lock changed while opening".into());
|
||||
}
|
||||
}
|
||||
fs2::FileExt::try_lock_exclusive(&file).map_err(|error| {
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock {
|
||||
"project dependency mutation is busy; retry after the current command finishes"
|
||||
.into()
|
||||
} else {
|
||||
format!("cannot acquire project mutation lock: {error}")
|
||||
}
|
||||
})?;
|
||||
Ok(Some(Self { _file: file }))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(metadata: &Metadata, directory: bool) -> Result<(), String> {
|
||||
if metadata.file_type().is_symlink()
|
||||
|| (directory && !metadata.is_dir())
|
||||
|| (!directory && !metadata.is_file())
|
||||
{
|
||||
return Err(
|
||||
"project mutation path must be a real directory or regular lock file, without symlinks"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let forbidden = if directory { 0o022 } else { 0o077 };
|
||||
if metadata.uid() != rustix::process::geteuid().as_raw()
|
||||
|| metadata.mode() & forbidden != 0
|
||||
|| (!directory && metadata.nlink() != 1)
|
||||
{
|
||||
return Err("project mutation directory must be owned and not group/other writable; lock must be owned, private and unlinked elsewhere".into());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,995 @@
|
||||
//! npm package instances follow Node's ancestor lookup, not one global version
|
||||
//! per name. Archives remain content-addressed and shared between instances.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet, VecDeque};
|
||||
|
||||
use node_semver::{Range, Version};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{Ecosystem, LibSpec, LockFile, LockedLib, RegistryRuntime, archive_files};
|
||||
|
||||
mod resolve;
|
||||
pub(super) use resolve::resolve;
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Kind {
|
||||
Dependency,
|
||||
Optional,
|
||||
Peer,
|
||||
OptionalPeer,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Omission {
|
||||
Target,
|
||||
Unavailable,
|
||||
Conflict,
|
||||
OptionalPeerAbsent,
|
||||
BuildFailure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Edge {
|
||||
pub name: String,
|
||||
pub requirement: String,
|
||||
pub kind: Kind,
|
||||
pub target: Option<String>,
|
||||
pub omission: Option<Omission>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Source {
|
||||
pub archive: LibSpec,
|
||||
pub prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Instance {
|
||||
pub path: String,
|
||||
pub spec: LibSpec,
|
||||
pub source: Source,
|
||||
/// Exact bundled layout of a root archive, including hoisted transitives.
|
||||
pub bundled: BTreeMap<String, LibSpec>,
|
||||
pub edges: Vec<Edge>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Environment {
|
||||
pub roots: Vec<LibSpec>,
|
||||
pub instances: Vec<Instance>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub build: Option<String>,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
/// Explicit single-package graph for author fixtures with no dependencies.
|
||||
/// Materialization still checks package.json against this graph.
|
||||
pub fn single(spec: LibSpec) -> Self {
|
||||
Self {
|
||||
build: None,
|
||||
roots: vec![spec.clone()],
|
||||
instances: vec![Instance {
|
||||
path: child("", &spec.name),
|
||||
spec: spec.clone(),
|
||||
source: Source {
|
||||
archive: spec,
|
||||
prefix: "package/".into(),
|
||||
},
|
||||
bundled: BTreeMap::new(),
|
||||
edges: Vec::new(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn roots(specs: &[LibSpec]) -> Vec<LibSpec> {
|
||||
let mut roots = specs
|
||||
.iter()
|
||||
.filter(|spec| spec.ecosystem == Ecosystem::Npm)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
roots.sort();
|
||||
roots.dedup();
|
||||
roots
|
||||
}
|
||||
|
||||
pub fn environment<'a>(
|
||||
lock: &'a LockFile,
|
||||
requested: &[LibSpec],
|
||||
) -> Result<Option<&'a Environment>, String> {
|
||||
let requested = roots(requested);
|
||||
if requested.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
lock.npm
|
||||
.iter()
|
||||
.find(|env| env.roots == requested)
|
||||
.map(Some)
|
||||
.ok_or("npm Worker roots have no locked installation environment".into())
|
||||
}
|
||||
|
||||
pub fn archive_roots(lock: &LockFile, requested: &[LibSpec]) -> Result<BTreeSet<LibSpec>, String> {
|
||||
let requested = roots(requested);
|
||||
if requested.is_empty() {
|
||||
return Ok(BTreeSet::new());
|
||||
}
|
||||
if let Some(environment) = lock
|
||||
.npm
|
||||
.iter()
|
||||
.find(|environment| environment.roots == requested)
|
||||
{
|
||||
return Ok(environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| node.source.archive.clone())
|
||||
.collect());
|
||||
}
|
||||
// Built-in metadata fixtures are not registry packages and have no tree.
|
||||
let fixtures = super::super::FixtureRegistry::builtin();
|
||||
if requested
|
||||
.iter()
|
||||
.all(|spec| fixtures.package(spec).is_some())
|
||||
{
|
||||
return Ok(requested.into_iter().collect());
|
||||
}
|
||||
Err("npm roots have no exact locked installation environment".into())
|
||||
}
|
||||
|
||||
pub fn retain(lock: &LockFile, groups: &[Vec<LibSpec>]) -> Result<Vec<Environment>, String> {
|
||||
let mut environments = Vec::new();
|
||||
for group in groups {
|
||||
let requested = roots(group);
|
||||
if requested.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let original = lock
|
||||
.npm
|
||||
.iter()
|
||||
.find(|env| env.roots == requested)
|
||||
.or_else(|| {
|
||||
lock.npm
|
||||
.iter()
|
||||
.find(|env| requested.iter().all(|root| env.roots.contains(root)))
|
||||
});
|
||||
let Some(original) = original else {
|
||||
archive_roots(lock, group)?;
|
||||
continue;
|
||||
};
|
||||
let mut queue = requested
|
||||
.iter()
|
||||
.map(|spec| child("", &spec.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut paths = BTreeSet::new();
|
||||
while let Some(path) = queue.pop_front() {
|
||||
if !paths.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = original
|
||||
.instances
|
||||
.iter()
|
||||
.find(|node| node.path == path)
|
||||
.ok_or("npm retained graph lacks a dependency instance")?;
|
||||
queue.extend(node.edges.iter().filter_map(|edge| edge.target.clone()));
|
||||
}
|
||||
environments.push(Environment {
|
||||
build: original.build.clone(),
|
||||
roots: requested,
|
||||
instances: original
|
||||
.instances
|
||||
.iter()
|
||||
.filter(|node| paths.contains(&node.path))
|
||||
.cloned()
|
||||
.collect(),
|
||||
});
|
||||
}
|
||||
normalize(&mut environments)?;
|
||||
Ok(environments)
|
||||
}
|
||||
|
||||
pub(super) fn child(parent: &str, name: &str) -> String {
|
||||
if parent.is_empty() {
|
||||
format!("node_modules/{name}")
|
||||
} else {
|
||||
format!("{parent}/node_modules/{name}")
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn parent(path: &str) -> &str {
|
||||
path.rsplit_once("/node_modules/")
|
||||
.map_or("", |(parent, _)| parent)
|
||||
}
|
||||
|
||||
fn validate_path(path: &str) -> Result<(), String> {
|
||||
if path.len() > 4096 {
|
||||
return Err("npm instance path exceeds its bound".into());
|
||||
}
|
||||
let path = path
|
||||
.strip_prefix("node_modules/")
|
||||
.ok_or("invalid npm instance path")?;
|
||||
for name in path.split("/node_modules/") {
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn visible<'a>(
|
||||
instances: &'a BTreeMap<String, Instance>,
|
||||
consumer: &str,
|
||||
name: &str,
|
||||
peer: bool,
|
||||
) -> Option<&'a Instance> {
|
||||
let mut scope = if peer { parent(consumer) } else { consumer };
|
||||
loop {
|
||||
if let Some(found) = instances.get(&child(scope, name)) {
|
||||
return Some(found);
|
||||
}
|
||||
if scope.is_empty() {
|
||||
return None;
|
||||
}
|
||||
scope = parent(scope);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(super) struct Metadata {
|
||||
pub spec: LibSpec,
|
||||
pub edges: Vec<Edge>,
|
||||
pub bundles: BTreeSet<String>,
|
||||
pub document: Value,
|
||||
}
|
||||
|
||||
fn requirements(document: &Value, field: &str) -> Result<BTreeMap<String, String>, String> {
|
||||
let Some(value) = document.get(field) else {
|
||||
return Ok(BTreeMap::new());
|
||||
};
|
||||
value
|
||||
.as_object()
|
||||
.ok_or_else(|| format!("npm {field} must be an object"))?
|
||||
.iter()
|
||||
.map(|(name, value)| {
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
let rule = value.as_str().ok_or("npm dependency range must be text")?;
|
||||
rule.parse::<Range>()
|
||||
.map_err(|error| format!("invalid npm range for {name}: {error}"))?;
|
||||
Ok((name.clone(), rule.into()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Metadata {
|
||||
pub(super) fn parse(document: Value) -> Result<Self, String> {
|
||||
let spec: LibSpec = format!(
|
||||
"npm:{}@{}",
|
||||
super::field(&document, "name")?,
|
||||
super::field(&document, "version")?
|
||||
)
|
||||
.parse()?;
|
||||
let mut regular = requirements(&document, "dependencies")?;
|
||||
let optional = requirements(&document, "optionalDependencies")?;
|
||||
regular.retain(|name, _| !optional.contains_key(name));
|
||||
let mut peers = requirements(&document, "peerDependencies")?;
|
||||
let peer_meta = document
|
||||
.get("peerDependenciesMeta")
|
||||
.map(|value| {
|
||||
value
|
||||
.as_object()
|
||||
.ok_or("npm peerDependenciesMeta must be an object")
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(meta) = peer_meta {
|
||||
for (name, value) in meta {
|
||||
if !peers.contains_key(name)
|
||||
|| !value.is_object()
|
||||
|| value.get("optional").is_some_and(|flag| !flag.is_boolean())
|
||||
{
|
||||
return Err("invalid npm optional peer metadata".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
// npm has one outgoing edge per name: optional > production > peer.
|
||||
peers.retain(|name, _| !regular.contains_key(name) && !optional.contains_key(name));
|
||||
let mut edges = Vec::new();
|
||||
for (entries, kind) in [
|
||||
(regular.clone(), Kind::Dependency),
|
||||
(optional.clone(), Kind::Optional),
|
||||
(peers, Kind::Peer),
|
||||
] {
|
||||
for (name, requirement) in entries {
|
||||
let kind = if kind == Kind::Peer
|
||||
&& peer_meta
|
||||
.and_then(|meta| meta.get(&name))
|
||||
.and_then(|value| value.get("optional"))
|
||||
.and_then(Value::as_bool)
|
||||
== Some(true)
|
||||
{
|
||||
Kind::OptionalPeer
|
||||
} else {
|
||||
kind.clone()
|
||||
};
|
||||
edges.push(Edge {
|
||||
name,
|
||||
requirement,
|
||||
kind,
|
||||
target: None,
|
||||
omission: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
edges.sort();
|
||||
let bundles = |value: &Value| -> Result<BTreeSet<String>, String> {
|
||||
match value {
|
||||
Value::Bool(true) => Ok(regular.keys().chain(optional.keys()).cloned().collect()),
|
||||
Value::Bool(false) => Ok(BTreeSet::new()),
|
||||
Value::Array(values) => values
|
||||
.iter()
|
||||
.map(|value| {
|
||||
let name = value
|
||||
.as_str()
|
||||
.ok_or("npm bundled dependency name must be text")?;
|
||||
super::super::validate_name(&Ecosystem::Npm, name)?;
|
||||
if !regular.contains_key(name) && !optional.contains_key(name) {
|
||||
return Err(
|
||||
"npm bundled dependency is not a declared dependency".into()
|
||||
);
|
||||
}
|
||||
Ok(name.to_owned())
|
||||
})
|
||||
.collect(),
|
||||
_ => Err("npm bundleDependencies must be an array or boolean".into()),
|
||||
}
|
||||
};
|
||||
let first = document
|
||||
.get("bundleDependencies")
|
||||
.map(bundles)
|
||||
.transpose()?;
|
||||
let second = document
|
||||
.get("bundledDependencies")
|
||||
.map(bundles)
|
||||
.transpose()?;
|
||||
if first.is_some() && second.is_some() && first != second {
|
||||
return Err("npm bundled dependency aliases disagree".into());
|
||||
}
|
||||
Ok(Self {
|
||||
spec,
|
||||
edges,
|
||||
bundles: first.or(second).unwrap_or_default(),
|
||||
document,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn eligible(&self, runtime: &RegistryRuntime) -> Result<bool, String> {
|
||||
let (os, cpu) = runtime
|
||||
.target
|
||||
.split_once('-')
|
||||
.ok_or("npm runtime target lacks platform identity")?;
|
||||
let os = match os {
|
||||
"windows" => "win32",
|
||||
"macos" => "darwin",
|
||||
other => other,
|
||||
};
|
||||
let cpu = match cpu {
|
||||
"x86_64" => "x64",
|
||||
"aarch64" => "arm64",
|
||||
other => other,
|
||||
};
|
||||
for (field, selected) in [
|
||||
("os", Some(os)),
|
||||
("cpu", Some(cpu)),
|
||||
("libc", runtime.npm_libc.as_deref()),
|
||||
] {
|
||||
if field == "libc" && os != "linux" {
|
||||
continue;
|
||||
}
|
||||
let Some(value) = self.document.get(field) else {
|
||||
continue;
|
||||
};
|
||||
let values = value
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("npm {field} must be an array"))?;
|
||||
let values = values
|
||||
.iter()
|
||||
.map(|value| value.as_str().ok_or("npm platform selector must be text"))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let Some(selected) = selected else {
|
||||
return Ok(false);
|
||||
};
|
||||
if values
|
||||
.iter()
|
||||
.any(|value| value.strip_prefix('!') == Some(selected))
|
||||
|| (values.iter().any(|value| !value.starts_with('!'))
|
||||
&& !values.contains(&selected)
|
||||
&& !values.contains(&"any"))
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Archive {
|
||||
pub files: Vec<(String, Vec<u8>)>,
|
||||
pub packages: BTreeMap<String, Metadata>,
|
||||
}
|
||||
|
||||
impl Archive {
|
||||
pub(super) fn bundled(&self) -> BTreeMap<String, LibSpec> {
|
||||
self.packages
|
||||
.iter()
|
||||
.filter(|(prefix, _)| prefix.as_str() != "package/")
|
||||
.map(|(prefix, metadata)| {
|
||||
(
|
||||
prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/')
|
||||
.into(),
|
||||
metadata.spec.clone(),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(super) fn parse(spec: &LibSpec, bytes: &[u8]) -> Result<Self, String> {
|
||||
let files = archive_files("tgz", bytes)?;
|
||||
let mut packages = BTreeMap::<String, Metadata>::new();
|
||||
for (path, bytes) in &files {
|
||||
let relative = path
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm archive must have a package/ root")?;
|
||||
if relative == "package.json" || relative.ends_with("/package.json") {
|
||||
let prefix = path.strip_suffix("package.json").unwrap();
|
||||
if prefix != "package/" {
|
||||
let nested = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
if validate_path(nested).is_err() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let document = super::parse_json(bytes)?;
|
||||
let metadata = Metadata::parse(document)?;
|
||||
if prefix == "package/" {
|
||||
if metadata.spec != *spec {
|
||||
return Err(
|
||||
"npm package.json identity differs from its registry archive".into(),
|
||||
);
|
||||
}
|
||||
} else if !prefix.ends_with(&format!("node_modules/{}/", metadata.spec.name)) {
|
||||
return Err("npm bundled package identity differs from its path".into());
|
||||
}
|
||||
packages.insert(prefix.into(), metadata);
|
||||
}
|
||||
}
|
||||
let root = packages
|
||||
.get("package/")
|
||||
.ok_or("npm archive lacks package.json")?;
|
||||
for name in &root.bundles {
|
||||
if !packages.contains_key(&format!("package/node_modules/{name}/")) {
|
||||
return Err("npm declared bundled dependency is absent from its archive".into());
|
||||
}
|
||||
}
|
||||
for (path, _) in &files {
|
||||
if !packages.contains_key(&source_owner(path)?) {
|
||||
return Err("npm archive has unowned node_modules content".into());
|
||||
}
|
||||
}
|
||||
let nodes = packages
|
||||
.iter()
|
||||
.filter(|(prefix, _)| prefix.as_str() != "package/")
|
||||
.map(|(prefix, metadata)| {
|
||||
let path = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/')
|
||||
.to_owned();
|
||||
(
|
||||
path.clone(),
|
||||
Instance {
|
||||
path,
|
||||
spec: metadata.spec.clone(),
|
||||
source: Source {
|
||||
archive: spec.clone(),
|
||||
prefix: prefix.clone(),
|
||||
},
|
||||
bundled: BTreeMap::new(),
|
||||
edges: Vec::new(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut pending = root
|
||||
.bundles
|
||||
.iter()
|
||||
.map(|name| child("", name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = nodes
|
||||
.get(&path)
|
||||
.ok_or("npm bundle references a missing package")?;
|
||||
for edge in &packages[&node.source.prefix].edges {
|
||||
let peer = matches!(edge.kind, Kind::Peer | Kind::OptionalPeer);
|
||||
if let Some(found) = visible(&nodes, &path, &edge.name, peer) {
|
||||
if !Version::from_str(&found.spec.version)
|
||||
.map_err(|error| error.to_string())?
|
||||
.satisfies(
|
||||
&edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| error.to_string())?,
|
||||
)
|
||||
{
|
||||
return Err("npm bundled dependency has an incompatible version".into());
|
||||
}
|
||||
pending.push_back(found.path.clone());
|
||||
} else if edge.kind == Kind::Dependency
|
||||
&& edge.name != spec.name
|
||||
&& !root.edges.iter().any(|root| root.name == edge.name)
|
||||
{
|
||||
return Err("npm bundled dependency closure is incomplete".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
if reached.len() != nodes.len() {
|
||||
return Err("npm archive contains undeclared or unreachable bundled packages".into());
|
||||
}
|
||||
Ok(Self { files, packages })
|
||||
}
|
||||
}
|
||||
|
||||
fn source_owner(path: &str) -> Result<String, String> {
|
||||
let Some((parent, tail)) = path.rsplit_once("/node_modules/") else {
|
||||
return Ok("package/".into());
|
||||
};
|
||||
let mut parts = tail.split('/');
|
||||
let first = parts.next().ok_or("npm archive has invalid package path")?;
|
||||
let name = if first.starts_with('@') {
|
||||
format!(
|
||||
"{first}/{}",
|
||||
parts
|
||||
.next()
|
||||
.ok_or("npm scoped package path is incomplete")?
|
||||
)
|
||||
} else {
|
||||
first.into()
|
||||
};
|
||||
let prefix = format!("{parent}/node_modules/{name}");
|
||||
validate_path(
|
||||
prefix
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm bundled archive has invalid root")?,
|
||||
)?;
|
||||
Ok(format!("{prefix}/"))
|
||||
}
|
||||
|
||||
fn validate_source_owner(environment: &Environment, node: &Instance) -> Result<(), String> {
|
||||
for owner in &environment.instances {
|
||||
for (relative, spec) in &owner.bundled {
|
||||
if node.path == format!("{}/{relative}", owner.path)
|
||||
&& (node.spec != *spec
|
||||
|| node.source.archive != owner.source.archive
|
||||
|| node.source.prefix != format!("package/{relative}/"))
|
||||
{
|
||||
return Err(
|
||||
"npm bundled instance is not owned by its parent archive installation".into(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if node.source.prefix == "package/" {
|
||||
for (relative, spec) in &node.bundled {
|
||||
validate_path(relative)?;
|
||||
if spec.ecosystem != Ecosystem::Npm
|
||||
|| !relative.ends_with(&format!("node_modules/{}", spec.name))
|
||||
{
|
||||
return Err("invalid npm bundled layout identity".into());
|
||||
}
|
||||
super::super::validate_name(&Ecosystem::Npm, &spec.name)?;
|
||||
super::super::validate_version(&Ecosystem::Npm, &spec.version)?;
|
||||
}
|
||||
return if node.source.archive == node.spec {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("npm root archive identity differs from its instance".into())
|
||||
};
|
||||
}
|
||||
let relative = node
|
||||
.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.and_then(|prefix| prefix.strip_suffix('/'))
|
||||
.ok_or("invalid npm bundled source prefix")?;
|
||||
validate_path(relative)?;
|
||||
if !node.bundled.is_empty()
|
||||
|| !environment.instances.iter().any(|owner| {
|
||||
owner.source.prefix == "package/"
|
||||
&& owner.source.archive == node.source.archive
|
||||
&& owner.spec == node.source.archive
|
||||
&& node.path == format!("{}/{relative}", owner.path)
|
||||
&& owner.bundled.get(relative) == Some(&node.spec)
|
||||
})
|
||||
{
|
||||
return Err("npm bundled instance is not owned by its parent archive installation".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate immutable manifests and plan borrowed files before cloning any
|
||||
/// installation output. Reused archives must not multiply past the tree budget.
|
||||
pub fn files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
source_files(environment, lock, archive, false)
|
||||
}
|
||||
|
||||
pub fn build_files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
source_files(environment, lock, archive, true)
|
||||
}
|
||||
|
||||
pub(crate) fn build_executables(
|
||||
environment: &Environment,
|
||||
archives: &BTreeMap<LibSpec, Vec<u8>>,
|
||||
) -> Result<BTreeSet<String>, String> {
|
||||
let modes = archives
|
||||
.iter()
|
||||
.map(|(spec, bytes)| Ok((spec, super::archive_executables("tgz", bytes)?)))
|
||||
.collect::<Result<BTreeMap<_, _>, String>>()?;
|
||||
let mut installed = BTreeSet::new();
|
||||
for node in &environment.instances {
|
||||
for path in modes
|
||||
.get(&node.source.archive)
|
||||
.ok_or("npm source modes missing")?
|
||||
{
|
||||
if source_owner(path)? == node.source.prefix {
|
||||
let relative = path
|
||||
.strip_prefix(&node.source.prefix)
|
||||
.ok_or("npm source mode outside owner")?;
|
||||
installed.insert(format!("{}/{relative}", node.path));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(installed)
|
||||
}
|
||||
|
||||
pub(crate) fn installed_owner(path: &str) -> Result<String, String> {
|
||||
source_owner(&format!("package/{path}"))?
|
||||
.strip_prefix("package/")
|
||||
.map(|owner| owner.trim_end_matches('/').into())
|
||||
.ok_or("npm output is outside its installation".into())
|
||||
}
|
||||
|
||||
fn source_files<'a>(
|
||||
environment: &Environment,
|
||||
lock: &LockFile,
|
||||
mut archive: impl FnMut(&LockedLib) -> Result<&'a [u8], String>,
|
||||
preparing: bool,
|
||||
) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
let mut archives = BTreeMap::new();
|
||||
let mut archive_bytes = 0usize;
|
||||
for node in &environment.instances {
|
||||
validate_source_owner(environment, node)?;
|
||||
if archives.contains_key(&node.source.archive) {
|
||||
continue;
|
||||
}
|
||||
let lib = lock
|
||||
.libs
|
||||
.iter()
|
||||
.find(|lib| lib.spec == node.source.archive)
|
||||
.ok_or("npm graph source archive is missing")?;
|
||||
let parsed = Archive::parse(&lib.spec, archive(lib)?)?;
|
||||
for (_, bytes) in &parsed.files {
|
||||
archive_bytes = archive_bytes
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm archive size overflow")?;
|
||||
if archive_bytes > crate::workers::MAX_TREE_BYTES {
|
||||
return Err("npm archive cache exceeds its tree byte budget".into());
|
||||
}
|
||||
}
|
||||
archives.insert(lib.spec.clone(), parsed);
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
let mut installed_bytes = 0usize;
|
||||
for node in &environment.instances {
|
||||
let source = &archives[&node.source.archive];
|
||||
if node.source.prefix == "package/" && node.bundled != source.bundled() {
|
||||
return Err("npm bundled layout differs from its parent archive".into());
|
||||
}
|
||||
let metadata = source
|
||||
.packages
|
||||
.get(&node.source.prefix)
|
||||
.ok_or("npm instance source package is absent")?;
|
||||
if metadata.spec != node.spec
|
||||
|| metadata.edges.len() != node.edges.len()
|
||||
|| metadata.edges.iter().any(|expected| {
|
||||
!node.edges.iter().any(|edge| {
|
||||
edge.name == expected.name
|
||||
&& edge.requirement == expected.requirement
|
||||
&& edge.kind == expected.kind
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm locked dependency graph differs from package.json".into());
|
||||
}
|
||||
if !preparing
|
||||
&& (metadata.document.get("gypfile").and_then(Value::as_bool) == Some(true)
|
||||
|| metadata
|
||||
.document
|
||||
.get("scripts")
|
||||
.and_then(Value::as_object)
|
||||
.is_some_and(|scripts| {
|
||||
["preinstall", "install", "postinstall"]
|
||||
.iter()
|
||||
.any(|name| scripts.contains_key(*name))
|
||||
}))
|
||||
{
|
||||
return Err(
|
||||
"managed npm Worker rejects lifecycle scripts and native build hooks".into(),
|
||||
);
|
||||
}
|
||||
for (path, bytes) in &source.files {
|
||||
if source_owner(path)? != node.source.prefix {
|
||||
continue;
|
||||
}
|
||||
let relative = path
|
||||
.strip_prefix(&node.source.prefix)
|
||||
.ok_or("npm file is outside its instance source")?;
|
||||
if !preparing && (relative.ends_with(".node") || relative == "binding.gyp") {
|
||||
return Err("managed npm Worker rejects native package content".into());
|
||||
}
|
||||
let installed = format!("{}/{relative}", node.path);
|
||||
installed_bytes = installed_bytes
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm installed size overflow")?;
|
||||
if installed_bytes > crate::workers::MAX_TREE_BYTES
|
||||
|| files.len() >= crate::workers::MAX_TREE_FILES
|
||||
{
|
||||
return Err("npm installation exceeds its tree file/byte budget".into());
|
||||
}
|
||||
if files.insert(installed, bytes).is_some() {
|
||||
return Err("npm installed files have conflicting owners".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(files
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| (path, bytes.clone()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn normalize(environments: &mut Vec<Environment>) -> Result<(), String> {
|
||||
for env in environments.iter_mut() {
|
||||
env.roots.sort();
|
||||
env.roots.dedup();
|
||||
for node in &mut env.instances {
|
||||
node.edges.sort();
|
||||
}
|
||||
env.instances
|
||||
.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
}
|
||||
environments.sort_by(|left, right| left.roots.cmp(&right.roots));
|
||||
for pair in environments.windows(2) {
|
||||
if pair[0].roots == pair[1].roots && pair[0] != pair[1] {
|
||||
return Err("npm roots have inconsistent installation environments".into());
|
||||
}
|
||||
}
|
||||
environments.dedup();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn requires_build(files: &[(String, Vec<u8>)]) -> Result<bool, String> {
|
||||
for (path, bytes) in files {
|
||||
if path.ends_with(".node") || path.ends_with("/binding.gyp") {
|
||||
return Ok(true);
|
||||
}
|
||||
if path.ends_with("/package.json") {
|
||||
let metadata: Value = serde_json::from_slice(bytes)
|
||||
.map_err(|error| format!("invalid npm installation metadata: {error}"))?;
|
||||
if metadata.get("gypfile").and_then(Value::as_bool) == Some(true)
|
||||
|| metadata
|
||||
.get("scripts")
|
||||
.and_then(Value::as_object)
|
||||
.is_some_and(|scripts| {
|
||||
["preinstall", "install", "postinstall"]
|
||||
.iter()
|
||||
.any(|name| scripts.contains_key(*name))
|
||||
})
|
||||
{
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Lifecycle output may add generated files, but cannot rewrite dependency
|
||||
/// identities, create undeclared package instances, or escape the locked tree.
|
||||
pub fn validate_built_files(
|
||||
environment: &Environment,
|
||||
files: &[(String, Vec<u8>)],
|
||||
) -> Result<(), String> {
|
||||
let nodes = environment
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| (node.path.as_str(), node))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
let mut manifests = BTreeSet::new();
|
||||
let mut names = BTreeSet::new();
|
||||
let mut total = 0usize;
|
||||
for (path, bytes) in files {
|
||||
crate::workers::valid_path(path)?;
|
||||
let owner = source_owner(&format!("package/{path}"))?;
|
||||
let owner = owner
|
||||
.strip_prefix("package/")
|
||||
.ok_or("npm output is outside its installation")?
|
||||
.trim_end_matches('/');
|
||||
let node = nodes
|
||||
.get(owner)
|
||||
.ok_or("npm build output introduces an unlocked instance")?;
|
||||
if !names.insert(path) {
|
||||
return Err("npm build output repeats a file".into());
|
||||
}
|
||||
total = total
|
||||
.checked_add(bytes.len())
|
||||
.ok_or("npm output byte overflow")?;
|
||||
if total > crate::workers::MAX_TREE_BYTES || names.len() > crate::workers::MAX_TREE_FILES {
|
||||
return Err("npm output exceeds installation budget".into());
|
||||
}
|
||||
if path == &format!("{owner}/package.json") {
|
||||
let metadata =
|
||||
Metadata::parse(serde_json::from_slice(bytes).map_err(|error| error.to_string())?)?;
|
||||
if metadata.spec != node.spec
|
||||
|| metadata.edges.len() != node.edges.len()
|
||||
|| metadata.edges.iter().any(|expected| {
|
||||
!node.edges.iter().any(|edge| {
|
||||
edge.name == expected.name
|
||||
&& edge.requirement == expected.requirement
|
||||
&& edge.kind == expected.kind
|
||||
})
|
||||
})
|
||||
{
|
||||
return Err("npm lifecycle changed locked package identity or dependencies".into());
|
||||
}
|
||||
manifests.insert(owner.to_owned());
|
||||
}
|
||||
}
|
||||
if manifests.len() != nodes.len() {
|
||||
return Err("npm lifecycle removed a locked package manifest".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn doctor(lock: &LockFile) -> Result<(), String> {
|
||||
for env in &lock.npm {
|
||||
if env.roots.is_empty()
|
||||
|| env.roots != roots(&env.roots)
|
||||
|| env.instances.len() > super::MAX_PACKAGES
|
||||
{
|
||||
return Err("invalid npm installation environment roots or size".into());
|
||||
}
|
||||
let nodes = env
|
||||
.instances
|
||||
.iter()
|
||||
.map(|node| (node.path.clone(), node.clone()))
|
||||
.collect::<BTreeMap<_, _>>();
|
||||
if nodes.len() != env.instances.len() {
|
||||
return Err("duplicate npm instance path".into());
|
||||
}
|
||||
let mut pending = VecDeque::new();
|
||||
for root in &env.roots {
|
||||
let path = child("", &root.name);
|
||||
if nodes.get(&path).map(|node| &node.spec) != Some(root) {
|
||||
return Err("npm root is not installed at its exact path".into());
|
||||
}
|
||||
pending.push_back(path);
|
||||
}
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let node = nodes
|
||||
.get(&path)
|
||||
.ok_or("npm edge references a missing instance")?;
|
||||
validate_source_owner(env, node)?;
|
||||
validate_path(&node.path)?;
|
||||
super::super::validate_name(&Ecosystem::Npm, &node.spec.name)?;
|
||||
super::super::validate_version(&Ecosystem::Npm, &node.spec.version)?;
|
||||
if node.spec.ecosystem != Ecosystem::Npm
|
||||
|| !node
|
||||
.path
|
||||
.ends_with(&format!("node_modules/{}", node.spec.name))
|
||||
{
|
||||
return Err("npm instance identity differs from its path".into());
|
||||
}
|
||||
if !lock
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec == node.source.archive && lib.spec.ecosystem == Ecosystem::Npm)
|
||||
{
|
||||
return Err("npm instance source archive is not locked".into());
|
||||
}
|
||||
if node.source.prefix != "package/" {
|
||||
validate_path(
|
||||
node.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.and_then(|path| path.strip_suffix('/'))
|
||||
.ok_or("invalid npm bundled source prefix")?,
|
||||
)?;
|
||||
} else if node.source.archive != node.spec {
|
||||
return Err("npm root archive identity differs from its instance".into());
|
||||
}
|
||||
let mut unique = BTreeSet::new();
|
||||
for edge in &node.edges {
|
||||
if !unique.insert((&edge.name, &edge.kind)) {
|
||||
return Err("duplicate npm dependency edge".into());
|
||||
}
|
||||
let range = edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let actual = visible(
|
||||
&nodes,
|
||||
&path,
|
||||
&edge.name,
|
||||
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
);
|
||||
match (&edge.target, &edge.omission) {
|
||||
(Some(target), None)
|
||||
if actual.is_some_and(|found| {
|
||||
found.path == *target
|
||||
&& Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&range))
|
||||
}) =>
|
||||
{
|
||||
pending.push_back(target.clone());
|
||||
}
|
||||
(None, Some(reason))
|
||||
if actual.is_none()
|
||||
&& (edge.kind == Kind::Optional
|
||||
&& *reason != Omission::OptionalPeerAbsent
|
||||
|| edge.kind == Kind::OptionalPeer
|
||||
&& matches!(
|
||||
reason,
|
||||
Omission::OptionalPeerAbsent | Omission::BuildFailure
|
||||
))
|
||||
&& (*reason != Omission::BuildFailure || env.build.is_some()) => {}
|
||||
_ => {
|
||||
return Err(
|
||||
"npm edge differs from Node lookup or its required range".into()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if reached.len() != nodes.len() {
|
||||
return Err("npm environment contains unreachable package instances".into());
|
||||
}
|
||||
}
|
||||
for worker in &lock.workers {
|
||||
if worker.ecosystem == "npm" {
|
||||
environment(lock, &worker.libs)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
use std::str::FromStr;
|
||||
@@ -0,0 +1,436 @@
|
||||
use super::*;
|
||||
use crate::libs::registry::{MAX_PACKAGES, MAX_RESOLUTION_ATTEMPTS, RegistryResolver, SearchError};
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Request {
|
||||
consumer: Option<String>,
|
||||
edge: Edge,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct State {
|
||||
nodes: BTreeMap<String, Instance>,
|
||||
pending: VecDeque<Request>,
|
||||
}
|
||||
|
||||
struct Catalog<'a, 'r> {
|
||||
resolver: &'a RegistryResolver<'r>,
|
||||
runtime: &'a RegistryRuntime,
|
||||
indexes: BTreeMap<String, Vec<String>>,
|
||||
metadata: BTreeMap<(String, String), Value>,
|
||||
archives: BTreeMap<LibSpec, (LockedLib, Archive)>,
|
||||
attempts: usize,
|
||||
archive_bytes: usize,
|
||||
}
|
||||
|
||||
pub(crate) fn resolve(
|
||||
resolver: &RegistryResolver<'_>,
|
||||
requested: &[LibSpec],
|
||||
runtime: &RegistryRuntime,
|
||||
) -> Result<(Vec<LockedLib>, Environment), String> {
|
||||
let roots = roots(requested);
|
||||
let mut state = State::default();
|
||||
for spec in &roots {
|
||||
state.pending.push_back(Request {
|
||||
consumer: None,
|
||||
edge: Edge {
|
||||
name: spec.name.clone(),
|
||||
requirement: format!("={}", spec.version),
|
||||
kind: Kind::Dependency,
|
||||
target: None,
|
||||
omission: None,
|
||||
},
|
||||
});
|
||||
}
|
||||
let mut catalog = Catalog {
|
||||
resolver,
|
||||
runtime,
|
||||
indexes: BTreeMap::new(),
|
||||
metadata: BTreeMap::new(),
|
||||
archives: BTreeMap::new(),
|
||||
attempts: 0,
|
||||
archive_bytes: 0,
|
||||
};
|
||||
let mut state = catalog.search(state).map_err(SearchError::message)?;
|
||||
let mut pending = roots
|
||||
.iter()
|
||||
.map(|root| child("", &root.name))
|
||||
.collect::<VecDeque<_>>();
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
pending.extend(
|
||||
state.nodes[&path]
|
||||
.edges
|
||||
.iter()
|
||||
.filter_map(|edge| edge.target.clone()),
|
||||
);
|
||||
}
|
||||
state.nodes.retain(|path, _| reached.contains(path));
|
||||
let used = state
|
||||
.nodes
|
||||
.values()
|
||||
.map(|node| node.source.archive.clone())
|
||||
.collect::<BTreeSet<_>>();
|
||||
let libraries = used
|
||||
.into_iter()
|
||||
.map(|spec| {
|
||||
catalog
|
||||
.archives
|
||||
.remove(&spec)
|
||||
.expect("resolved npm archive")
|
||||
.0
|
||||
})
|
||||
.collect();
|
||||
Ok((
|
||||
libraries,
|
||||
Environment {
|
||||
build: None,
|
||||
roots,
|
||||
instances: state.nodes.into_values().collect(),
|
||||
},
|
||||
))
|
||||
}
|
||||
|
||||
impl Catalog<'_, '_> {
|
||||
fn archive(&mut self, spec: &LibSpec) -> Result<(), SearchError> {
|
||||
if self.archives.contains_key(spec) {
|
||||
return Ok(());
|
||||
}
|
||||
let document = self
|
||||
.resolver
|
||||
.registry_metadata(
|
||||
Ecosystem::Npm,
|
||||
&spec.name,
|
||||
&spec.version,
|
||||
&mut self.metadata,
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
let bytes = self
|
||||
.resolver
|
||||
.npm_archive(spec, &document)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
let archive = Archive::parse(spec, &bytes).map_err(SearchError::Fatal)?;
|
||||
self.archive_bytes = self
|
||||
.archive_bytes
|
||||
.checked_add(
|
||||
archive
|
||||
.files
|
||||
.iter()
|
||||
.map(|(_, bytes)| bytes.len())
|
||||
.sum::<usize>(),
|
||||
)
|
||||
.ok_or_else(|| SearchError::Fatal("npm archive cache size overflow".into()))?;
|
||||
if self.archive_bytes > 256 * 1024 * 1024 || self.archives.len() >= MAX_PACKAGES {
|
||||
return Err(SearchError::Fatal(
|
||||
"npm archive cache exceeds its 256 MiB/package budget".into(),
|
||||
));
|
||||
}
|
||||
let lib = self
|
||||
.resolver
|
||||
.lock_archive(
|
||||
spec,
|
||||
Vec::new(),
|
||||
self.runtime,
|
||||
"tgz",
|
||||
&bytes,
|
||||
"npm-instances-v1",
|
||||
)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.archives.insert(spec.clone(), (lib, archive));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn metadata(&self, node: &Instance) -> &Metadata {
|
||||
&self.archives[&node.source.archive].1.packages[&node.source.prefix]
|
||||
}
|
||||
|
||||
fn bind(
|
||||
state: &mut State,
|
||||
request: &Request,
|
||||
target: Option<String>,
|
||||
omission: Option<Omission>,
|
||||
) {
|
||||
if let Some(consumer) = &request.consumer {
|
||||
let edge = state
|
||||
.nodes
|
||||
.get_mut(consumer)
|
||||
.expect("pending npm consumer")
|
||||
.edges
|
||||
.iter_mut()
|
||||
.find(|edge| edge.name == request.edge.name && edge.kind == request.edge.kind)
|
||||
.expect("pending npm edge");
|
||||
edge.target = target;
|
||||
edge.omission = omission;
|
||||
}
|
||||
}
|
||||
|
||||
fn add(&self, state: &mut State, path: &str, spec: &LibSpec) -> Result<(), SearchError> {
|
||||
for (prefix, metadata) in &self.archives[spec].1.packages {
|
||||
let relative = prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
let location = if relative.is_empty() {
|
||||
path.into()
|
||||
} else {
|
||||
format!("{path}/{relative}")
|
||||
};
|
||||
let instance = Instance {
|
||||
path: location.clone(),
|
||||
spec: metadata.spec.clone(),
|
||||
source: Source {
|
||||
archive: spec.clone(),
|
||||
prefix: prefix.clone(),
|
||||
},
|
||||
bundled: if relative.is_empty() {
|
||||
self.archives[spec].1.bundled()
|
||||
} else {
|
||||
BTreeMap::new()
|
||||
},
|
||||
edges: metadata.edges.clone(),
|
||||
};
|
||||
if state.nodes.insert(location.clone(), instance).is_some() {
|
||||
return Err(SearchError::Conflict(
|
||||
"npm package installation paths collide".into(),
|
||||
));
|
||||
}
|
||||
for edge in &metadata.edges {
|
||||
if edge.kind != Kind::OptionalPeer {
|
||||
state.pending.push_back(Request {
|
||||
consumer: Some(location.clone()),
|
||||
edge: edge.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prune_bundled_orphans(&self, state: &mut State, root: &str) {
|
||||
let mut pending = VecDeque::from([root.to_owned()]);
|
||||
let mut reached = BTreeSet::new();
|
||||
while let Some(path) = pending.pop_front() {
|
||||
if !reached.insert(path.clone()) {
|
||||
continue;
|
||||
}
|
||||
let Some(node) = state.nodes.get(&path) else {
|
||||
continue;
|
||||
};
|
||||
for edge in &self.metadata(node).edges {
|
||||
if let Some(found) = visible(
|
||||
&state.nodes,
|
||||
&path,
|
||||
&edge.name,
|
||||
matches!(edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
) {
|
||||
pending.push_back(found.path.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let prefix = format!("{root}/");
|
||||
state.nodes.retain(|path, node| {
|
||||
node.source.prefix == "package/" || !path.starts_with(&prefix) || reached.contains(path)
|
||||
});
|
||||
}
|
||||
|
||||
fn search(&mut self, mut state: State) -> Result<State, SearchError> {
|
||||
self.attempts += 1;
|
||||
if self.attempts > MAX_RESOLUTION_ATTEMPTS
|
||||
|| state.nodes.len() > MAX_PACKAGES
|
||||
|| state.nodes.keys().any(|path| path.len() > 4096)
|
||||
{
|
||||
return Err(SearchError::Fatal(
|
||||
"npm instance graph exceeds its resolution budget".into(),
|
||||
));
|
||||
}
|
||||
while let Some(request) = state.pending.pop_front() {
|
||||
if request
|
||||
.consumer
|
||||
.as_ref()
|
||||
.is_some_and(|consumer| !state.nodes.contains_key(consumer))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let consumer = request.consumer.as_deref().unwrap_or("");
|
||||
let peer = request.edge.kind == Kind::Peer;
|
||||
let scope = if peer { parent(consumer) } else { consumer };
|
||||
let location = child(scope, &request.edge.name);
|
||||
let rule = request
|
||||
.edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| SearchError::Fatal(error.to_string()))?;
|
||||
let existing = visible(&state.nodes, consumer, &request.edge.name, peer).cloned();
|
||||
if let Some(found) = &existing {
|
||||
let eligible = self
|
||||
.metadata(found)
|
||||
.eligible(self.runtime)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
if !eligible
|
||||
&& request.edge.kind == Kind::Optional
|
||||
&& found.source.prefix != "package/"
|
||||
{
|
||||
let prefix = format!("{}/", found.path);
|
||||
state
|
||||
.nodes
|
||||
.retain(|path, _| path != &found.path && !path.starts_with(&prefix));
|
||||
let relative = found
|
||||
.source
|
||||
.prefix
|
||||
.strip_prefix("package/")
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
let owner = found
|
||||
.path
|
||||
.strip_suffix(relative)
|
||||
.unwrap()
|
||||
.trim_end_matches('/');
|
||||
self.prune_bundled_orphans(&mut state, owner);
|
||||
Self::bind(&mut state, &request, None, Some(Omission::Target));
|
||||
continue;
|
||||
}
|
||||
// Ordinary dependencies get a private nested slot. Reuse an
|
||||
// ancestor only to close a real package cycle, never a sibling's
|
||||
// independently resolved peer context.
|
||||
let bundled = state.nodes.get(consumer).is_some_and(|node| {
|
||||
node.source.prefix != "package/" && node.source.archive == found.source.archive
|
||||
});
|
||||
let reusable = peer
|
||||
|| bundled
|
||||
|| found.path == location
|
||||
|| consumer == found.path
|
||||
|| consumer.starts_with(&format!("{}/", found.path));
|
||||
if reusable
|
||||
&& eligible
|
||||
&& Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&rule))
|
||||
{
|
||||
Self::bind(&mut state, &request, Some(found.path.clone()), None);
|
||||
continue;
|
||||
}
|
||||
if found.path == location {
|
||||
return Err(SearchError::Conflict(format!(
|
||||
"npm dependency/peer conflict for {} at {location}",
|
||||
request.edge.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(node) = request
|
||||
.consumer
|
||||
.as_ref()
|
||||
.and_then(|consumer| state.nodes.get(consumer))
|
||||
&& node.source.prefix != "package/"
|
||||
&& !peer
|
||||
{
|
||||
if request.edge.kind == Kind::Optional && existing.is_none() {
|
||||
Self::bind(&mut state, &request, None, Some(Omission::Unavailable));
|
||||
continue;
|
||||
}
|
||||
return Err(SearchError::Fatal(
|
||||
"npm bundled dependency closure is incomplete".into(),
|
||||
));
|
||||
}
|
||||
let versions = if let Some(versions) = self.indexes.get(&request.edge.name) {
|
||||
versions.clone()
|
||||
} else {
|
||||
let versions = self
|
||||
.resolver
|
||||
.registry_versions(Ecosystem::Npm, &request.edge.name)
|
||||
.map_err(SearchError::Fatal)?;
|
||||
self.indexes
|
||||
.insert(request.edge.name.clone(), versions.clone());
|
||||
versions
|
||||
};
|
||||
let mut reason = Omission::Unavailable;
|
||||
for version in versions.into_iter().filter(|version| {
|
||||
Version::from_str(version).is_ok_and(|version| version.satisfies(&rule))
|
||||
}) {
|
||||
self.attempts += 1;
|
||||
if self.attempts > MAX_RESOLUTION_ATTEMPTS {
|
||||
return Err(SearchError::Fatal(
|
||||
"npm resolution exceeds its candidate budget".into(),
|
||||
));
|
||||
}
|
||||
let spec = LibSpec {
|
||||
ecosystem: Ecosystem::Npm,
|
||||
name: request.edge.name.clone(),
|
||||
version,
|
||||
};
|
||||
self.archive(&spec)?;
|
||||
if !self.archives[&spec].1.packages["package/"]
|
||||
.eligible(self.runtime)
|
||||
.map_err(SearchError::Fatal)?
|
||||
{
|
||||
reason = Omission::Target;
|
||||
continue;
|
||||
}
|
||||
reason = Omission::Conflict;
|
||||
let mut candidate = state.clone();
|
||||
self.add(&mut candidate, &location, &spec)?;
|
||||
Self::bind(&mut candidate, &request, Some(location.clone()), None);
|
||||
match self.search(candidate) {
|
||||
Ok(resolved) => return Ok(resolved),
|
||||
Err(SearchError::Conflict(_)) => {}
|
||||
Err(fatal) => return Err(fatal),
|
||||
}
|
||||
}
|
||||
if request.edge.kind == Kind::Optional && existing.is_none() {
|
||||
Self::bind(&mut state, &request, None, Some(reason));
|
||||
continue;
|
||||
}
|
||||
return Err(SearchError::Conflict(format!(
|
||||
"no npm version satisfies {} at {location}",
|
||||
request.edge.name
|
||||
)));
|
||||
}
|
||||
let edges = state
|
||||
.nodes
|
||||
.values()
|
||||
.flat_map(|node| {
|
||||
node.edges.iter().map(|edge| Request {
|
||||
consumer: Some(node.path.clone()),
|
||||
edge: edge.clone(),
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for request in edges {
|
||||
let found = visible(
|
||||
&state.nodes,
|
||||
request.consumer.as_deref().unwrap(),
|
||||
&request.edge.name,
|
||||
matches!(request.edge.kind, Kind::Peer | Kind::OptionalPeer),
|
||||
);
|
||||
if let Some(found) = found {
|
||||
let range = request
|
||||
.edge
|
||||
.requirement
|
||||
.parse::<Range>()
|
||||
.map_err(|error| SearchError::Fatal(error.to_string()))?;
|
||||
if !Version::from_str(&found.spec.version)
|
||||
.is_ok_and(|version| version.satisfies(&range))
|
||||
{
|
||||
return Err(SearchError::Conflict(
|
||||
"npm dependency or peer conflicts with final Node lookup".into(),
|
||||
));
|
||||
}
|
||||
let target = found.path.clone();
|
||||
Self::bind(&mut state, &request, Some(target), None);
|
||||
} else if request.edge.kind == Kind::OptionalPeer {
|
||||
Self::bind(
|
||||
&mut state,
|
||||
&request,
|
||||
None,
|
||||
Some(Omission::OptionalPeerAbsent),
|
||||
);
|
||||
} else if request.edge.kind != Kind::Optional || request.edge.omission.is_none() {
|
||||
return Err(SearchError::Conflict(
|
||||
"npm required dependency is absent from final Node lookup".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(state)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
//! Authenticated Go checksum database receipts. Trust comes from the verifier
|
||||
//! supplied by the caller, never from a key stored in a project's lock file.
|
||||
//! Formats follow golang.org/x/mod/sumdb/{note,tlog,dirhash} and RFC 6962.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Mutex, MutexGuard};
|
||||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use ring::signature::{ED25519, UnparsedPublicKey};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{RegistryTransport, go_path, zip_entries};
|
||||
|
||||
mod merkle;
|
||||
|
||||
pub const OFFICIAL_VERIFIER: &str =
|
||||
"sum.golang.org+033de0ae+Ac4zctda0e5eza+HJyk9SxEdh+s3Ux18htTTAD8OuAn8";
|
||||
const MAX_NOTE_BYTES: usize = 16 * 1024;
|
||||
const MAX_RECORDS: usize = 1024;
|
||||
const MAX_EVIDENCE_BYTES: usize = 16 * 1024 * 1024;
|
||||
type Hash = [u8; 32];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct Tree {
|
||||
size: u64,
|
||||
hash: Hash,
|
||||
}
|
||||
|
||||
fn encode_hash(hash: Hash) -> String {
|
||||
STANDARD.encode(hash)
|
||||
}
|
||||
|
||||
fn decode_hash(value: &str) -> Result<Hash, String> {
|
||||
STANDARD
|
||||
.decode(value)
|
||||
.map_err(|_| "sumdb hash is not base64")?
|
||||
.try_into()
|
||||
.map_err(|_| "sumdb hash must contain 32 bytes".into())
|
||||
}
|
||||
|
||||
/// Explicit verifier injection supports registry protocol tests and author
|
||||
/// tooling. Released project commands always select `official()`.
|
||||
#[derive(Clone)]
|
||||
pub struct Verifier {
|
||||
name: String,
|
||||
key_hash: [u8; 4],
|
||||
key: [u8; 32],
|
||||
}
|
||||
|
||||
impl Verifier {
|
||||
pub fn official() -> Self {
|
||||
Self::new(OFFICIAL_VERIFIER).expect("fixed official sumdb verifier")
|
||||
}
|
||||
|
||||
pub fn new(encoded: &str) -> Result<Self, String> {
|
||||
// '+' also occurs inside base64; only the first two separators delimit.
|
||||
let (name, rest) = encoded.split_once('+').ok_or("invalid sumdb verifier")?;
|
||||
let (key_hash, encoded_key) = rest.split_once('+').ok_or("invalid sumdb verifier")?;
|
||||
if name.is_empty() || name.chars().any(char::is_whitespace) || key_hash.len() != 8 {
|
||||
return Err("invalid sumdb verifier identity".into());
|
||||
}
|
||||
let expected = u32::from_str_radix(key_hash, 16)
|
||||
.map_err(|_| "invalid sumdb key hash")?
|
||||
.to_be_bytes();
|
||||
let key = STANDARD
|
||||
.decode(encoded_key)
|
||||
.map_err(|_| "invalid sumdb verifier encoding")?;
|
||||
if key.len() != 33 || key[0] != 1 {
|
||||
return Err("sumdb requires an Ed25519 verifier".into());
|
||||
}
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(name.as_bytes());
|
||||
hash.update(b"\n");
|
||||
hash.update(&key);
|
||||
if hash.finalize()[..4] != expected {
|
||||
return Err("sumdb verifier key hash mismatch".into());
|
||||
}
|
||||
Ok(Self {
|
||||
name: name.into(),
|
||||
key_hash: expected,
|
||||
key: key[1..].try_into().expect("validated Ed25519 key length"),
|
||||
})
|
||||
}
|
||||
|
||||
fn checkpoint(&self, note: &str) -> Result<Tree, String> {
|
||||
if note.len() > MAX_NOTE_BYTES || note.bytes().any(|byte| byte < 32 && byte != b'\n') {
|
||||
return Err("invalid sumdb checkpoint note".into());
|
||||
}
|
||||
let (text, signatures) = note
|
||||
.rsplit_once("\n\n")
|
||||
.ok_or("sumdb checkpoint lacks signatures")?;
|
||||
let message = format!("{text}\n");
|
||||
if !signatures.ends_with('\n') {
|
||||
return Err("unterminated sumdb signature".into());
|
||||
}
|
||||
let mut verified = false;
|
||||
for (index, line) in signatures.lines().enumerate() {
|
||||
if index >= 100 {
|
||||
return Err("sumdb checkpoint has too many signatures".into());
|
||||
}
|
||||
let (name, encoded) = line
|
||||
.strip_prefix("— ")
|
||||
.and_then(|line| line.split_once(' '))
|
||||
.ok_or("malformed sumdb signature")?;
|
||||
let signature = STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|_| "invalid sumdb signature encoding")?;
|
||||
if name.is_empty() || name.chars().any(char::is_whitespace) || signature.len() < 5 {
|
||||
return Err("malformed sumdb signature identity".into());
|
||||
}
|
||||
if name == self.name && signature[..4] == self.key_hash {
|
||||
UnparsedPublicKey::new(&ED25519, self.key)
|
||||
.verify(message.as_bytes(), &signature[4..])
|
||||
.map_err(|_| "sumdb checkpoint signature verification failed")?;
|
||||
verified = true;
|
||||
}
|
||||
}
|
||||
if !verified {
|
||||
return Err("sumdb checkpoint has no trusted signature".into());
|
||||
}
|
||||
let mut lines = text.lines();
|
||||
if lines.next() != Some("go.sum database tree") {
|
||||
return Err("unsupported sumdb checkpoint format".into());
|
||||
}
|
||||
let size_text = lines.next().ok_or("sumdb checkpoint lacks tree size")?;
|
||||
let size = decimal(size_text)?;
|
||||
let hash = decode_hash(lines.next().ok_or("sumdb checkpoint lacks root hash")?)?;
|
||||
let empty: Hash = Sha256::digest([]).into();
|
||||
if size == 0 && hash != empty {
|
||||
return Err("invalid sumdb empty root".into());
|
||||
}
|
||||
Ok(Tree { size, hash })
|
||||
}
|
||||
}
|
||||
|
||||
fn decimal(value: &str) -> Result<u64, String> {
|
||||
let number = value
|
||||
.parse::<u64>()
|
||||
.map_err(|_| "invalid sumdb record/tree number")?;
|
||||
if number > i64::MAX as u64 || value != number.to_string() {
|
||||
return Err("noncanonical sumdb record/tree number".into());
|
||||
}
|
||||
Ok(number)
|
||||
}
|
||||
|
||||
struct Lookup<'a> {
|
||||
id: u64,
|
||||
text: &'a str,
|
||||
checkpoint: &'a str,
|
||||
}
|
||||
|
||||
fn lookup(value: &str) -> Result<Lookup<'_>, String> {
|
||||
if value.len() > MAX_NOTE_BYTES || value.bytes().any(|byte| byte < 32 && byte != b'\n') {
|
||||
return Err("invalid sumdb lookup text".into());
|
||||
}
|
||||
let (id, rest) = value
|
||||
.split_once('\n')
|
||||
.ok_or("sumdb lookup lacks record number")?;
|
||||
let end = rest
|
||||
.find("\n\n")
|
||||
.ok_or("sumdb lookup lacks record terminator")?;
|
||||
Ok(Lookup {
|
||||
id: decimal(id)?,
|
||||
text: &rest[..end + 1],
|
||||
checkpoint: &rest[end + 2..],
|
||||
})
|
||||
}
|
||||
|
||||
fn hashes(record: &str, module: &str, version: &str) -> Result<(String, String), String> {
|
||||
let version = format!("v{version}");
|
||||
let mod_version = format!("{version}/go.mod");
|
||||
let mut values = BTreeMap::new();
|
||||
for line in record.lines() {
|
||||
let parts = line.split(' ').collect::<Vec<_>>();
|
||||
if parts.len() != 3
|
||||
|| parts[0] != module
|
||||
|| (parts[1] != version && parts[1] != mod_version)
|
||||
{
|
||||
return Err("sumdb lookup record has a different module/version identity".into());
|
||||
}
|
||||
decode_hash(
|
||||
parts[2]
|
||||
.strip_prefix("h1:")
|
||||
.ok_or("sumdb record requires h1 hashes")?,
|
||||
)?;
|
||||
if values.insert(parts[1], parts[2]).is_some() {
|
||||
return Err("sumdb lookup repeats a module checksum".into());
|
||||
}
|
||||
}
|
||||
Ok((
|
||||
values
|
||||
.get(mod_version.as_str())
|
||||
.ok_or("sumdb lookup lacks go.mod hash")?
|
||||
.to_string(),
|
||||
values
|
||||
.get(version.as_str())
|
||||
.ok_or("sumdb lookup lacks module ZIP hash")?
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
fn dirhash(files: &[(String, Vec<u8>)]) -> Result<String, String> {
|
||||
let mut files = files.iter().collect::<Vec<_>>();
|
||||
files.sort_by(|left, right| left.0.cmp(&right.0));
|
||||
let mut seen = BTreeSet::new();
|
||||
let mut hash = Sha256::new();
|
||||
for (name, bytes) in files {
|
||||
if name.contains('\n') || !seen.insert(name) {
|
||||
return Err("invalid or duplicate Go dirhash filename".into());
|
||||
}
|
||||
hash.update(format!("{} {name}\n", crate::libs::sha256(bytes)));
|
||||
}
|
||||
Ok(format!("h1:{}", STANDARD.encode(hash.finalize())))
|
||||
}
|
||||
|
||||
pub fn mod_hash(bytes: &[u8]) -> String {
|
||||
// The Go protocol hashes a synthetic directory containing only go.mod.
|
||||
dirhash(&[("go.mod".into(), bytes.to_vec())]).expect("fixed dirhash filename")
|
||||
}
|
||||
|
||||
pub fn zip_hash(module: &str, version: &str, bytes: &[u8]) -> Result<String, String> {
|
||||
let files = zip_entries(bytes)?;
|
||||
let prefix = format!("{module}@v{version}/");
|
||||
if files.iter().any(|(path, _)| !path.starts_with(&prefix)) {
|
||||
return Err("Go module archive has a mismatched module root".into());
|
||||
}
|
||||
dirhash(&files)
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Record {
|
||||
pub module: String,
|
||||
pub version: String,
|
||||
pub go_mod: String,
|
||||
pub lookup: String,
|
||||
pub inclusion: Vec<String>,
|
||||
pub consistency: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Evidence {
|
||||
pub previous: Option<String>,
|
||||
pub checkpoint: String,
|
||||
pub consistency: Vec<String>,
|
||||
pub records: Vec<Record>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Verified {
|
||||
pub checkpoint: String,
|
||||
checksums: BTreeMap<(String, String), String>,
|
||||
}
|
||||
|
||||
impl Verified {
|
||||
pub fn verify_zip(&self, module: &str, version: &str, bytes: &[u8]) -> Result<(), String> {
|
||||
let expected = self
|
||||
.checksums
|
||||
.get(&(module.into(), version.into()))
|
||||
.ok_or("Go module lacks authenticated sumdb evidence")?;
|
||||
if zip_hash(module, version, bytes)? != *expected {
|
||||
return Err("Go module ZIP does not match authenticated sumdb h1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn contains(&self, module: &str, version: &str) -> bool {
|
||||
self.checksums
|
||||
.contains_key(&(module.into(), version.into()))
|
||||
}
|
||||
}
|
||||
|
||||
impl Evidence {
|
||||
pub fn verify(&self, verifier: &Verifier) -> Result<Verified, String> {
|
||||
if self.records.len() > MAX_RECORDS
|
||||
|| serde_json::to_vec(self).map_err(|e| e.to_string())?.len() > MAX_EVIDENCE_BYTES
|
||||
{
|
||||
return Err("sumdb evidence exceeds its size budget".into());
|
||||
}
|
||||
let mut current = verifier.checkpoint(&self.checkpoint)?;
|
||||
if let Some(previous) = &self.previous {
|
||||
let older = verifier.checkpoint(previous)?;
|
||||
if older.size > current.size {
|
||||
return Err("sumdb checkpoint rollback rejected".into());
|
||||
}
|
||||
merkle::check_consistency(&self.consistency, older, current)?;
|
||||
} else if !self.consistency.is_empty() {
|
||||
return Err("sumdb first checkpoint has an unexpected consistency proof".into());
|
||||
}
|
||||
let mut latest = self.checkpoint.clone();
|
||||
let mut checksums = BTreeMap::new();
|
||||
for record in &self.records {
|
||||
let entry = lookup(&record.lookup)?;
|
||||
let tree = verifier.checkpoint(entry.checkpoint)?;
|
||||
merkle::check_record(&record.inclusion, tree, entry.id, entry.text.as_bytes())?;
|
||||
let (older, newer) = if tree.size < current.size {
|
||||
(tree, current)
|
||||
} else {
|
||||
(current, tree)
|
||||
};
|
||||
merkle::check_consistency(&record.consistency, older, newer)?;
|
||||
let (mod_h1, zip_h1) = hashes(entry.text, &record.module, &record.version)?;
|
||||
if mod_hash(record.go_mod.as_bytes()) != mod_h1 {
|
||||
return Err("Go go.mod does not match authenticated sumdb h1".into());
|
||||
}
|
||||
if checksums
|
||||
.insert((record.module.clone(), record.version.clone()), zip_h1)
|
||||
.is_some()
|
||||
{
|
||||
return Err("sumdb evidence repeats a module/version".into());
|
||||
}
|
||||
if tree.size > current.size {
|
||||
current = tree;
|
||||
latest = entry.checkpoint.into();
|
||||
}
|
||||
}
|
||||
Ok(Verified {
|
||||
checkpoint: latest,
|
||||
checksums,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One explicit resolution sequence owns the checkpoint. The project lock is
|
||||
/// its persistent anchor; deleting that lock intentionally starts first trust.
|
||||
pub struct ChecksumDatabase {
|
||||
verifier: Verifier,
|
||||
latest: Mutex<Option<String>>,
|
||||
}
|
||||
|
||||
impl ChecksumDatabase {
|
||||
pub fn verifier(&self) -> &Verifier {
|
||||
&self.verifier
|
||||
}
|
||||
|
||||
pub fn new(verifier: Verifier, previous: Option<String>) -> Result<Self, String> {
|
||||
if let Some(note) = &previous {
|
||||
verifier.checkpoint(note)?;
|
||||
}
|
||||
Ok(Self {
|
||||
verifier,
|
||||
latest: Mutex::new(previous),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn official(previous: Option<String>) -> Result<Self, String> {
|
||||
Self::new(Verifier::official(), previous)
|
||||
}
|
||||
|
||||
pub(super) fn begin<'a>(
|
||||
&'a self,
|
||||
transport: &'a dyn RegistryTransport,
|
||||
) -> Result<Session<'a>, String> {
|
||||
let guard = self
|
||||
.latest
|
||||
.lock()
|
||||
.map_err(|_| "sumdb checkpoint state poisoned")?;
|
||||
let checkpoint = String::from_utf8(transport.get_sumdb("/latest", MAX_NOTE_BYTES)?)
|
||||
.map_err(|_| "sumdb checkpoint is not UTF-8")?;
|
||||
let current = self.verifier.checkpoint(&checkpoint)?;
|
||||
let mut tiles = merkle::Tiles::new(transport);
|
||||
let consistency = if let Some(previous) = &*guard {
|
||||
let older = self.verifier.checkpoint(previous)?;
|
||||
if older.size > current.size {
|
||||
return Err("sumdb checkpoint rollback rejected".into());
|
||||
}
|
||||
tiles.consistency(older, current)?
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
let evidence = Evidence {
|
||||
previous: guard.clone(),
|
||||
checkpoint: checkpoint.clone(),
|
||||
consistency,
|
||||
records: Vec::new(),
|
||||
};
|
||||
Ok(Session {
|
||||
verifier: &self.verifier,
|
||||
transport,
|
||||
guard,
|
||||
tiles,
|
||||
current,
|
||||
latest: checkpoint,
|
||||
evidence,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct Session<'a> {
|
||||
verifier: &'a Verifier,
|
||||
transport: &'a dyn RegistryTransport,
|
||||
guard: MutexGuard<'a, Option<String>>,
|
||||
tiles: merkle::Tiles<'a>,
|
||||
current: Tree,
|
||||
latest: String,
|
||||
evidence: Evidence,
|
||||
}
|
||||
|
||||
impl Session<'_> {
|
||||
pub(super) fn verify_mod(
|
||||
&mut self,
|
||||
module: &str,
|
||||
version: &str,
|
||||
source: &[u8],
|
||||
) -> Result<(), String> {
|
||||
if let Some(record) = self
|
||||
.evidence
|
||||
.records
|
||||
.iter()
|
||||
.find(|record| record.module == module && record.version == version)
|
||||
{
|
||||
if record.go_mod.as_bytes() != source {
|
||||
return Err("Go go.mod changed during resolution".into());
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
if self.evidence.records.len() >= MAX_RECORDS {
|
||||
return Err("sumdb resolution exceeds record limit".into());
|
||||
}
|
||||
let path = format!("/lookup/{}@v{}", go_path(module), go_path(version));
|
||||
let response = String::from_utf8(self.transport.get_sumdb(&path, MAX_NOTE_BYTES)?)
|
||||
.map_err(|_| "sumdb lookup is not UTF-8")?;
|
||||
let entry = lookup(&response)?;
|
||||
let tree = self.verifier.checkpoint(entry.checkpoint)?;
|
||||
let inclusion = self.tiles.inclusion(tree.size, entry.id)?;
|
||||
merkle::check_record(&inclusion, tree, entry.id, entry.text.as_bytes())?;
|
||||
let (older, newer) = if tree.size < self.current.size {
|
||||
(tree, self.current)
|
||||
} else {
|
||||
(self.current, tree)
|
||||
};
|
||||
let consistency = self.tiles.consistency(older, newer)?;
|
||||
let (expected, _) = hashes(entry.text, module, version)?;
|
||||
if mod_hash(source) != expected {
|
||||
return Err("Go go.mod does not match authenticated sumdb h1".into());
|
||||
}
|
||||
if tree.size > self.current.size {
|
||||
self.current = tree;
|
||||
self.latest = entry.checkpoint.into();
|
||||
}
|
||||
let record = Record {
|
||||
module: module.into(),
|
||||
version: version.into(),
|
||||
go_mod: String::from_utf8(source.to_vec()).map_err(|_| "Go go.mod is not UTF-8")?,
|
||||
lookup: response,
|
||||
inclusion,
|
||||
consistency,
|
||||
};
|
||||
self.evidence.records.push(record);
|
||||
if serde_json::to_vec(&self.evidence)
|
||||
.map_err(|e| e.to_string())?
|
||||
.len()
|
||||
> MAX_EVIDENCE_BYTES
|
||||
{
|
||||
return Err("sumdb evidence exceeds its size budget".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn verify_zip(
|
||||
&self,
|
||||
module: &str,
|
||||
version: &str,
|
||||
bytes: &[u8],
|
||||
) -> Result<(), String> {
|
||||
let record = self
|
||||
.evidence
|
||||
.records
|
||||
.iter()
|
||||
.find(|record| record.module == module && record.version == version)
|
||||
.ok_or("Go module ZIP has no authenticated metadata")?;
|
||||
let (_, expected) = hashes(lookup(&record.lookup)?.text, module, version)?;
|
||||
if zip_hash(module, version, bytes)? != expected {
|
||||
return Err("Go module ZIP does not match authenticated sumdb h1".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn finish(mut self) -> Evidence {
|
||||
*self.guard = Some(self.latest);
|
||||
self.evidence
|
||||
}
|
||||
}
|
||||
|
||||
pub fn verify_chain(evidence: &[Evidence], verifier: &Verifier) -> Result<Verified, String> {
|
||||
let mut latest = None;
|
||||
let mut checksums = BTreeMap::new();
|
||||
if evidence.len() > MAX_RECORDS {
|
||||
return Err("sumdb resolution history exceeds its limit".into());
|
||||
}
|
||||
let mut bytes = 0usize;
|
||||
for receipt in evidence {
|
||||
bytes = bytes
|
||||
.checked_add(
|
||||
serde_json::to_vec(receipt)
|
||||
.map_err(|error| error.to_string())?
|
||||
.len(),
|
||||
)
|
||||
.ok_or("sumdb resolution history size overflow")?;
|
||||
if bytes > MAX_EVIDENCE_BYTES {
|
||||
return Err("sumdb resolution history exceeds its size budget".into());
|
||||
}
|
||||
if let Some(previous) = &latest
|
||||
&& receipt.previous.as_ref() != Some(previous)
|
||||
{
|
||||
return Err("sumdb resolution checkpoints are not connected".into());
|
||||
}
|
||||
let verified = receipt.verify(verifier)?;
|
||||
for (identity, hash) in verified.checksums {
|
||||
if let Some(old) = checksums.insert(identity, hash.clone())
|
||||
&& old != hash
|
||||
{
|
||||
return Err("sumdb resolutions disagree about module content".into());
|
||||
}
|
||||
}
|
||||
latest = Some(verified.checkpoint);
|
||||
}
|
||||
Ok(Verified {
|
||||
checkpoint: latest.unwrap_or_default(),
|
||||
checksums,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
//! RFC 6962 tree proofs and Go's height-eight tile addressing.
|
||||
//! Protocol reference: golang.org/x/mod/sumdb/tlog (BSD-3-Clause).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{Hash, Tree, decode_hash, encode_hash};
|
||||
use crate::libs::RegistryTransport;
|
||||
|
||||
pub(super) fn leaf(bytes: &[u8]) -> Hash {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update([0]);
|
||||
hash.update(bytes);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
fn node(left: Hash, right: Hash) -> Hash {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update([1]);
|
||||
hash.update(left);
|
||||
hash.update(right);
|
||||
hash.finalize().into()
|
||||
}
|
||||
|
||||
fn split(size: u64) -> u64 {
|
||||
1 << (63 - (size - 1).leading_zeros())
|
||||
}
|
||||
|
||||
pub(super) fn check_record(
|
||||
proof: &[String],
|
||||
tree: Tree,
|
||||
id: u64,
|
||||
bytes: &[u8],
|
||||
) -> Result<(), String> {
|
||||
if id >= tree.size || proof.len() > 63 {
|
||||
return Err("sumdb record proof has invalid bounds".into());
|
||||
}
|
||||
fn root(proof: &[String], size: u64, id: u64, value: Hash) -> Result<Hash, String> {
|
||||
if size == 1 {
|
||||
return if proof.is_empty() {
|
||||
Ok(value)
|
||||
} else {
|
||||
Err("sumdb inclusion proof has surplus hashes".into())
|
||||
};
|
||||
}
|
||||
let (last, rest) = proof
|
||||
.split_last()
|
||||
.ok_or("sumdb inclusion proof is incomplete")?;
|
||||
let sibling = decode_hash(last)?;
|
||||
let left = split(size);
|
||||
if id < left {
|
||||
Ok(node(root(rest, left, id, value)?, sibling))
|
||||
} else {
|
||||
Ok(node(sibling, root(rest, size - left, id - left, value)?))
|
||||
}
|
||||
}
|
||||
if root(proof, tree.size, id, leaf(bytes))? != tree.hash {
|
||||
return Err("sumdb inclusion proof does not match signed checkpoint".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn check_consistency(proof: &[String], older: Tree, newer: Tree) -> Result<(), String> {
|
||||
if older.size > newer.size || proof.len() > 63 {
|
||||
return Err("sumdb consistency proof has invalid bounds".into());
|
||||
}
|
||||
if older.size == 0 {
|
||||
let empty: Hash = Sha256::digest([]).into();
|
||||
return if proof.is_empty() && older.hash == empty {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("sumdb empty checkpoint is invalid".into())
|
||||
};
|
||||
}
|
||||
fn roots(
|
||||
proof: &[String],
|
||||
lo: u64,
|
||||
size: u64,
|
||||
old_size: u64,
|
||||
old_hash: Hash,
|
||||
) -> Result<(Hash, Hash), String> {
|
||||
if old_size == size {
|
||||
return if lo == 0 && proof.is_empty() {
|
||||
Ok((old_hash, old_hash))
|
||||
} else if lo != 0 && proof.len() == 1 {
|
||||
let hash = decode_hash(&proof[0])?;
|
||||
Ok((hash, hash))
|
||||
} else {
|
||||
Err("sumdb consistency proof has surplus or missing hashes".into())
|
||||
};
|
||||
}
|
||||
let (last, rest) = proof
|
||||
.split_last()
|
||||
.ok_or("sumdb consistency proof is incomplete")?;
|
||||
let sibling = decode_hash(last)?;
|
||||
let left = split(size);
|
||||
if old_size <= left {
|
||||
let (old, new) = roots(rest, lo, left, old_size, old_hash)?;
|
||||
Ok((old, node(new, sibling)))
|
||||
} else {
|
||||
let (old, new) = roots(rest, lo + left, size - left, old_size - left, old_hash)?;
|
||||
Ok((node(sibling, old), node(sibling, new)))
|
||||
}
|
||||
}
|
||||
let (old, new) = roots(proof, 0, newer.size, older.size, older.hash)?;
|
||||
if old != older.hash || new != newer.hash {
|
||||
return Err("sumdb consistency proof detects a conflicting history".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetched tiles are only inputs to proofs; no byte is trusted merely because
|
||||
/// it is cached. Persisted Evidence contains only completely verified proofs.
|
||||
pub(super) struct Tiles<'a> {
|
||||
transport: &'a dyn RegistryTransport,
|
||||
cache: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
impl<'a> Tiles<'a> {
|
||||
pub(super) fn new(transport: &'a dyn RegistryTransport) -> Self {
|
||||
Self {
|
||||
transport,
|
||||
cache: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn subtree(&mut self, lo: u64, size: u64, tree_size: u64) -> Result<Hash, String> {
|
||||
if size.is_power_of_two() {
|
||||
let level = size.trailing_zeros();
|
||||
let tile_level = level / 8;
|
||||
let tile_scale = tile_level * 8;
|
||||
let count = 1u64 << (level % 8);
|
||||
let first = lo >> tile_scale;
|
||||
let index = first / 256;
|
||||
let offset = first % 256;
|
||||
let width = (tree_size >> tile_scale)
|
||||
.checked_sub(index * 256)
|
||||
.ok_or("sumdb tile lies outside checkpoint")?
|
||||
.min(256);
|
||||
if offset + count > width {
|
||||
return Err("sumdb tile does not cover requested subtree".into());
|
||||
}
|
||||
let mut chunks = vec![format!("{:03}", index % 1000)];
|
||||
let mut prefix = index / 1000;
|
||||
while prefix > 0 {
|
||||
chunks.push(format!("x{:03}", prefix % 1000));
|
||||
prefix /= 1000;
|
||||
}
|
||||
chunks.reverse();
|
||||
let full = format!("/tile/8/{tile_level}/{}", chunks.join("/"));
|
||||
let path = if width == 256 {
|
||||
full.clone()
|
||||
} else {
|
||||
format!("{full}.p/{width}")
|
||||
};
|
||||
if !self.cache.contains_key(&path) {
|
||||
if self.cache.len() >= 4096 {
|
||||
return Err("sumdb tile cache exceeds its 32 MiB budget".into());
|
||||
}
|
||||
let bytes = match self.transport.get_sumdb(&path, 8192) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) if width < 256 => {
|
||||
let mut bytes = self.transport.get_sumdb(&full, 8192)?;
|
||||
if bytes.len() != 8192 {
|
||||
return Err("sumdb full tile has wrong length".into());
|
||||
}
|
||||
bytes.truncate(width as usize * 32);
|
||||
bytes
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if bytes.len() != width as usize * 32 {
|
||||
return Err("sumdb tile has wrong length".into());
|
||||
}
|
||||
self.cache.insert(path.clone(), bytes);
|
||||
}
|
||||
fn fold(bytes: &[u8]) -> Hash {
|
||||
if bytes.len() == 32 {
|
||||
return bytes.try_into().expect("one tile hash");
|
||||
}
|
||||
let (left, right) = bytes.split_at(bytes.len() / 2);
|
||||
node(fold(left), fold(right))
|
||||
}
|
||||
return Ok(fold(
|
||||
&self.cache[&path][offset as usize * 32..(offset + count) as usize * 32],
|
||||
));
|
||||
}
|
||||
let left = split(size);
|
||||
Ok(node(
|
||||
self.subtree(lo, left, tree_size)?,
|
||||
self.subtree(lo + left, size - left, tree_size)?,
|
||||
))
|
||||
}
|
||||
|
||||
pub(super) fn inclusion(&mut self, size: u64, id: u64) -> Result<Vec<String>, String> {
|
||||
fn walk(
|
||||
tiles: &mut Tiles<'_>,
|
||||
lo: u64,
|
||||
size: u64,
|
||||
id: u64,
|
||||
total: u64,
|
||||
proof: &mut Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
if size == 1 {
|
||||
return Ok(());
|
||||
}
|
||||
let left = split(size);
|
||||
let sibling = if id < lo + left {
|
||||
walk(tiles, lo, left, id, total, proof)?;
|
||||
tiles.subtree(lo + left, size - left, total)?
|
||||
} else {
|
||||
walk(tiles, lo + left, size - left, id, total, proof)?;
|
||||
tiles.subtree(lo, left, total)?
|
||||
};
|
||||
proof.push(encode_hash(sibling));
|
||||
Ok(())
|
||||
}
|
||||
if id >= size {
|
||||
return Err("sumdb record lies outside signed checkpoint".into());
|
||||
}
|
||||
let mut proof = Vec::new();
|
||||
walk(self, 0, size, id, size, &mut proof)?;
|
||||
Ok(proof)
|
||||
}
|
||||
|
||||
pub(super) fn consistency(&mut self, older: Tree, newer: Tree) -> Result<Vec<String>, String> {
|
||||
fn walk(
|
||||
tiles: &mut Tiles<'_>,
|
||||
lo: u64,
|
||||
size: u64,
|
||||
old_size: u64,
|
||||
total: u64,
|
||||
proof: &mut Vec<String>,
|
||||
) -> Result<(), String> {
|
||||
if old_size == size {
|
||||
if lo != 0 {
|
||||
proof.push(encode_hash(tiles.subtree(lo, size, total)?));
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let left = split(size);
|
||||
let sibling = if old_size <= left {
|
||||
walk(tiles, lo, left, old_size, total, proof)?;
|
||||
tiles.subtree(lo + left, size - left, total)?
|
||||
} else {
|
||||
walk(tiles, lo + left, size - left, old_size - left, total, proof)?;
|
||||
tiles.subtree(lo, left, total)?
|
||||
};
|
||||
proof.push(encode_hash(sibling));
|
||||
Ok(())
|
||||
}
|
||||
let mut proof = Vec::new();
|
||||
if older.size != 0 {
|
||||
walk(self, 0, newer.size, older.size, newer.size, &mut proof)?;
|
||||
}
|
||||
check_consistency(&proof, older, newer)?;
|
||||
Ok(proof)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user