feat: ship slim releases with shared on-demand extensions and locked restore
This commit is contained in:
1 parent
83b78a0db6
commit
4fda7c8c1a
58 files changed
+3774
-425
No files matched your search
@@ -30,6 +30,8 @@ mod native_elf;
|
||||
mod npm_build_tests;
|
||||
#[path = "external_libs/npm.rs"]
|
||||
mod npm_tests;
|
||||
#[path = "external_libs/restore.rs"]
|
||||
mod restore_tests;
|
||||
#[path = "external_libs/sumdb.rs"]
|
||||
mod sumdb_tests;
|
||||
#[path = "support/wheel.rs"]
|
||||
@@ -1694,6 +1696,7 @@ fn expanded_worker_resources_keep_archives_needed_by_exec_or_declarations() {
|
||||
.collect(),
|
||||
runtime: registry_runtime(spec.ecosystem.clone()).pack,
|
||||
artifacts: vec![LockedArtifact {
|
||||
source: None,
|
||||
target: target.clone(),
|
||||
path: path(spec),
|
||||
bytes: 1,
|
||||
|
||||
@@ -280,14 +280,16 @@ fn backend_fixture(mutate: bool) -> LocalRegistry {
|
||||
),
|
||||
],
|
||||
);
|
||||
let key = (Ecosystem::Pip, "/pypi/buildhelper/1.0.0/json".into());
|
||||
let mut release: serde_json::Value = serde_json::from_slice(®istry.0[&key]).unwrap();
|
||||
let url = release["urls"][0]["url"].as_str().unwrap().to_owned();
|
||||
release["urls"][0]["digests"]["sha256"] = serde_json::json!(digest(&helper));
|
||||
let helper_source = source_archive(vec![
|
||||
("buildhelper-1.0.0/pyproject.toml", b"[build-system]\nrequires = []\nbuild-backend = 'backend'\nbackend-path = ['.']\n".to_vec()),
|
||||
("buildhelper-1.0.0/backend.py", b"import shutil\nfrom pathlib import Path\ndef build_wheel(directory, config_settings=None, metadata_directory=None):\n name = 'buildhelper-1.0.0-py3-none-any.whl'\n shutil.copy('result.whl', Path(directory) / name)\n return name\n".to_vec()),
|
||||
("buildhelper-1.0.0/result.whl", helper),
|
||||
]);
|
||||
registry.0.insert((Ecosystem::Pip, "/pypi/buildhelper/1.0.0/json".into()),
|
||||
serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"buildhelper-1.0.0.tar.gz","url":"helper-source","digests":{"sha256":digest(&helper_source)}}]})).unwrap());
|
||||
registry
|
||||
.0
|
||||
.insert(key, serde_json::to_vec(&release).unwrap());
|
||||
registry.0.insert((Ecosystem::Pip, url), helper);
|
||||
.insert((Ecosystem::Pip, "helper-source".into()), helper_source);
|
||||
let original = wheel_fixture::metadata("sample", "1.0.0", &[], &[]);
|
||||
let changed = format!("{original}Summary: changed by wheel hook\n");
|
||||
let wheel = wheel_fixture::pack(
|
||||
@@ -334,11 +336,20 @@ def build_wheel(directory, config_settings=None, metadata_directory=None):
|
||||
("sample-1.0.0/changed-metadata", changed.into_bytes()),
|
||||
("sample-1.0.0/result.whl", wheel),
|
||||
];
|
||||
let source = source_archive(sourcefiles);
|
||||
registry.0.insert((Ecosystem::Pip,"/pypi/sample/1.0.0/json".into()),serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"sample-1.0.0.tar.gz","url":"sample-source","digests":{"sha256":digest(&source)}}]})).unwrap());
|
||||
registry
|
||||
.0
|
||||
.insert((Ecosystem::Pip, "sample-source".into()), source);
|
||||
registry
|
||||
}
|
||||
|
||||
fn source_archive(files: Vec<(&str, Vec<u8>)>) -> Vec<u8> {
|
||||
let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(
|
||||
Vec::new(),
|
||||
flate2::Compression::default(),
|
||||
));
|
||||
for (name, bytes) in sourcefiles {
|
||||
for (name, bytes) in files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_size(bytes.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
@@ -346,12 +357,7 @@ def build_wheel(directory, config_settings=None, metadata_directory=None):
|
||||
tar.append_data(&mut header, name, bytes.as_slice())
|
||||
.unwrap();
|
||||
}
|
||||
let source = tar.into_inner().unwrap().finish().unwrap();
|
||||
registry.0.insert((Ecosystem::Pip,"/pypi/sample/1.0.0/json".into()),serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"sample-1.0.0.tar.gz","url":"sample-source","digests":{"sha256":digest(&source)}}]})).unwrap());
|
||||
registry
|
||||
.0
|
||||
.insert((Ecosystem::Pip, "sample-source".into()), source);
|
||||
registry
|
||||
tar.into_inner().unwrap().finish().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -383,12 +389,55 @@ fn pep517_hooks_keep_requirements_cli_and_metadata_boundaries() {
|
||||
let lock = result.unwrap();
|
||||
dever_cli::libs::doctor(&lock).unwrap();
|
||||
assert_eq!(lock.builds[0].dynamic_requires, vec!["buildhelper==1.0.0"]);
|
||||
assert_eq!(
|
||||
lock.builds[0].inputs.builds.len(),
|
||||
1,
|
||||
"nested build requirement receipt must be retained"
|
||||
);
|
||||
let before = lock.encode().unwrap();
|
||||
let cold = FixtureArtifactStore::default();
|
||||
let archives = super::restore_tests::archives_only(&lock, &transport);
|
||||
dever_cli::libs::restore::restore_artifacts(
|
||||
&lock,
|
||||
&archives,
|
||||
&cold,
|
||||
&inputs,
|
||||
&inputs.settings.runtime.target,
|
||||
&dever_cli::libs::sumdb::Verifier::official(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(lock.encode().unwrap(), before);
|
||||
let mut mismatch = lock.clone();
|
||||
mismatch.builds[0].output.sha256 = "0".repeat(64);
|
||||
mismatch.libs[0].artifacts = vec![mismatch.builds[0].output.clone()];
|
||||
mismatch.libs[0].build = Some(mismatch.builds[0].identity().unwrap());
|
||||
let rejected = FixtureArtifactStore::default();
|
||||
assert!(
|
||||
dever_cli::libs::restore::restore_artifacts(
|
||||
&mismatch,
|
||||
&archives,
|
||||
&rejected,
|
||||
&inputs,
|
||||
&inputs.settings.runtime.target,
|
||||
&dever_cli::libs::sumdb::Verifier::official()
|
||||
)
|
||||
.unwrap_err()
|
||||
.contains("not byte reproducible")
|
||||
);
|
||||
assert!(
|
||||
!rejected
|
||||
.has(
|
||||
&mismatch.builds[0].output.sha256,
|
||||
&inputs.settings.runtime.target
|
||||
)
|
||||
.unwrap()
|
||||
);
|
||||
for mutation in 0..3 {
|
||||
let mut altered = lock.clone();
|
||||
match mutation {
|
||||
0 => altered.builds[0].tools.sha256 = "not-a-hash".into(),
|
||||
1 => altered.builds[0].dynamic_requires = vec!["buildhelper>=2".into()],
|
||||
_ => *altered.builds[0].dependencies = LockFile::new(vec![]).unwrap(),
|
||||
_ => *altered.builds[0].inputs = LockFile::new(vec![]).unwrap(),
|
||||
}
|
||||
altered.libs[0].build = Some(altered.builds[0].identity().unwrap());
|
||||
assert!(
|
||||
@@ -441,7 +490,7 @@ fn builds_official_python_source_with_isolated_pep517() {
|
||||
let lock = dever_cli::libs::LibResolver::resolve(&resolver, &requested).unwrap();
|
||||
dever_cli::libs::doctor(&lock).unwrap();
|
||||
assert_eq!(lock.builds.len(), 1);
|
||||
assert!(!lock.builds[0].dependencies.libs.is_empty());
|
||||
assert!(!lock.builds[0].inputs.libs.is_empty());
|
||||
let artifact = &lock
|
||||
.libs
|
||||
.iter()
|
||||
|
||||
@@ -65,7 +65,7 @@ fn npm_registry_install_hooks_do_not_run_publishing_lifecycle() {
|
||||
);
|
||||
}
|
||||
|
||||
fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
|
||||
fn locked_installation_fixture() -> (LockFile, Vec<u8>, FixtureArtifactStore) {
|
||||
let mut registry = LocalRegistry(BTreeMap::new());
|
||||
npm_tests::release(
|
||||
&mut registry,
|
||||
@@ -85,11 +85,13 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
|
||||
"npm:other@1.0.0".parse().unwrap(),
|
||||
])
|
||||
.unwrap();
|
||||
let sources = lock
|
||||
let mut sources: Vec<_> = lock
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| lib.artifacts.clone())
|
||||
.collect();
|
||||
sources.sort();
|
||||
sources.dedup();
|
||||
let archives = lock
|
||||
.libs
|
||||
.iter()
|
||||
@@ -159,9 +161,10 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
|
||||
),
|
||||
python: None,
|
||||
dynamic_requires: vec![],
|
||||
dependencies: Box::new(LockFile::new(vec![]).unwrap()),
|
||||
inputs: Box::new(lock.clone()),
|
||||
config_settings: BTreeMap::new(),
|
||||
output: dever_cli::libs::LockedArtifact {
|
||||
source: None,
|
||||
target: runtime.target,
|
||||
path: format!("build/npm/{hash}/installation.tgz"),
|
||||
bytes: bytes.len() as u64,
|
||||
@@ -171,7 +174,24 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
|
||||
lock.npm[0].build = Some(receipt.identity().unwrap());
|
||||
lock.builds.push(receipt);
|
||||
doctor(&lock).unwrap();
|
||||
(lock, bytes)
|
||||
(lock, bytes, store)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_npm_build_restore_does_not_load_build_tools_or_auxiliary_runtime() {
|
||||
let (lock, bytes, store) = locked_installation_fixture();
|
||||
store
|
||||
.publish(&bytes, &lock.builds[0].output.target)
|
||||
.unwrap();
|
||||
dever_cli::libs::restore::restore_artifacts(
|
||||
&lock,
|
||||
&LocalRegistry(BTreeMap::new()),
|
||||
&store,
|
||||
&super::restore_tests::RuntimeOnly,
|
||||
"linux-x86_64",
|
||||
&sumdb_tests::verifier(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -179,7 +199,7 @@ fn npm_expanded_outputs_are_pruned_only_when_no_raw_consumer_remains() {
|
||||
use dever_cli::libs::{LockedWorker, prune_expanded_resources};
|
||||
use dever_core::native::EmbeddedResource;
|
||||
|
||||
let (mut lock, bytes) = locked_installation_fixture();
|
||||
let (mut lock, bytes, _) = locked_installation_fixture();
|
||||
let roots = lock.npm[0].roots.clone();
|
||||
let other = vec!["npm:other@1.0.0".parse().unwrap()];
|
||||
lock.npm = npm::retain(&lock, &[roots.clone(), other.clone()]).unwrap();
|
||||
@@ -231,7 +251,7 @@ fn npm_expanded_outputs_are_pruned_only_when_no_raw_consumer_remains() {
|
||||
|
||||
#[test]
|
||||
fn npm_native_admission_accounts_every_addon_and_prunes_unsupported_objects() {
|
||||
let (lock, bytes) = locked_installation_fixture();
|
||||
let (lock, bytes, _) = locked_installation_fixture();
|
||||
let files = dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap();
|
||||
assert!(files.iter().all(|(path, _, _)| !path.ends_with(".node")));
|
||||
let mut missing = lock.clone();
|
||||
@@ -262,7 +282,7 @@ fn npm_native_admission_accounts_every_addon_and_prunes_unsupported_objects() {
|
||||
|
||||
#[test]
|
||||
fn npm_locked_output_projects_retained_roots_without_rerunning_hooks() {
|
||||
let (lock, bytes) = locked_installation_fixture();
|
||||
let (lock, bytes, _) = locked_installation_fixture();
|
||||
let project = TemporaryDirectory::new();
|
||||
fs::create_dir(project.path().join("config")).unwrap();
|
||||
fs::write(
|
||||
@@ -468,6 +488,33 @@ fn npm_lifecycle_preserves_optional_failures_bins_and_offline_remove() {
|
||||
.any(|edge| edge.omission == Some(npm::Omission::BuildFailure))
|
||||
);
|
||||
let receipt = &lock.builds[0];
|
||||
assert!(
|
||||
receipt
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.name == "broken")
|
||||
);
|
||||
assert!(
|
||||
receipt
|
||||
.inputs
|
||||
.libs
|
||||
.iter()
|
||||
.any(|lib| lib.spec.name == "cycle")
|
||||
);
|
||||
assert!(receipt.inputs.npm[0].build.is_none());
|
||||
let cold = FixtureArtifactStore::default();
|
||||
let before = lock.encode().unwrap();
|
||||
dever_cli::libs::restore::restore_artifacts(
|
||||
&lock,
|
||||
&super::restore_tests::archives_only(&lock, &transport),
|
||||
&cold,
|
||||
&inputs,
|
||||
&inputs.settings.runtime.target,
|
||||
&dever_cli::libs::sumdb::Verifier::official(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(lock.encode().unwrap(), before);
|
||||
let bytes = store
|
||||
.verify_exact(
|
||||
&receipt.output.sha256,
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
use super::*;
|
||||
use dever_cli::libs::build::{Inputs, pack};
|
||||
use dever_cli::libs::restore::restore_artifacts;
|
||||
|
||||
pub(super) struct RuntimeOnly;
|
||||
|
||||
pub(super) fn archives_only(lock: &LockFile, registry: &LocalRegistry) -> LocalRegistry {
|
||||
fn collect(
|
||||
lock: &LockFile,
|
||||
registry: &LocalRegistry,
|
||||
files: &mut BTreeMap<(Ecosystem, String), Vec<u8>>,
|
||||
) {
|
||||
for artifact in lock
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| &lib.artifacts)
|
||||
.chain(lock.builds.iter().flat_map(|receipt| &receipt.sources))
|
||||
{
|
||||
if let Some(source) = &artifact.source {
|
||||
let key = (source.ecosystem.clone(), source.locator.clone());
|
||||
files.insert(key.clone(), registry.0[&key].clone());
|
||||
}
|
||||
}
|
||||
for receipt in &lock.builds {
|
||||
collect(&receipt.inputs, registry, files);
|
||||
}
|
||||
}
|
||||
let mut files = BTreeMap::new();
|
||||
collect(lock, registry, &mut files);
|
||||
LocalRegistry(files)
|
||||
}
|
||||
impl Inputs for RuntimeOnly {
|
||||
fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> {
|
||||
Ok((registry_runtime(ecosystem), vec![]))
|
||||
}
|
||||
fn tools(&self, _: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String> {
|
||||
panic!("archive restoration must not load build extensions")
|
||||
}
|
||||
fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> {
|
||||
panic!("archive restoration must not execute a sandbox")
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_cache_restore_uses_only_locked_sources_and_never_registry_metadata() {
|
||||
let mut registry = local_registry();
|
||||
let original = FixtureArtifactStore::default();
|
||||
let roots = ["pip:a@1.0.0", "npm:root@1.0.0", "go:example.com/a@1.0.0"]
|
||||
.map(|spec| spec.parse().unwrap());
|
||||
let lock = dever_cli::libs::LibResolver::resolve(&local_resolver(®istry, &original), &roots)
|
||||
.unwrap();
|
||||
let before = lock.encode().unwrap();
|
||||
let locators = lock
|
||||
.libs
|
||||
.iter()
|
||||
.flat_map(|lib| &lib.artifacts)
|
||||
.map(|artifact| {
|
||||
let source = artifact.source.as_ref().unwrap();
|
||||
assert!(!source.filename.is_empty());
|
||||
(source.ecosystem.clone(), source.locator.clone())
|
||||
})
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
registry.0.retain(|key, _| locators.contains(key));
|
||||
let cold = FixtureArtifactStore::default();
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
®istry,
|
||||
&cold,
|
||||
&RuntimeOnly,
|
||||
"linux-x86_64",
|
||||
&sumdb_tests::verifier(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(lock.encode().unwrap(), before);
|
||||
assert_eq!(
|
||||
verify_locked_artifacts(&lock, &cold, "linux-x86_64").unwrap(),
|
||||
verify_locked_artifacts(&lock, &original, "linux-x86_64").unwrap()
|
||||
);
|
||||
// A warm restore must also succeed with no available registry at all.
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
&LocalRegistry(BTreeMap::new()),
|
||||
&cold,
|
||||
&RuntimeOnly,
|
||||
"linux-x86_64",
|
||||
&sumdb_tests::verifier(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restored_archive_mismatch_fails_before_cache_publication() {
|
||||
let mut registry = local_registry();
|
||||
let lock = dever_cli::libs::LibResolver::resolve(
|
||||
&local_resolver(®istry, &FixtureArtifactStore::default()),
|
||||
&["pip:a@1.0.0".parse().unwrap()],
|
||||
)
|
||||
.unwrap();
|
||||
let source = lock.libs[0].artifacts[0].source.as_ref().unwrap();
|
||||
registry.0.insert(
|
||||
(source.ecosystem.clone(), source.locator.clone()),
|
||||
b"different bytes".to_vec(),
|
||||
);
|
||||
let cold = FixtureArtifactStore::default();
|
||||
assert!(
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
®istry,
|
||||
&cold,
|
||||
&RuntimeOnly,
|
||||
"linux-x86_64",
|
||||
&sumdb_tests::verifier()
|
||||
)
|
||||
.unwrap_err()
|
||||
.contains("byte/hash")
|
||||
);
|
||||
let artifact = &lock.libs[0].artifacts[0];
|
||||
assert!(!cold.has(&artifact.sha256, &artifact.target).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corrupt_cache_is_an_error_and_never_falls_back_to_download() {
|
||||
struct CorruptCache;
|
||||
impl ArtifactStore for CorruptCache {
|
||||
fn has(&self, _: &str, _: &str) -> Result<bool, String> {
|
||||
unreachable!()
|
||||
}
|
||||
fn publish(&self, _: &[u8], _: &str) -> Result<String, String> {
|
||||
panic!("corrupt cache cannot be replaced")
|
||||
}
|
||||
fn get(&self, _: &str, _: &str) -> Result<Option<Vec<u8>>, String> {
|
||||
Ok(Some(b"corrupt".to_vec()))
|
||||
}
|
||||
}
|
||||
let registry = local_registry();
|
||||
let lock = dever_cli::libs::LibResolver::resolve(
|
||||
&local_resolver(®istry, &FixtureArtifactStore::default()),
|
||||
&["pip:a@1.0.0".parse().unwrap()],
|
||||
)
|
||||
.unwrap();
|
||||
// No registry entries exist: a fallback would produce a distinct missing-path error.
|
||||
assert!(
|
||||
restore_artifacts(
|
||||
&lock,
|
||||
&LocalRegistry(BTreeMap::new()),
|
||||
&CorruptCache,
|
||||
&RuntimeOnly,
|
||||
"linux-x86_64",
|
||||
&sumdb_tests::verifier()
|
||||
)
|
||||
.unwrap_err()
|
||||
.contains("byte/hash")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lib_install_preserves_setting_and_lock_bytes_and_rejects_declaration_drift() {
|
||||
let root = TemporaryDirectory::new();
|
||||
fs::create_dir(root.path().join("config")).unwrap();
|
||||
let setting = b"{\n \"lib\": [\"pip:fixture@1.0.0\"]\n}\n";
|
||||
fs::write(root.path().join("config/setting.json"), setting).unwrap();
|
||||
assert!(
|
||||
dever_cli::libs::execute("install", root.path(), &[])
|
||||
.unwrap_err()
|
||||
.contains("existing dever.lock")
|
||||
);
|
||||
let lock = FixtureRegistry::builtin()
|
||||
.resolve(&["pip:fixture@1.0.0".parse().unwrap()])
|
||||
.unwrap();
|
||||
lock.write_atomic(root.path()).unwrap();
|
||||
let before = fs::read(root.path().join("dever.lock")).unwrap();
|
||||
dever_cli::libs::execute("install", root.path(), &[]).unwrap();
|
||||
assert_eq!(
|
||||
fs::read(root.path().join("config/setting.json")).unwrap(),
|
||||
setting
|
||||
);
|
||||
assert_eq!(fs::read(root.path().join("dever.lock")).unwrap(), before);
|
||||
assert!(
|
||||
dever_cli::libs::execute("install", root.path(), &["pip:fixture@1.0.0".into()])
|
||||
.unwrap_err()
|
||||
.contains("no Lib specs")
|
||||
);
|
||||
fs::write(root.path().join("config/setting.json"), b"{\"lib\":[]}").unwrap();
|
||||
assert!(
|
||||
dever_cli::libs::execute("install", root.path(), &[])
|
||||
.unwrap_err()
|
||||
.contains("declared closure")
|
||||
);
|
||||
assert_eq!(fs::read(root.path().join("dever.lock")).unwrap(), before);
|
||||
}
|
||||
@@ -262,6 +262,7 @@ fn fixture_for_target(
|
||||
dependencies: vec![],
|
||||
runtime: runtime.clone(),
|
||||
artifacts: vec![LockedArtifact {
|
||||
source: None,
|
||||
target: target.into(),
|
||||
path: path.clone(),
|
||||
bytes: bytes.len() as u64,
|
||||
@@ -794,6 +795,7 @@ fn run_native_python_fixture(fixture: &str) {
|
||||
dependencies: vec![],
|
||||
runtime: runtime.clone(),
|
||||
artifacts: vec![LockedArtifact {
|
||||
source: None,
|
||||
target: dever_cli::toolchain::platform_identity(),
|
||||
path: probe_path.clone(),
|
||||
bytes: probe.len() as u64,
|
||||
|
||||
@@ -358,6 +358,56 @@ fn install_release(layout: &Layout, version: &Version, key: &Ed25519KeyPair) {
|
||||
MachineManager::new(layout.clone())
|
||||
.install_requested(version.as_str())
|
||||
.unwrap();
|
||||
// These opt-in whole-release fixtures explicitly prepare every declared resource.
|
||||
// Product installation only installs the base; each preparation still exercises
|
||||
// the ordinary signed extraction and immutable extension publication owner.
|
||||
struct PreparedInputs(PathBuf);
|
||||
impl dever_cli::toolchain::ExtensionSource for PreparedInputs {
|
||||
fn open(
|
||||
&self,
|
||||
_version: &Version,
|
||||
id: &dever_cli::toolchain::ExtensionId,
|
||||
) -> Result<Box<dyn std::io::Read>, String> {
|
||||
let catalog: dever_cli::toolchain::ReleaseManifest = serde_json::from_slice(
|
||||
&fs::read(self.0.join("manifest.json")).map_err(|error| error.to_string())?,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let extension = catalog
|
||||
.extensions
|
||||
.iter()
|
||||
.find(|extension| extension.id() == *id)
|
||||
.ok_or("fixture extension is undeclared")?;
|
||||
let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 1)
|
||||
.map_err(|error| error.to_string())?;
|
||||
encoder
|
||||
.include_checksum(true)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let mut archive = tar::Builder::new(encoder);
|
||||
for artifact in &extension.artifacts {
|
||||
let mut header = tar::Header::new_ustar();
|
||||
header.set_size(artifact.bytes);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
let file = fs::File::open(self.0.join(&artifact.path))
|
||||
.map_err(|error| error.to_string())?;
|
||||
archive
|
||||
.append_data(&mut header, &artifact.path, file)
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
let bytes = archive
|
||||
.into_inner()
|
||||
.map_err(|error| error.to_string())?
|
||||
.finish()
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(Box::new(std::io::Cursor::new(bytes)))
|
||||
}
|
||||
}
|
||||
let resources = dever_cli::toolchain::SignedResources::load(layout, version).unwrap();
|
||||
let source = PreparedInputs(layout.downloads().join(version.as_str()));
|
||||
for extension in &resources.manifest().extensions {
|
||||
dever_cli::toolchain::prepare_extension_with(layout, version, &extension.id(), &source)
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn archive_report(workspace: &Path) -> Value {
|
||||
|
||||
@@ -3,7 +3,7 @@ use super::*;
|
||||
use dever_cli::libs::{
|
||||
self, LibSpec, LockFile, LockedArtifact, LockedLib, LockedWorker, RegistryRuntime,
|
||||
};
|
||||
use dever_cli::toolchain::{Artifact, ReleaseManifest};
|
||||
use dever_cli::toolchain::{Artifact, Extension, ExtensionKind, ReleaseManifest};
|
||||
use dever_core::hir::ExternalWorkerContract;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::BTreeMap;
|
||||
@@ -68,6 +68,7 @@ fn sign_target_runtimes(
|
||||
serde_json::to_vec(&json!({"format":"dever-registry-runtime-v1","runtime":runtime}))
|
||||
.unwrap(),
|
||||
);
|
||||
let mut artifacts = Vec::new();
|
||||
for leaf in ["manifest.json", "runtime.pack"] {
|
||||
let path = format!("{prefix}/{leaf}");
|
||||
assert!(
|
||||
@@ -76,17 +77,21 @@ fn sign_target_runtimes(
|
||||
.iter()
|
||||
.any(|artifact| artifact.path == path)
|
||||
);
|
||||
manifest.artifacts.push(Artifact {
|
||||
artifacts.push(Artifact {
|
||||
bytes: fs::metadata(release.join(&path)).unwrap().len(),
|
||||
sha256: sha256_file(&release.join(&path)).unwrap(),
|
||||
path,
|
||||
});
|
||||
}
|
||||
manifest.extensions.push(Extension {
|
||||
kind: ExtensionKind::Runtime(ecosystem.parse().unwrap()),
|
||||
target: TARGET,
|
||||
artifacts,
|
||||
});
|
||||
runtimes.insert(ecosystem.to_owned(), runtime);
|
||||
}
|
||||
manifest
|
||||
.artifacts
|
||||
.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
manifest.extensions.sort_by_key(Extension::id);
|
||||
dever_cli::toolchain::validate_catalog(&manifest).unwrap();
|
||||
let encoded = serde_json::to_vec(&manifest).unwrap();
|
||||
fs::write(release.join("manifest.json"), &encoded).unwrap();
|
||||
fs::write(
|
||||
@@ -167,6 +172,7 @@ fn pure_package_lock(
|
||||
.unwrap();
|
||||
let sha256 = digest(&bytes);
|
||||
let artifact = LockedArtifact {
|
||||
source: None,
|
||||
target: TARGET.platform().into(),
|
||||
path: format!(
|
||||
"lib/{}/{}/{}/archive.{extension}",
|
||||
|
||||
@@ -121,8 +121,9 @@ fn sign_build_runtimes(
|
||||
for suffix in ["runtime.pack", "manifest.json"] {
|
||||
let name = format!("{prefix}/{suffix}");
|
||||
let artifact = manifest
|
||||
.artifacts
|
||||
.extensions
|
||||
.iter_mut()
|
||||
.flat_map(|extension| &mut extension.artifacts)
|
||||
.find(|artifact| artifact.path == name)
|
||||
.unwrap();
|
||||
artifact.bytes = fs::metadata(release.join(&name)).unwrap().len();
|
||||
@@ -311,8 +312,9 @@ fn signed_ecosystems_build_and_run_without_system_language_installations() {
|
||||
for ecosystem in ECOSYSTEMS {
|
||||
let path = format!("runtime/{ecosystem}/linux-x86_64/runtime.pack");
|
||||
let artifact = signed
|
||||
.artifacts
|
||||
.extensions
|
||||
.iter()
|
||||
.flat_map(|extension| &extension.artifacts)
|
||||
.find(|artifact| artifact.path == path)
|
||||
.unwrap();
|
||||
runtimes.insert(
|
||||
|
||||
@@ -148,8 +148,9 @@ fn ecosystem_archives_are_deterministic_and_signed_with_computed_identities() {
|
||||
);
|
||||
for path in [&metadata_path, &pack_path] {
|
||||
let signed = manifest
|
||||
.artifacts
|
||||
.extensions
|
||||
.iter()
|
||||
.flat_map(|extension| &extension.artifacts)
|
||||
.find(|artifact| &artifact.path == path)
|
||||
.unwrap();
|
||||
assert_eq!(signed.sha256, sha256_file(&first.join(path)).unwrap());
|
||||
@@ -286,8 +287,9 @@ fn go_cross_pack_binds_host_tools_and_arm_standard_library_separately() {
|
||||
let manifest = result.unwrap();
|
||||
assert!(
|
||||
manifest
|
||||
.artifacts
|
||||
.extensions
|
||||
.iter()
|
||||
.flat_map(|extension| &extension.artifacts)
|
||||
.any(|file| file.path == "runtime/go/linux-aarch64/runtime.pack")
|
||||
);
|
||||
}
|
||||
@@ -295,6 +297,7 @@ fn go_cross_pack_binds_host_tools_and_arm_standard_library_separately() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn sandbox_cross_assets_preserve_loader_mode_and_validate_deployment_target() {
|
||||
let mut files = vec![
|
||||
("bin/bwrap".into(), native_elf::static_executable()),
|
||||
@@ -312,7 +315,7 @@ fn sandbox_cross_assets_preserve_loader_mode_and_validate_deployment_target() {
|
||||
bytes[18..20].copy_from_slice(&183u16.to_le_bytes());
|
||||
}
|
||||
dever_sandbox::validate_assets_for_target(&files, "linux-aarch64").unwrap();
|
||||
let mut author = Author::new();
|
||||
let mut author = super::targets::cross_author();
|
||||
let inputs = files
|
||||
.iter()
|
||||
.map(|(path, bytes)| {
|
||||
|
||||
@@ -3,7 +3,7 @@ use dever_cli::toolchain::runtime_pack::RuntimePack;
|
||||
use dever_runtime::config::RuntimeProfile;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn cross_author() -> Author {
|
||||
pub(super) fn cross_author() -> Author {
|
||||
let mut author = Author::new();
|
||||
let target = BuildTarget::LinuxAarch64;
|
||||
let input = |name: &str, bytes: Vec<u8>| {
|
||||
@@ -80,6 +80,12 @@ fn host_core_packages_both_native_targets_with_separate_profiles_and_signed_clos
|
||||
manifest
|
||||
.artifacts
|
||||
.iter()
|
||||
.chain(
|
||||
manifest
|
||||
.extensions
|
||||
.iter()
|
||||
.flat_map(|extension| &extension.artifacts)
|
||||
)
|
||||
.any(|artifact| artifact.path == input.path
|
||||
&& artifact.sha256 == input.sha256
|
||||
&& artifact.bytes == input.bytes)
|
||||
|
||||
@@ -245,6 +245,29 @@ fn package_add_resolves_transitive_versions_and_feeds_the_common_lib_lock() {
|
||||
assert!(String::from_utf8_lossy(&files["module/catalog/value/app.dever"]).contains("two"));
|
||||
assert!(execute_with("doctor", root.path(), &[], ®istry, &store).is_ok());
|
||||
|
||||
let lock_before = fs::read(root.path().join("dever.lock")).unwrap();
|
||||
let setting_before = fs::read(root.path().join("config/setting.json")).unwrap();
|
||||
let archives_only = Registry(
|
||||
registry
|
||||
.0
|
||||
.iter()
|
||||
.filter(|(path, _)| path.ends_with(".zip"))
|
||||
.map(|(path, bytes)| (path.clone(), bytes.clone()))
|
||||
.collect(),
|
||||
);
|
||||
let restored = FixtureArtifactStore::default();
|
||||
dever_cli::packages::restore_locked_with(root.path(), &lock, &archives_only, &restored)
|
||||
.unwrap();
|
||||
assert_eq!(owned_files_with(root.path(), &restored).unwrap(), files);
|
||||
assert_eq!(
|
||||
fs::read(root.path().join("dever.lock")).unwrap(),
|
||||
lock_before
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(root.path().join("config/setting.json")).unwrap(),
|
||||
setting_before
|
||||
);
|
||||
|
||||
let mut damaged = lock.clone();
|
||||
damaged.packages[0].manifest_sha256 = "0".repeat(64);
|
||||
damaged.write_atomic(root.path()).unwrap();
|
||||
|
||||
@@ -140,9 +140,9 @@ fn official_asset_names_are_versioned_and_platform_specific() {
|
||||
)
|
||||
);
|
||||
assert_eq!(
|
||||
source.asset("1.2.3", "tar.gz"),
|
||||
source.asset("1.2.3", "tar.zst"),
|
||||
format!(
|
||||
"{OFFICIAL_RELEASES}/download/v1.2.3/dever-{}.tar.gz",
|
||||
"{OFFICIAL_RELEASES}/download/v1.2.3/dever-{}.tar.zst",
|
||||
platform_identity()
|
||||
)
|
||||
);
|
||||
@@ -219,9 +219,133 @@ fn compressed_release(core: &[u8]) -> Vec<u8> {
|
||||
("dever-core", core),
|
||||
("skills/dever-language/SKILL.md", b"skill"),
|
||||
]);
|
||||
let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
|
||||
gzip.write_all(&bytes).unwrap();
|
||||
gzip.finish().unwrap()
|
||||
compressed(&bytes)
|
||||
}
|
||||
|
||||
fn compressed(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap();
|
||||
encoder.include_checksum(true).unwrap();
|
||||
encoder.write_all(bytes).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zstandard_frames_require_checksum_bounded_window_and_exact_eof() {
|
||||
let tar = archive(&[("dever-core", b"core")]);
|
||||
let good = compressed(&tar);
|
||||
let root = Download::new(&std::env::temp_dir()).unwrap();
|
||||
super::super::release_archive::extract(good.as_slice(), &root.0, &manifest().artifacts)
|
||||
.unwrap();
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
assert_eq!(
|
||||
fs::metadata(root.0.join("dever-core"))
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o755,
|
||||
"signed executable roles retain execution after extraction"
|
||||
);
|
||||
}
|
||||
let mut bad_checksum = good.clone();
|
||||
*bad_checksum.last_mut().unwrap() ^= 1;
|
||||
let mut concatenated = good.clone();
|
||||
concatenated.extend(&good);
|
||||
let mut garbage = good.clone();
|
||||
garbage.push(0);
|
||||
let oversized_window = vec![
|
||||
0x28,
|
||||
0xb5,
|
||||
0x2f,
|
||||
0xfd,
|
||||
4,
|
||||
(28 - 10) << 3,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
];
|
||||
for bytes in [
|
||||
bad_checksum,
|
||||
concatenated,
|
||||
garbage,
|
||||
oversized_window,
|
||||
good[..good.len() - 1].to_vec(),
|
||||
zstd::stream::encode_all(tar.as_slice(), 1).unwrap(),
|
||||
] {
|
||||
let root = Download::new(&std::env::temp_dir()).unwrap();
|
||||
assert!(
|
||||
super::super::release_archive::extract(
|
||||
bytes.as_slice(),
|
||||
&root.0,
|
||||
&manifest().artifacts
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tar_trailers_and_same_sized_corruption_are_rejected() {
|
||||
let tar = archive(&[("dever-core", b"core")]);
|
||||
let mut nonzero_tail = tar.clone();
|
||||
nonzero_tail.push(1);
|
||||
let mut nonzero_padding = tar.clone();
|
||||
nonzero_padding[516] = 1;
|
||||
for bytes in [
|
||||
nonzero_tail,
|
||||
nonzero_padding,
|
||||
tar[..tar.len() - 512].to_vec(),
|
||||
archive(&[("dever-core", b"evil")]),
|
||||
] {
|
||||
let root = Download::new(&std::env::temp_dir()).unwrap();
|
||||
assert!(
|
||||
super::super::release_archive::extract(
|
||||
compressed(&bytes).as_slice(),
|
||||
&root.0,
|
||||
&manifest().artifacts
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signed_extensions_cannot_duplicate_or_escape_their_typed_namespace() {
|
||||
use super::super::{BuildTarget, Extension, ExtensionKind};
|
||||
use crate::libs::Ecosystem;
|
||||
let mut catalog = manifest();
|
||||
catalog.extensions.push(Extension {
|
||||
kind: ExtensionKind::Runtime(Ecosystem::Pip),
|
||||
target: BuildTarget::LinuxX86_64,
|
||||
artifacts: vec![Artifact {
|
||||
path: "runtime/pip/linux-x86_64/runtime.pack".into(),
|
||||
bytes: 4,
|
||||
sha256: catalog.artifacts[0].sha256.clone(),
|
||||
}],
|
||||
});
|
||||
validate_catalog(&catalog).unwrap();
|
||||
assert_eq!(
|
||||
catalog.extensions[0].id().asset_suffix(),
|
||||
"ext-runtime-pip-linux-x86_64.tar.zst"
|
||||
);
|
||||
let mut duplicate = catalog.clone();
|
||||
duplicate.extensions.push(duplicate.extensions[0].clone());
|
||||
assert!(validate_catalog(&duplicate).is_err());
|
||||
for path in [
|
||||
"dever-core",
|
||||
"runtime/npm/linux-x86_64/runtime.pack",
|
||||
"runtime/pip/linux-aarch64/runtime.pack",
|
||||
"runtime/pip/linux-x86_64/../escape",
|
||||
] {
|
||||
catalog.extensions[0].artifacts[0].path = path.into();
|
||||
assert!(validate_catalog(&catalog).is_err(), "{path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -18,6 +18,10 @@ use sha2::{Digest, Sha256};
|
||||
|
||||
static NEXT_TEMPORARY: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[cfg(unix)]
|
||||
#[path = "shared_toolchain_extensions.rs"]
|
||||
mod extensions;
|
||||
|
||||
struct TemporaryDirectory(PathBuf);
|
||||
|
||||
impl TemporaryDirectory {
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
use super::*;
|
||||
use dever_cli::libs::Ecosystem;
|
||||
use dever_cli::toolchain::{
|
||||
BuildTarget, Extension, ExtensionId, ExtensionKind, ExtensionSource, SignedResources,
|
||||
artifact_get_optional, prepare_extension_with,
|
||||
};
|
||||
use std::sync::atomic::AtomicUsize;
|
||||
|
||||
fn id(ecosystem: Ecosystem) -> ExtensionId {
|
||||
ExtensionId {
|
||||
kind: ExtensionKind::Runtime(ecosystem),
|
||||
target: BuildTarget::host().unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
fn logical_path(id: &ExtensionId) -> String {
|
||||
let ExtensionKind::Runtime(ecosystem) = &id.kind else {
|
||||
panic!("runtime fixture")
|
||||
};
|
||||
format!(
|
||||
"runtime/{}/{}/runtime.pack",
|
||||
ecosystem.as_str(),
|
||||
id.target.platform()
|
||||
)
|
||||
}
|
||||
|
||||
struct Source {
|
||||
requests: AtomicUsize,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
impl Source {
|
||||
fn new(fail: bool) -> Self {
|
||||
Self {
|
||||
requests: AtomicUsize::new(0),
|
||||
fail,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExtensionSource for Source {
|
||||
fn open(&self, version: &Version, extension: &ExtensionId) -> Result<Box<dyn Read>, String> {
|
||||
self.requests.fetch_add(1, Ordering::Relaxed);
|
||||
if self.fail {
|
||||
return Err("fixture extension download failed".into());
|
||||
}
|
||||
let mut tar = tar::Builder::new(Vec::new());
|
||||
let mut header = tar::Header::new_ustar();
|
||||
header.set_size(version.as_str().len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
tar.append_data(
|
||||
&mut header,
|
||||
logical_path(extension),
|
||||
version.as_str().as_bytes(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut encoder = zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap();
|
||||
encoder.include_checksum(true).unwrap();
|
||||
encoder.write_all(&tar.into_inner().unwrap()).unwrap();
|
||||
Ok(Box::new(std::io::Cursor::new(encoder.finish().unwrap())))
|
||||
}
|
||||
}
|
||||
|
||||
fn release(layout: &Layout, key: &Ed25519KeyPair, version: &str) {
|
||||
write_release(
|
||||
layout,
|
||||
key,
|
||||
version,
|
||||
&layout.root().join("dispatch"),
|
||||
true,
|
||||
true,
|
||||
);
|
||||
let directory = layout.downloads().join(version);
|
||||
let path = directory.join("manifest.json");
|
||||
let mut manifest: ReleaseManifest = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
|
||||
for ecosystem in [Ecosystem::Pip, Ecosystem::Npm] {
|
||||
let id = id(ecosystem);
|
||||
manifest.extensions.push(Extension {
|
||||
kind: id.kind.clone(),
|
||||
target: id.target,
|
||||
artifacts: vec![Artifact {
|
||||
path: logical_path(&id),
|
||||
bytes: version.len() as u64,
|
||||
sha256: hex(&Sha256::digest(version.as_bytes())),
|
||||
}],
|
||||
});
|
||||
}
|
||||
let bytes = serde_json::to_vec(&manifest).unwrap();
|
||||
fs::write(path, &bytes).unwrap();
|
||||
fs::write(
|
||||
directory.join("manifest.sig"),
|
||||
hex(key.sign(&bytes).as_ref()),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn machine() -> (TemporaryDirectory, Layout, MachineManager, Ed25519KeyPair) {
|
||||
let root = TemporaryDirectory::new();
|
||||
let layout = Layout::new(root.path().join("machine"));
|
||||
layout.initialize().unwrap();
|
||||
let key = signing_key();
|
||||
fs::write(
|
||||
layout.state().join("trusted-release-key"),
|
||||
hex(key.public_key().as_ref()),
|
||||
)
|
||||
.unwrap();
|
||||
let manager = MachineManager::new(layout.clone());
|
||||
(root, layout, manager, key)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signed_extension_preparation_is_shared_immutable_and_concurrent() {
|
||||
let (_root, layout, manager, key) = machine();
|
||||
release(&layout, &key, "1.0.0");
|
||||
let version = manager.install_requested("1.0.0").unwrap();
|
||||
let extension = id(Ecosystem::Pip);
|
||||
let logical = logical_path(&extension);
|
||||
let resources = SignedResources::load(&layout, &version).unwrap();
|
||||
assert!(
|
||||
resources
|
||||
.read(&logical)
|
||||
.unwrap_err()
|
||||
.contains("dever lib install")
|
||||
);
|
||||
let source = Source::new(false);
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..2 {
|
||||
scope.spawn(|| prepare_extension_with(&layout, &version, &extension, &source).unwrap());
|
||||
}
|
||||
});
|
||||
assert_eq!(source.requests.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(resources.read(&logical).unwrap(), b"1.0.0");
|
||||
let path = resources.path(&logical).unwrap();
|
||||
assert!(path.starts_with(layout.extensions()));
|
||||
assert!(!layout.versions().join("1.0.0").join(&logical).exists());
|
||||
fs::write(&path, b"other").unwrap();
|
||||
assert!(resources.read(&logical).unwrap_err().contains("integrity"));
|
||||
assert!(
|
||||
prepare_extension_with(&layout, &version, &extension, &source)
|
||||
.unwrap_err()
|
||||
.contains("integrity")
|
||||
);
|
||||
assert_eq!(source.requests.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_prepares_only_installed_extensions_and_preserves_active_on_failure() {
|
||||
let (_root, layout, manager, key) = machine();
|
||||
for version in ["1.0.0", "2.0.0"] {
|
||||
release(&layout, &key, version);
|
||||
}
|
||||
let old = manager.install_requested("1.0.0").unwrap();
|
||||
let extension = id(Ecosystem::Pip);
|
||||
prepare_extension_with(&layout, &old, &extension, &Source::new(false)).unwrap();
|
||||
fs::write(layout.downloads().join("latest"), "2.0.0").unwrap();
|
||||
let journal = fs::read(layout.state().join("active-version")).unwrap();
|
||||
assert!(
|
||||
manager
|
||||
.update_with(&Source::new(true))
|
||||
.unwrap_err()
|
||||
.contains("download failed")
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(layout.state().join("active-version")).unwrap(),
|
||||
journal
|
||||
);
|
||||
assert_eq!(
|
||||
SignedResources::load(&layout, &old)
|
||||
.unwrap()
|
||||
.read(&logical_path(&extension))
|
||||
.unwrap(),
|
||||
b"1.0.0"
|
||||
);
|
||||
let source = Source::new(false);
|
||||
let new = manager.update_with(&source).unwrap();
|
||||
assert_eq!(new.as_str(), "2.0.0");
|
||||
assert_eq!(source.requests.load(Ordering::Relaxed), 1);
|
||||
let resources = SignedResources::load(&layout, &new).unwrap();
|
||||
assert_eq!(resources.read(&logical_path(&extension)).unwrap(), b"2.0.0");
|
||||
assert!(
|
||||
resources
|
||||
.read(&logical_path(&id(Ecosystem::Npm)))
|
||||
.unwrap_err()
|
||||
.contains("not installed")
|
||||
);
|
||||
assert_eq!(manager.active_version().unwrap(), Some(new));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optional_artifact_ipc_distinguishes_absence_corruption_and_service_failure() {
|
||||
let (_root, layout, _manager, _key) = machine();
|
||||
let mut daemon = TestDaemon::start(&layout);
|
||||
let digest = hex(&Sha256::digest(b"payload"));
|
||||
assert!(
|
||||
artifact_get_optional(&layout, &digest, 7)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
artifact_put(&layout, b"payload").unwrap();
|
||||
assert_eq!(
|
||||
artifact_get_optional(&layout, &digest, 7).unwrap().unwrap(),
|
||||
b"payload"
|
||||
);
|
||||
fs::write(
|
||||
layout.native_cache().join("artifacts").join(&digest),
|
||||
b"changed",
|
||||
)
|
||||
.unwrap();
|
||||
assert!(
|
||||
artifact_get_optional(&layout, &digest, 7)
|
||||
.unwrap_err()
|
||||
.contains("SHA-256")
|
||||
);
|
||||
daemon.stop();
|
||||
assert!(artifact_get_optional(&layout, &digest, 7).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_ipc_rejects_caller_paths_and_invalid_typed_identity() {
|
||||
let (_root, layout, _manager, _key) = machine();
|
||||
let _daemon = TestDaemon::start(&layout);
|
||||
let token = fs::read_to_string(layout.service_token()).unwrap();
|
||||
for selection in [
|
||||
serde_json::json!({"version":"1.0.0","extension":{"kind":{"type":"runtime","ecosystem":"pip"},"target":"linux-x86_64"},"url":"http://localhost/"}),
|
||||
serde_json::json!({"version":"../../outside","extension":{"kind":{"type":"target"},"target":"linux-aarch64"}}),
|
||||
serde_json::json!({"version":"1.0.0","extension":{"kind":{"type":"shell"},"target":"linux-x86_64"}}),
|
||||
] {
|
||||
let result = raw_operation(
|
||||
&layout,
|
||||
token.trim(),
|
||||
serde_json::json!({"ensure_extension":selection}),
|
||||
);
|
||||
assert_eq!(result["ok"], false, "{result}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
//! Opt-in actual-asset acceptance: only the download transport is local.
|
||||
|
||||
use std::fs::{self, File};
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use dever_cli::toolchain::{
|
||||
BuildTarget, ExtensionId, ExtensionSource, Layout, MachineManager, ReleaseManifest,
|
||||
SignedResources, Version, prepare_extension_with, validate_catalog,
|
||||
};
|
||||
use ring::signature::Ed25519KeyPair;
|
||||
|
||||
struct LocalAssets(PathBuf);
|
||||
|
||||
impl ExtensionSource for LocalAssets {
|
||||
fn open(&self, _: &Version, extension: &ExtensionId) -> Result<Box<dyn Read>, String> {
|
||||
let name = format!(
|
||||
"dever-{}.{}",
|
||||
BuildTarget::host()?.platform(),
|
||||
extension.asset_suffix()
|
||||
);
|
||||
File::open(self.0.join(name))
|
||||
.map(|file| Box::new(file) as Box<dyn Read>)
|
||||
.map_err(|error| format!("cannot open actual extension asset: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
let arguments: Vec<_> = std::env::args().skip(1).collect();
|
||||
if let [operation, manifest, key] = arguments.as_slice()
|
||||
&& operation == "--sign-catalog"
|
||||
{
|
||||
let bytes = fs::read(manifest).map_err(|error| error.to_string())?;
|
||||
let catalog: ReleaseManifest =
|
||||
serde_json::from_slice(&bytes).map_err(|error| error.to_string())?;
|
||||
validate_catalog(&catalog)?;
|
||||
let key = fs::read(key).map_err(|error| error.to_string())?;
|
||||
let pair = Ed25519KeyPair::from_pkcs8(&key).map_err(|error| error.to_string())?;
|
||||
let signature: String = pair
|
||||
.sign(&bytes)
|
||||
.as_ref()
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
return fs::write(
|
||||
Path::new(manifest).with_file_name("manifest.sig"),
|
||||
signature,
|
||||
)
|
||||
.map_err(|error| error.to_string());
|
||||
}
|
||||
let [machine, assets, version, names @ ..] = arguments.as_slice() else {
|
||||
return Err(
|
||||
"usage: prepare-extensions <machine> <assets> <version> <extension-name>...".into(),
|
||||
);
|
||||
};
|
||||
if names.is_empty() {
|
||||
return Err("at least one signed extension must be selected".into());
|
||||
}
|
||||
let layout = Layout::new(machine);
|
||||
let version = Version::parse(version)?;
|
||||
let source = LocalAssets(PathBuf::from(assets));
|
||||
if names == ["--update"] {
|
||||
let installed = MachineManager::new(layout).update_with(&source)?;
|
||||
if installed != version {
|
||||
return Err(format!(
|
||||
"expected update to {version}, received {installed}"
|
||||
));
|
||||
}
|
||||
println!("updated {installed}");
|
||||
return Ok(());
|
||||
}
|
||||
let resources = SignedResources::load(&layout, &version)?;
|
||||
for name in names {
|
||||
let extension = resources
|
||||
.manifest()
|
||||
.extensions
|
||||
.iter()
|
||||
.find(|extension| extension.id().name() == *name)
|
||||
.ok_or_else(|| format!("signed catalog does not declare extension '{name}'"))?;
|
||||
prepare_extension_with(&layout, &version, &extension.id(), &source)?;
|
||||
println!("prepared {}", extension.id().name());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("prepare-extensions: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import gzip
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
@@ -40,6 +39,7 @@ class InstallerTests(unittest.TestCase):
|
||||
self.payloads = {
|
||||
"dever-core": b"fixture core, never executed\n",
|
||||
"bootstrap/dever": b"fixture launcher, never executed\n",
|
||||
"bootstrap/lib/libfixture.so": b"fixture private closure\n",
|
||||
"skills/dever-language/SKILL.md": b"fixture skill\n",
|
||||
}
|
||||
self.catalog = {}
|
||||
@@ -52,8 +52,9 @@ class InstallerTests(unittest.TestCase):
|
||||
"sha256": hashlib.sha256(content).hexdigest(),
|
||||
}
|
||||
manifest = {
|
||||
"format": "dever-release-v1", "platform": self.platform,
|
||||
"format": "dever-release-v2", "platform": self.platform,
|
||||
"version": "1.2.3", "artifacts": list(self.catalog.values()),
|
||||
"extensions": [],
|
||||
}
|
||||
(self.release / "manifest.json").write_text(json.dumps(manifest))
|
||||
self.private = self.root / "test-key.pem"
|
||||
@@ -82,24 +83,14 @@ class InstallerTests(unittest.TestCase):
|
||||
return installer.authenticate(self.release, key or self.public, self.scratch,
|
||||
self.platform, "1.2.3")
|
||||
|
||||
def archive(self, entries, trailer=None):
|
||||
stream = io.BytesIO()
|
||||
with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive:
|
||||
for name, content, kind in entries:
|
||||
member = tarfile.TarInfo(name)
|
||||
member.type = kind
|
||||
member.size = len(content) if kind == tarfile.REGTYPE else 0
|
||||
if kind == tarfile.SYMTYPE:
|
||||
member.linkname = "../../outside"
|
||||
archive.addfile(member, io.BytesIO(content) if member.size else None)
|
||||
contents = stream.getvalue()
|
||||
if trailer is not None:
|
||||
contents += trailer
|
||||
return gzip.compress(contents)
|
||||
def create_assets(self, output):
|
||||
return assets.create(self.release, output, Path("/usr/bin/zstd"))
|
||||
|
||||
def extract(self, compressed, destination):
|
||||
with mock.patch.object(installer, "download", return_value=io.BytesIO(compressed)):
|
||||
installer.payload(destination, None, "1.2.3", self.platform, self.catalog)
|
||||
def archive_payloads(self, path):
|
||||
contents = subprocess.run(["/usr/bin/zstd", "-q", "-d", "-c", str(path)],
|
||||
env={}, capture_output=True, check=True, timeout=15).stdout
|
||||
with tarfile.open(fileobj=io.BytesIO(contents)) as archive:
|
||||
return {member.name: archive.extractfile(member).read() for member in archive}
|
||||
|
||||
def test_signature_accepts_independent_key_and_rejects_wrong_key_or_changed_manifest(self):
|
||||
version, catalog, key = self.authenticate()
|
||||
@@ -119,19 +110,47 @@ class InstallerTests(unittest.TestCase):
|
||||
first = self.root / "first"
|
||||
second = self.root / "second"
|
||||
(self.release / "unlisted").write_text("not published")
|
||||
names = assets.create(self.release, first)
|
||||
self.assertEqual(names, assets.create(self.release, second))
|
||||
self.assertEqual(len(names), 3)
|
||||
names = self.create_assets(first)
|
||||
self.assertEqual(names, self.create_assets(second))
|
||||
self.assertEqual(len(names), 5)
|
||||
for name in names:
|
||||
self.assertEqual((first / name).read_bytes(), (second / name).read_bytes())
|
||||
extracted = self.directory("extracted")
|
||||
self.extract((first / f"dever-{self.platform}.tar.gz").read_bytes(), extracted)
|
||||
actual = {str(path.relative_to(extracted)): path.read_bytes()
|
||||
for path in extracted.rglob("*") if path.is_file()}
|
||||
actual = self.archive_payloads(first / f"dever-{self.platform}.tar.zst")
|
||||
self.assertEqual(actual, self.payloads)
|
||||
self.assertEqual((extracted / "bootstrap/dever").stat().st_mode & 0o777, 0o755)
|
||||
for name in ("bootstrap/dever", "bootstrap/lib/libfixture.so"):
|
||||
blob = first / f"dever-{self.platform}.blob-{self.catalog[name]['sha256']}"
|
||||
self.assertEqual(blob.read_bytes(), self.payloads[name])
|
||||
with self.assertRaisesRegex(ValueError, "already exists"):
|
||||
assets.create(self.release, first)
|
||||
self.create_assets(first)
|
||||
|
||||
def test_extension_assets_are_separate_and_cannot_overlap_base(self):
|
||||
manifest_path = self.release / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
name = f"runtime/pip/{self.platform}/runtime.pack"
|
||||
path = self.release / name
|
||||
path.parent.mkdir(parents=True)
|
||||
contents = b"optional Python bytes"
|
||||
path.write_bytes(contents)
|
||||
entry = {"path": name, "bytes": len(contents), "sha256": hashlib.sha256(contents).hexdigest()}
|
||||
manifest["extensions"] = [{"kind": {"type": "runtime", "ecosystem": "pip"},
|
||||
"target": self.platform, "artifacts": [entry]}]
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
self.sign_manifest()
|
||||
self.authenticate()
|
||||
output = self.root / "split"
|
||||
self.create_assets(output)
|
||||
self.assertEqual(self.archive_payloads(output / f"dever-{self.platform}.tar.zst"), self.payloads)
|
||||
extension = output / f"dever-{self.platform}.ext-runtime-pip-{self.platform}.tar.zst"
|
||||
self.assertEqual(self.archive_payloads(extension), {name: contents})
|
||||
manifest["extensions"][0]["artifacts"].append(self.catalog["dever-core"])
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, "resource namespace"):
|
||||
self.create_assets(self.root / "overlap")
|
||||
manifest["extensions"] = [{"kind": {"type": "target"}, "target": self.platform,
|
||||
"artifacts": [self.catalog["dever-core"]]}]
|
||||
manifest_path.write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, "base package"):
|
||||
self.create_assets(self.root / "host-target")
|
||||
|
||||
def test_signed_release_without_matching_skill_is_rejected(self):
|
||||
path = self.release / "manifest.json"
|
||||
@@ -179,7 +198,7 @@ class InstallerTests(unittest.TestCase):
|
||||
|
||||
with mock.patch.object(assets, "publish", side_effect=publish), \
|
||||
self.assertRaises(OSError):
|
||||
assets.create(self.release, output)
|
||||
self.create_assets(output)
|
||||
self.assertTrue(output.is_dir())
|
||||
if populated:
|
||||
self.assertEqual((output / "user-file").read_bytes(), b"concurrent user bytes")
|
||||
@@ -187,23 +206,48 @@ class InstallerTests(unittest.TestCase):
|
||||
self.assertEqual(list(output.iterdir()), [])
|
||||
self.assertEqual(list(self.root.glob(".dever-release-assets-*")), [])
|
||||
|
||||
def test_archive_rejects_unsigned_duplicate_symlink_wrong_size_missing_and_bad_tail(self):
|
||||
normal = [(name, content, tarfile.REGTYPE) for name, content in self.payloads.items()]
|
||||
cases = {
|
||||
"unsigned": normal + [("outside", b"extra", tarfile.REGTYPE)],
|
||||
"duplicate": normal + [normal[0]],
|
||||
"symlink": [(normal[0][0], b"", tarfile.SYMTYPE)] + normal[1:],
|
||||
"size": [(normal[0][0], b"short", tarfile.REGTYPE)] + normal[1:],
|
||||
"missing": normal[:-1],
|
||||
}
|
||||
for name, entries in cases.items():
|
||||
with self.subTest(name=name), self.assertRaises(ValueError):
|
||||
self.extract(self.archive(entries), self.directory(name))
|
||||
with self.assertRaisesRegex(ValueError, "尾部"):
|
||||
self.extract(self.archive(normal, b"hidden"), self.directory("tail"))
|
||||
corrupted = [(normal[0][0], b"x" * len(normal[0][1]), tarfile.REGTYPE)] + normal[1:]
|
||||
with self.assertRaisesRegex(ValueError, "摘要"):
|
||||
self.extract(self.archive(corrupted), self.directory("digest"))
|
||||
def test_network_payload_executes_only_verified_helper_with_private_libraries(self):
|
||||
assets_path = self.root / "assets"
|
||||
self.create_assets(assets_path)
|
||||
calls = []
|
||||
|
||||
def download(address):
|
||||
return (assets_path / address.rsplit("/", 1)[-1]).open("rb")
|
||||
|
||||
def extract(command, **kwargs):
|
||||
self.assertEqual(command[1], "--dever-extract")
|
||||
launcher = Path(command[0])
|
||||
self.assertEqual(launcher.read_bytes(), self.payloads["bootstrap/dever"])
|
||||
self.assertEqual((launcher.parent / "lib/libfixture.so").read_bytes(),
|
||||
self.payloads["bootstrap/lib/libfixture.so"])
|
||||
self.assertEqual(launcher.stat().st_mode & 0o777, 0o755)
|
||||
configuration = json.loads(Path(command[2]).read_text())
|
||||
self.assertEqual(configuration["trusted_key"], str(self.public))
|
||||
self.assertEqual(kwargs["env"], {})
|
||||
self.assertTrue(Path(configuration["archive"]).is_file())
|
||||
calls.append(command)
|
||||
|
||||
with mock.patch.object(installer, "download", side_effect=download), \
|
||||
mock.patch.object(installer.subprocess, "run", side_effect=extract):
|
||||
installer.payload(self.directory("network"), None, "1.2.3", self.platform,
|
||||
self.catalog, self.public)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(list(self.root.glob(".extract-*")), [])
|
||||
|
||||
blob = assets_path / f"dever-{self.platform}.blob-{self.catalog['bootstrap/dever']['sha256']}"
|
||||
for corrupt in (b"x" * len(self.payloads["bootstrap/dever"]), self.payloads["bootstrap/dever"] + b"extra"):
|
||||
blob.write_bytes(corrupt)
|
||||
with mock.patch.object(installer, "download", side_effect=download), \
|
||||
mock.patch.object(installer.subprocess, "run") as execute, \
|
||||
self.assertRaises(ValueError):
|
||||
installer.payload(self.release, None, "1.2.3", self.platform, self.catalog, self.public)
|
||||
execute.assert_not_called()
|
||||
|
||||
def test_download_archive_is_bounded(self):
|
||||
with mock.patch.object(installer, "MAX_RELEASE", 4), \
|
||||
mock.patch.object(installer, "download", return_value=io.BytesIO(b"12345")), \
|
||||
self.assertRaisesRegex(ValueError, "超过"):
|
||||
installer.download_archive("unused", self.root / "oversized.zst")
|
||||
|
||||
def test_local_payload_rejects_symlink_and_source_directory(self):
|
||||
copied = self.directory("copied")
|
||||
@@ -215,7 +259,7 @@ class InstallerTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, "符号链接"):
|
||||
installer.payload(self.directory("symlink"), self.release, "1.2.3", self.platform, self.catalog)
|
||||
with self.assertRaisesRegex(ValueError, "symlinks"):
|
||||
assets.create(self.release, self.root / "invalid-assets")
|
||||
self.create_assets(self.root / "invalid-assets")
|
||||
core.unlink()
|
||||
core.mkdir()
|
||||
with self.assertRaisesRegex(ValueError, "类型"):
|
||||
|
||||
@@ -33,8 +33,8 @@ def execute(arguments, timeout=180):
|
||||
return completed.stdout.strip()
|
||||
|
||||
|
||||
def response(output):
|
||||
expected = {"code": 0, "message": "ok", "data": "Hello, Dever"}
|
||||
def response(output, data="Hello, Dever"):
|
||||
expected = {"code": 0, "message": "ok", "data": data}
|
||||
if json.loads(output) != expected:
|
||||
raise RuntimeError(f"unexpected application response: {output}")
|
||||
|
||||
@@ -50,30 +50,56 @@ def minimal_os(image):
|
||||
(image / name).mkdir()
|
||||
|
||||
|
||||
def isolated(image):
|
||||
def worker_project(image, ecosystem):
|
||||
"""Small no-Lib Adapter: preparation must still acquire its managed runtime."""
|
||||
entries = {
|
||||
"pip": ("py", "import gzip\n\nasync def probe(payload, setting):\n return {\"okay\": gzip.decompress(gzip.compress(b'dever')) == b'dever'}\n"),
|
||||
"npm": ("mjs", "import {createHash} from 'node:crypto';\nexport async function probe() { return {okay:createHash('sha256').update('dever').digest().length === 32}; }\n"),
|
||||
"go": ("go", (WORKSPACE / "test/ecosystem-release/probe.go").read_text()),
|
||||
}
|
||||
extension, source = entries[ecosystem]
|
||||
root = image / f"worker-{ecosystem}"
|
||||
files = {
|
||||
"config/setting.json": "{}\n",
|
||||
"module/sample/worker/port.dever": "probe() (okay: Bool) fails app.ProbeResult\n",
|
||||
"module/sample/worker/app.dever": "type ProbeResult { error Unavailable(message: Text) }\nprobe() (okay: Bool) { okay = port.probe() }\n",
|
||||
"module/sample/worker/api.dever": "cmd probe = app.probe\n",
|
||||
"module/sample/worker/adapter.dever": f'external {ecosystem} "worker/probe.{extension}" {{}}\n',
|
||||
f"module/sample/worker/worker/probe.{extension}": source,
|
||||
}
|
||||
for relative, contents in files.items():
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(contents)
|
||||
return root
|
||||
|
||||
|
||||
def isolated(image, init):
|
||||
return [
|
||||
WORKSPACE / "target/sandbox-inputs/assets/bin/bwrap",
|
||||
"--unshare-all", "--die-with-parent", "--new-session", "--bind", image,
|
||||
"/", "--proc", "/proc", "--dev", "/dev", "--chdir", "/", "--ro-bind",
|
||||
WORKSPACE / "target/debug/examples/native-acceptance-init", "/fixture-proc-init",
|
||||
init, "/fixture-proc-init",
|
||||
]
|
||||
|
||||
|
||||
class InstalledDaemon:
|
||||
"""All clients enter the same PID namespace so SO_PEERCRED stays valid."""
|
||||
|
||||
def __init__(self, image, records):
|
||||
def __init__(self, image, records, init):
|
||||
self.image = image
|
||||
self.records = records
|
||||
self.init = init
|
||||
self.child = None
|
||||
self.pidfd = None
|
||||
self.pid = None
|
||||
|
||||
def __enter__(self):
|
||||
self.records.mkdir(parents=True, exist_ok=True)
|
||||
self.information = (self.records / "namespace.json").open("w+")
|
||||
self.errors = (self.records / "daemon.log").open("w+")
|
||||
self.child = subprocess.Popen(
|
||||
[str(value) for value in isolated(self.image) + [
|
||||
[str(value) for value in isolated(self.image, self.init) + [
|
||||
"--info-fd", "1", "--", "/fixture-proc-init",
|
||||
"/opt/dever/bin/deverd", "--root", "/opt/dever",
|
||||
]], env={}, stdout=self.information, stderr=self.errors,
|
||||
@@ -134,6 +160,38 @@ class InstalledDaemon:
|
||||
self.errors.close()
|
||||
|
||||
|
||||
def ordinary_uid_target_reuse(image):
|
||||
# bwrap's single-UID user namespace cannot represent a second identity.
|
||||
# Use the installed private machine with real kernel UIDs for this check;
|
||||
# no host language tools or global service participate in target reuse.
|
||||
machine = image / "opt/dever"
|
||||
child = subprocess.Popen([str(machine / "bin/deverd"), "--root", str(machine)],
|
||||
env={}, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
||||
try:
|
||||
deadline = time.monotonic() + 10
|
||||
while not (machine / "state/deverd.sock").exists():
|
||||
if child.poll() is not None or time.monotonic() >= deadline:
|
||||
raise RuntimeError("private UID acceptance daemon did not become ready")
|
||||
time.sleep(0.02)
|
||||
for uid in (65533, 65534):
|
||||
completed = subprocess.run(
|
||||
[str(machine / "bin/dever"), "target", "add", "linux-aarch64"],
|
||||
env={}, user=uid, group=uid, extra_groups=[], capture_output=True,
|
||||
text=True, timeout=180, check=False,
|
||||
)
|
||||
if completed.returncode:
|
||||
raise RuntimeError(f"UID {uid} target reuse failed: {completed.stderr}")
|
||||
finally:
|
||||
if child.poll() is None:
|
||||
child.terminate()
|
||||
try:
|
||||
child.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
child.kill()
|
||||
child.wait(timeout=10)
|
||||
child.stderr.close()
|
||||
|
||||
|
||||
def install_assets(assets, image, manifest):
|
||||
# Only transport is local: signature verification, full tar extraction and
|
||||
# the final bootstrap executable are the real first-install implementation.
|
||||
@@ -146,8 +204,12 @@ def install_assets(assets, image, manifest):
|
||||
routes = {
|
||||
f"{base}/latest/download/{stem}.manifest.json": assets / f"{stem}.manifest.json",
|
||||
f"{base}/latest/download/{stem}.manifest.sig": assets / f"{stem}.manifest.sig",
|
||||
f"{base}/download/v{manifest['version']}/{stem}.tar.gz": assets / f"{stem}.tar.gz",
|
||||
f"{base}/download/v{manifest['version']}/{stem}.tar.zst": assets / f"{stem}.tar.zst",
|
||||
}
|
||||
for artifact in manifest["artifacts"]:
|
||||
if artifact["path"] == "bootstrap/dever" or artifact["path"].startswith("bootstrap/lib/"):
|
||||
name = f"{stem}.blob-{artifact['sha256']}"
|
||||
routes[f"{base}/download/v{manifest['version']}/{name}"] = assets / name
|
||||
installer.download = lambda address: routes[address].open("rb")
|
||||
installer.install(argparse.Namespace(
|
||||
version="latest", release=None, system_root=image, no_start=True,
|
||||
@@ -155,7 +217,7 @@ def install_assets(assets, image, manifest):
|
||||
))
|
||||
|
||||
|
||||
def accept(release, assets, image, report):
|
||||
def accept(release, assets, image, report, init, extension_helper=None):
|
||||
image.mkdir(mode=0o755)
|
||||
report.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest = json.loads((release / "manifest.json").read_bytes())
|
||||
@@ -185,13 +247,19 @@ def accept(release, assets, image, report):
|
||||
if (installed / "manifest.json").read_bytes() != (release / "manifest.json").read_bytes():
|
||||
raise RuntimeError("installed package differs from the release under acceptance")
|
||||
passed("actual_signed_release_first_install")
|
||||
for extension in manifest["extensions"]:
|
||||
if any((installed / artifact["path"]).exists() for artifact in extension["artifacts"]):
|
||||
raise RuntimeError("base install unexpectedly includes optional extension bytes")
|
||||
passed("base_install_excludes_optional_extensions")
|
||||
launcher = image / "opt/dever/bin/dever"
|
||||
if execute([launcher, "version"]) != manifest["version"]:
|
||||
raise RuntimeError("installed version mismatch")
|
||||
if (image / "usr/local/bin/dever").readlink() != Path("/opt/dever/bin/dever"):
|
||||
raise RuntimeError("public launcher link mismatch")
|
||||
passed("installed_version_and_public_entry")
|
||||
with InstalledDaemon(image, report.parent) as daemon:
|
||||
standalone = [("first", "hello.greeting.greet", '{"name":"Dever"}', "Hello, Dever"),
|
||||
("second", "hello.greeting.greet", '{"name":"Dever"}', "Hello, Dever")]
|
||||
with InstalledDaemon(image, report.parent / f"{report.stem}-process", init) as daemon:
|
||||
for project, markdown in (("first", False), ("second", True)):
|
||||
root = "/" + project
|
||||
args = ["new", root] + (["--markdown"] if markdown else [])
|
||||
@@ -211,27 +279,62 @@ def accept(release, assets, image, report):
|
||||
if not (image / "ai-skill/SKILL.md").is_file():
|
||||
raise RuntimeError("missing installed skill entry")
|
||||
passed("versioned_skill_resolution_and_loader_install")
|
||||
daemon.command("build", "/first", "--target", "linux-aarch64", "--output", "/first/program.arm64")
|
||||
with (image / "first/program.arm64").open("rb") as executable:
|
||||
header = executable.read(20)
|
||||
if header[:4] != b"\x7fELF" or header[18:20] != b"\xb7\x00":
|
||||
raise RuntimeError("cross build did not produce an ARM64 ELF")
|
||||
passed("arm64_application_cross_build")
|
||||
try:
|
||||
daemon.command("build", "/first", "--target", "linux-aarch64", "--output", "/first/program.arm64")
|
||||
except RuntimeError as error:
|
||||
if "dever target add linux-aarch64" not in str(error):
|
||||
raise
|
||||
else:
|
||||
raise RuntimeError("unprepared cross build should fail without fetching resources")
|
||||
passed("missing_target_fails_offline_with_preparation_command")
|
||||
if assets and extension_helper:
|
||||
execute([extension_helper, image / "opt/dever", assets, manifest["version"], "target-linux-aarch64"])
|
||||
daemon.command("target", "add", "linux-aarch64")
|
||||
daemon.command("build", "/first", "--target", "linux-aarch64", "--output", "/first/program.arm64")
|
||||
with (image / "first/program.arm64").open("rb") as executable:
|
||||
header = executable.read(20)
|
||||
if header[:4] != b"\x7fELF" or header[18:20] != b"\xb7\x00":
|
||||
raise RuntimeError("cross build did not produce an ARM64 ELF")
|
||||
passed("actual_target_extension_install_reuse_and_arm64_application_cross_build")
|
||||
for ecosystem in ("pip", "npm", "go"):
|
||||
execute([extension_helper, image / "opt/dever", assets, manifest["version"],
|
||||
f"runtime-{ecosystem}-{manifest['platform']}"])
|
||||
project = worker_project(image, ecosystem)
|
||||
root = "/" + project.name
|
||||
daemon.command("lib", "update", root)
|
||||
before = [(project / name).read_bytes() for name in ("config/setting.json", "dever.lock")]
|
||||
daemon.command("lib", "install", root)
|
||||
after = [(project / name).read_bytes() for name in ("config/setting.json", "dever.lock")]
|
||||
if before != after:
|
||||
raise RuntimeError("exact Lib installation modified project configuration or lock")
|
||||
daemon.command("check", root)
|
||||
response(daemon.command("run", root, "--", "sample.worker.probe", "{}"), True)
|
||||
daemon.command("build", root, "--output", root + "/program")
|
||||
standalone.append((project.name, "sample.worker.probe", "{}", True))
|
||||
passed(f"actual_{ecosystem}_extension_no_lib_worker_exact_install_run_build")
|
||||
result["cache"] = json.loads(daemon.command("cache", "status"))
|
||||
if assets and extension_helper:
|
||||
ordinary_uid_target_reuse(image)
|
||||
passed("two_ordinary_uids_reuse_shared_signed_target_extension")
|
||||
# Remove sources and the entire machine from the visible OS root. Keep
|
||||
# them outside that root for a subsequent online-update acceptance.
|
||||
moved = []
|
||||
try:
|
||||
for relative in ("opt/dever", "first/module", "second/module"):
|
||||
for relative in ["opt/dever"] + [f"{project}/module" for project, *_ in standalone]:
|
||||
source = image / relative
|
||||
hidden = image.parent / ("hidden-" + relative.replace("/", "-"))
|
||||
source.rename(hidden)
|
||||
moved.append((source, hidden))
|
||||
for project in ("first", "second"):
|
||||
response(execute(isolated(image) + [
|
||||
for project, entry, parameters, data in standalone:
|
||||
# These projects were created above; remove only their generated
|
||||
# Worker cache so standalone execution must extract its own bundle.
|
||||
cache = image / project / "data/cache/lib"
|
||||
if cache.exists():
|
||||
shutil.rmtree(cache)
|
||||
response(execute(isolated(image, init) + [
|
||||
"--", "/fixture-proc-init", f"/{project}/program",
|
||||
"hello.greeting.greet", '{"name":"Dever"}',
|
||||
]))
|
||||
entry, parameters,
|
||||
]), data)
|
||||
passed("standalone_programs_without_sources_or_machine")
|
||||
finally:
|
||||
for source, hidden in reversed(moved):
|
||||
@@ -248,14 +351,17 @@ def accept(release, assets, image, report):
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--release", required=True, type=Path)
|
||||
parser.add_argument("--assets", type=Path, help="also exercise the three actual download assets")
|
||||
parser.add_argument("--assets", type=Path, help="exercise actual split download assets")
|
||||
parser.add_argument("--extension-helper", type=Path, help="explicit prepare-extensions author fixture")
|
||||
parser.add_argument("--init", type=Path, required=True, help="explicit static native-acceptance-init fixture")
|
||||
parser.add_argument("--image", required=True, type=Path)
|
||||
parser.add_argument("--report", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
if os.geteuid() != 0:
|
||||
parser.error("requires root for an owned image/namespace; never installs host services")
|
||||
accept(args.release.resolve(), args.assets.resolve() if args.assets else None,
|
||||
args.image.absolute(), args.report.absolute())
|
||||
args.image.absolute(), args.report.absolute(), args.init.absolute(),
|
||||
args.extension_helper.absolute() if args.extension_helper else None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Actual-asset update transaction in an owned image; never publishes the fixture version."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from verify_prepared_release import InstalledDaemon, execute, install_assets, minimal_os, response
|
||||
|
||||
|
||||
def prepare_update(image, manifest, key, helper):
|
||||
# Reuse real payloads under a newly signed, acceptance-only catalog identity.
|
||||
# This tests the update owner without claiming a second public release exists.
|
||||
next_manifest = dict(manifest)
|
||||
major, minor, patch = map(int, manifest["version"].split("."))
|
||||
version = f"{major}.{minor}.{patch + 1}"
|
||||
next_manifest["version"] = version
|
||||
machine = image / "opt/dever"
|
||||
package = machine / "cache/downloads" / version
|
||||
package.mkdir()
|
||||
installed = machine / "versions" / manifest["version"]
|
||||
for artifact in manifest["artifacts"]:
|
||||
destination = package / artifact["path"]
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.link(installed / artifact["path"], destination)
|
||||
catalog = package / "manifest.json"
|
||||
catalog.write_text(json.dumps(next_manifest, separators=(",", ":")))
|
||||
execute([helper, "--sign-catalog", catalog, key])
|
||||
(machine / "cache/downloads/latest").write_text(version)
|
||||
return version, hashlib.sha256(catalog.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def accept(args):
|
||||
manifest = json.loads((args.release / "manifest.json").read_bytes())
|
||||
result = {"version": manifest["version"], "checks": [],
|
||||
"transport": "local actual assets; acceptance-only next-version catalog",
|
||||
"online_update": "not_run"}
|
||||
|
||||
def passed(name):
|
||||
result["checks"].append(name)
|
||||
args.report.write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(f"PASS {name}", flush=True)
|
||||
|
||||
try:
|
||||
args.image.mkdir()
|
||||
minimal_os(args.image)
|
||||
install_assets(args.assets, args.image, manifest)
|
||||
machine = args.image / "opt/dever"
|
||||
extension = f"runtime-pip-{manifest['platform']}"
|
||||
execute([args.extension_helper, machine, args.assets, manifest["version"], extension])
|
||||
records = args.report.parent / f"{args.report.stem}-process"
|
||||
with InstalledDaemon(args.image, records, args.init) as daemon:
|
||||
daemon.command("new", "/project")
|
||||
response(daemon.command("run", "/project", "--", "hello.greeting.greet", '{"name":"Dever"}'))
|
||||
passed("actual_release_and_selected_extension_before_update")
|
||||
version, catalog = prepare_update(args.image, manifest, args.key, args.extension_helper)
|
||||
result["fixture_version"] = version
|
||||
previous_active = (machine / "state/active-version").read_bytes()
|
||||
empty_assets = args.image / "missing-extension-assets"
|
||||
empty_assets.mkdir()
|
||||
try:
|
||||
execute([args.extension_helper, machine, empty_assets, version, "--update"])
|
||||
except RuntimeError as error:
|
||||
if "cannot open actual extension asset" not in str(error):
|
||||
raise
|
||||
else:
|
||||
raise RuntimeError("update with a missing extension should fail")
|
||||
if (machine / "state/active-version").read_bytes() != previous_active:
|
||||
raise RuntimeError("failed update changed active version")
|
||||
if daemon.command("version") != manifest["version"]:
|
||||
raise RuntimeError("failed update changed the public launcher selection")
|
||||
response(daemon.command("run", "/project", "--", "hello.greeting.greet", '{"name":"Dever"}'))
|
||||
passed("failed_extension_update_keeps_old_active_and_application_working")
|
||||
execute([args.extension_helper, machine, args.assets, version, "--update"])
|
||||
if daemon.command("version") != version:
|
||||
raise RuntimeError("successful update did not activate the new version")
|
||||
if daemon.command("skill", "path") != f"/opt/dever/versions/{version}/skills/dever-language":
|
||||
raise RuntimeError("skill did not follow the updated core")
|
||||
extensions = machine / "cache/extensions" / version / catalog
|
||||
if {path.name for path in extensions.iterdir()} != {extension}:
|
||||
raise RuntimeError("update did not preserve exactly the installed extension selection")
|
||||
passed("successful_update_activates_core_skill_and_only_installed_extensions")
|
||||
# Only the catalog version changes in this transaction fixture.
|
||||
# The managed compiler correctly requires its real build version;
|
||||
# execute it again only after selecting the actual release.
|
||||
daemon.command("use", manifest["version"])
|
||||
response(daemon.command("run", "/project", "--", "hello.greeting.greet", '{"name":"Dever"}'))
|
||||
passed("actual_release_remains_usable_after_update_and_explicit_switch_back")
|
||||
result["status"] = "passed"
|
||||
except BaseException as error:
|
||||
result["status"] = "failed"
|
||||
result["error"] = str(error)
|
||||
raise
|
||||
finally:
|
||||
args.report.write_text(json.dumps(result, indent=2) + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name in ("release", "assets", "image", "report", "init", "extension-helper", "key"):
|
||||
parser.add_argument(f"--{name}", required=True, type=Path)
|
||||
args = parser.parse_args()
|
||||
if os.geteuid() != 0:
|
||||
parser.error("requires root for an owned image; never installs host services")
|
||||
for name, value in vars(args).items():
|
||||
setattr(args, name, value.absolute())
|
||||
args.report.parent.mkdir(parents=True, exist_ok=True)
|
||||
accept(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user