feat: ship slim releases with shared on-demand extensions and locked restore

test/dever-cli-tests/external_libs/build.rs

@@ -280,14 +280,16 @@ fn backend_fixture(mutate: bool) -> LocalRegistry {
             ),
         ],
     );
-    let key = (Ecosystem::Pip, "/pypi/buildhelper/1.0.0/json".into());
-    let mut release: serde_json::Value = serde_json::from_slice(&registry.0[&key]).unwrap();
-    let url = release["urls"][0]["url"].as_str().unwrap().to_owned();
-    release["urls"][0]["digests"]["sha256"] = serde_json::json!(digest(&helper));
+    let helper_source = source_archive(vec![
+        ("buildhelper-1.0.0/pyproject.toml", b"[build-system]\nrequires = []\nbuild-backend = 'backend'\nbackend-path = ['.']\n".to_vec()),
+        ("buildhelper-1.0.0/backend.py", b"import shutil\nfrom pathlib import Path\ndef build_wheel(directory, config_settings=None, metadata_directory=None):\n    name = 'buildhelper-1.0.0-py3-none-any.whl'\n    shutil.copy('result.whl', Path(directory) / name)\n    return name\n".to_vec()),
+        ("buildhelper-1.0.0/result.whl", helper),
+    ]);
+    registry.0.insert((Ecosystem::Pip, "/pypi/buildhelper/1.0.0/json".into()),
+        serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"buildhelper-1.0.0.tar.gz","url":"helper-source","digests":{"sha256":digest(&helper_source)}}]})).unwrap());
     registry
         .0
-        .insert(key, serde_json::to_vec(&release).unwrap());
-    registry.0.insert((Ecosystem::Pip, url), helper);
+        .insert((Ecosystem::Pip, "helper-source".into()), helper_source);
     let original = wheel_fixture::metadata("sample", "1.0.0", &[], &[]);
     let changed = format!("{original}Summary: changed by wheel hook\n");
     let wheel = wheel_fixture::pack(
@@ -334,11 +336,20 @@ def build_wheel(directory, config_settings=None, metadata_directory=None):
         ("sample-1.0.0/changed-metadata", changed.into_bytes()),
         ("sample-1.0.0/result.whl", wheel),
     ];
+    let source = source_archive(sourcefiles);
+    registry.0.insert((Ecosystem::Pip,"/pypi/sample/1.0.0/json".into()),serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"sample-1.0.0.tar.gz","url":"sample-source","digests":{"sha256":digest(&source)}}]})).unwrap());
+    registry
+        .0
+        .insert((Ecosystem::Pip, "sample-source".into()), source);
+    registry
+}
+
+fn source_archive(files: Vec<(&str, Vec<u8>)>) -> Vec<u8> {
     let mut tar = tar::Builder::new(flate2::write::GzEncoder::new(
         Vec::new(),
         flate2::Compression::default(),
     ));
-    for (name, bytes) in sourcefiles {
+    for (name, bytes) in files {
         let mut header = tar::Header::new_gnu();
         header.set_size(bytes.len() as u64);
         header.set_mode(0o644);
@@ -346,12 +357,7 @@ def build_wheel(directory, config_settings=None, metadata_directory=None):
         tar.append_data(&mut header, name, bytes.as_slice())
             .unwrap();
     }
-    let source = tar.into_inner().unwrap().finish().unwrap();
-    registry.0.insert((Ecosystem::Pip,"/pypi/sample/1.0.0/json".into()),serde_json::to_vec(&serde_json::json!({"info":{},"urls":[{"packagetype":"sdist","filename":"sample-1.0.0.tar.gz","url":"sample-source","digests":{"sha256":digest(&source)}}]})).unwrap());
-    registry
-        .0
-        .insert((Ecosystem::Pip, "sample-source".into()), source);
-    registry
+    tar.into_inner().unwrap().finish().unwrap()
 }
 
 #[test]
@@ -383,12 +389,55 @@ fn pep517_hooks_keep_requirements_cli_and_metadata_boundaries() {
         let lock = result.unwrap();
         dever_cli::libs::doctor(&lock).unwrap();
         assert_eq!(lock.builds[0].dynamic_requires, vec!["buildhelper==1.0.0"]);
+        assert_eq!(
+            lock.builds[0].inputs.builds.len(),
+            1,
+            "nested build requirement receipt must be retained"
+        );
+        let before = lock.encode().unwrap();
+        let cold = FixtureArtifactStore::default();
+        let archives = super::restore_tests::archives_only(&lock, &transport);
+        dever_cli::libs::restore::restore_artifacts(
+            &lock,
+            &archives,
+            &cold,
+            &inputs,
+            &inputs.settings.runtime.target,
+            &dever_cli::libs::sumdb::Verifier::official(),
+        )
+        .unwrap();
+        assert_eq!(lock.encode().unwrap(), before);
+        let mut mismatch = lock.clone();
+        mismatch.builds[0].output.sha256 = "0".repeat(64);
+        mismatch.libs[0].artifacts = vec![mismatch.builds[0].output.clone()];
+        mismatch.libs[0].build = Some(mismatch.builds[0].identity().unwrap());
+        let rejected = FixtureArtifactStore::default();
+        assert!(
+            dever_cli::libs::restore::restore_artifacts(
+                &mismatch,
+                &archives,
+                &rejected,
+                &inputs,
+                &inputs.settings.runtime.target,
+                &dever_cli::libs::sumdb::Verifier::official()
+            )
+            .unwrap_err()
+            .contains("not byte reproducible")
+        );
+        assert!(
+            !rejected
+                .has(
+                    &mismatch.builds[0].output.sha256,
+                    &inputs.settings.runtime.target
+                )
+                .unwrap()
+        );
         for mutation in 0..3 {
             let mut altered = lock.clone();
             match mutation {
                 0 => altered.builds[0].tools.sha256 = "not-a-hash".into(),
                 1 => altered.builds[0].dynamic_requires = vec!["buildhelper>=2".into()],
-                _ => *altered.builds[0].dependencies = LockFile::new(vec![]).unwrap(),
+                _ => *altered.builds[0].inputs = LockFile::new(vec![]).unwrap(),
             }
             altered.libs[0].build = Some(altered.builds[0].identity().unwrap());
             assert!(
@@ -441,7 +490,7 @@ fn builds_official_python_source_with_isolated_pep517() {
     let lock = dever_cli::libs::LibResolver::resolve(&resolver, &requested).unwrap();
     dever_cli::libs::doctor(&lock).unwrap();
     assert_eq!(lock.builds.len(), 1);
-    assert!(!lock.builds[0].dependencies.libs.is_empty());
+    assert!(!lock.builds[0].inputs.libs.is_empty());
     let artifact = &lock
         .libs
         .iter()

test/dever-cli-tests/external_libs/npm_build.rs

@@ -65,7 +65,7 @@ fn npm_registry_install_hooks_do_not_run_publishing_lifecycle() {
     );
 }
 
-fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
+fn locked_installation_fixture() -> (LockFile, Vec<u8>, FixtureArtifactStore) {
     let mut registry = LocalRegistry(BTreeMap::new());
     npm_tests::release(
         &mut registry,
@@ -85,11 +85,13 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
             "npm:other@1.0.0".parse().unwrap(),
         ])
         .unwrap();
-    let sources = lock
+    let mut sources: Vec<_> = lock
         .libs
         .iter()
         .flat_map(|lib| lib.artifacts.clone())
         .collect();
+    sources.sort();
+    sources.dedup();
     let archives = lock
         .libs
         .iter()
@@ -159,9 +161,10 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
         ),
         python: None,
         dynamic_requires: vec![],
-        dependencies: Box::new(LockFile::new(vec![]).unwrap()),
+        inputs: Box::new(lock.clone()),
         config_settings: BTreeMap::new(),
         output: dever_cli::libs::LockedArtifact {
+            source: None,
             target: runtime.target,
             path: format!("build/npm/{hash}/installation.tgz"),
             bytes: bytes.len() as u64,
@@ -171,7 +174,24 @@ fn locked_installation_fixture() -> (LockFile, Vec<u8>) {
     lock.npm[0].build = Some(receipt.identity().unwrap());
     lock.builds.push(receipt);
     doctor(&lock).unwrap();
-    (lock, bytes)
+    (lock, bytes, store)
+}
+
+#[test]
+fn cached_npm_build_restore_does_not_load_build_tools_or_auxiliary_runtime() {
+    let (lock, bytes, store) = locked_installation_fixture();
+    store
+        .publish(&bytes, &lock.builds[0].output.target)
+        .unwrap();
+    dever_cli::libs::restore::restore_artifacts(
+        &lock,
+        &LocalRegistry(BTreeMap::new()),
+        &store,
+        &super::restore_tests::RuntimeOnly,
+        "linux-x86_64",
+        &sumdb_tests::verifier(),
+    )
+    .unwrap();
 }
 
 #[test]
@@ -179,7 +199,7 @@ fn npm_expanded_outputs_are_pruned_only_when_no_raw_consumer_remains() {
     use dever_cli::libs::{LockedWorker, prune_expanded_resources};
     use dever_core::native::EmbeddedResource;
 
-    let (mut lock, bytes) = locked_installation_fixture();
+    let (mut lock, bytes, _) = locked_installation_fixture();
     let roots = lock.npm[0].roots.clone();
     let other = vec!["npm:other@1.0.0".parse().unwrap()];
     lock.npm = npm::retain(&lock, &[roots.clone(), other.clone()]).unwrap();
@@ -231,7 +251,7 @@ fn npm_expanded_outputs_are_pruned_only_when_no_raw_consumer_remains() {
 
 #[test]
 fn npm_native_admission_accounts_every_addon_and_prunes_unsupported_objects() {
-    let (lock, bytes) = locked_installation_fixture();
+    let (lock, bytes, _) = locked_installation_fixture();
     let files = dever_cli::libs::build::npm_installation(&lock, &lock.npm[0], &bytes).unwrap();
     assert!(files.iter().all(|(path, _, _)| !path.ends_with(".node")));
     let mut missing = lock.clone();
@@ -262,7 +282,7 @@ fn npm_native_admission_accounts_every_addon_and_prunes_unsupported_objects() {
 
 #[test]
 fn npm_locked_output_projects_retained_roots_without_rerunning_hooks() {
-    let (lock, bytes) = locked_installation_fixture();
+    let (lock, bytes, _) = locked_installation_fixture();
     let project = TemporaryDirectory::new();
     fs::create_dir(project.path().join("config")).unwrap();
     fs::write(
@@ -468,6 +488,33 @@ fn npm_lifecycle_preserves_optional_failures_bins_and_offline_remove() {
             .any(|edge| edge.omission == Some(npm::Omission::BuildFailure))
     );
     let receipt = &lock.builds[0];
+    assert!(
+        receipt
+            .inputs
+            .libs
+            .iter()
+            .any(|lib| lib.spec.name == "broken")
+    );
+    assert!(
+        receipt
+            .inputs
+            .libs
+            .iter()
+            .any(|lib| lib.spec.name == "cycle")
+    );
+    assert!(receipt.inputs.npm[0].build.is_none());
+    let cold = FixtureArtifactStore::default();
+    let before = lock.encode().unwrap();
+    dever_cli::libs::restore::restore_artifacts(
+        &lock,
+        &super::restore_tests::archives_only(&lock, &transport),
+        &cold,
+        &inputs,
+        &inputs.settings.runtime.target,
+        &dever_cli::libs::sumdb::Verifier::official(),
+    )
+    .unwrap();
+    assert_eq!(lock.encode().unwrap(), before);
     let bytes = store
         .verify_exact(
             &receipt.output.sha256,

test/dever-cli-tests/external_libs/restore.rs

@@ -0,0 +1,190 @@
+use super::*;
+use dever_cli::libs::build::{Inputs, pack};
+use dever_cli::libs::restore::restore_artifacts;
+
+pub(super) struct RuntimeOnly;
+
+pub(super) fn archives_only(lock: &LockFile, registry: &LocalRegistry) -> LocalRegistry {
+    fn collect(
+        lock: &LockFile,
+        registry: &LocalRegistry,
+        files: &mut BTreeMap<(Ecosystem, String), Vec<u8>>,
+    ) {
+        for artifact in lock
+            .libs
+            .iter()
+            .flat_map(|lib| &lib.artifacts)
+            .chain(lock.builds.iter().flat_map(|receipt| &receipt.sources))
+        {
+            if let Some(source) = &artifact.source {
+                let key = (source.ecosystem.clone(), source.locator.clone());
+                files.insert(key.clone(), registry.0[&key].clone());
+            }
+        }
+        for receipt in &lock.builds {
+            collect(&receipt.inputs, registry, files);
+        }
+    }
+    let mut files = BTreeMap::new();
+    collect(lock, registry, &mut files);
+    LocalRegistry(files)
+}
+impl Inputs for RuntimeOnly {
+    fn runtime(&self, ecosystem: Ecosystem) -> Result<(RegistryRuntime, Vec<u8>), String> {
+        Ok((registry_runtime(ecosystem), vec![]))
+    }
+    fn tools(&self, _: Ecosystem) -> Result<(pack::Descriptor, Vec<u8>), String> {
+        panic!("archive restoration must not load build extensions")
+    }
+    fn assets(&self) -> Result<Vec<(String, Vec<u8>)>, String> {
+        panic!("archive restoration must not execute a sandbox")
+    }
+}
+
+#[test]
+fn cold_cache_restore_uses_only_locked_sources_and_never_registry_metadata() {
+    let mut registry = local_registry();
+    let original = FixtureArtifactStore::default();
+    let roots = ["pip:a@1.0.0", "npm:root@1.0.0", "go:example.com/a@1.0.0"]
+        .map(|spec| spec.parse().unwrap());
+    let lock = dever_cli::libs::LibResolver::resolve(&local_resolver(&registry, &original), &roots)
+        .unwrap();
+    let before = lock.encode().unwrap();
+    let locators = lock
+        .libs
+        .iter()
+        .flat_map(|lib| &lib.artifacts)
+        .map(|artifact| {
+            let source = artifact.source.as_ref().unwrap();
+            assert!(!source.filename.is_empty());
+            (source.ecosystem.clone(), source.locator.clone())
+        })
+        .collect::<std::collections::BTreeSet<_>>();
+    registry.0.retain(|key, _| locators.contains(key));
+    let cold = FixtureArtifactStore::default();
+    restore_artifacts(
+        &lock,
+        &registry,
+        &cold,
+        &RuntimeOnly,
+        "linux-x86_64",
+        &sumdb_tests::verifier(),
+    )
+    .unwrap();
+    assert_eq!(lock.encode().unwrap(), before);
+    assert_eq!(
+        verify_locked_artifacts(&lock, &cold, "linux-x86_64").unwrap(),
+        verify_locked_artifacts(&lock, &original, "linux-x86_64").unwrap()
+    );
+    // A warm restore must also succeed with no available registry at all.
+    restore_artifacts(
+        &lock,
+        &LocalRegistry(BTreeMap::new()),
+        &cold,
+        &RuntimeOnly,
+        "linux-x86_64",
+        &sumdb_tests::verifier(),
+    )
+    .unwrap();
+}
+
+#[test]
+fn restored_archive_mismatch_fails_before_cache_publication() {
+    let mut registry = local_registry();
+    let lock = dever_cli::libs::LibResolver::resolve(
+        &local_resolver(&registry, &FixtureArtifactStore::default()),
+        &["pip:a@1.0.0".parse().unwrap()],
+    )
+    .unwrap();
+    let source = lock.libs[0].artifacts[0].source.as_ref().unwrap();
+    registry.0.insert(
+        (source.ecosystem.clone(), source.locator.clone()),
+        b"different bytes".to_vec(),
+    );
+    let cold = FixtureArtifactStore::default();
+    assert!(
+        restore_artifacts(
+            &lock,
+            &registry,
+            &cold,
+            &RuntimeOnly,
+            "linux-x86_64",
+            &sumdb_tests::verifier()
+        )
+        .unwrap_err()
+        .contains("byte/hash")
+    );
+    let artifact = &lock.libs[0].artifacts[0];
+    assert!(!cold.has(&artifact.sha256, &artifact.target).unwrap());
+}
+
+#[test]
+fn corrupt_cache_is_an_error_and_never_falls_back_to_download() {
+    struct CorruptCache;
+    impl ArtifactStore for CorruptCache {
+        fn has(&self, _: &str, _: &str) -> Result<bool, String> {
+            unreachable!()
+        }
+        fn publish(&self, _: &[u8], _: &str) -> Result<String, String> {
+            panic!("corrupt cache cannot be replaced")
+        }
+        fn get(&self, _: &str, _: &str) -> Result<Option<Vec<u8>>, String> {
+            Ok(Some(b"corrupt".to_vec()))
+        }
+    }
+    let registry = local_registry();
+    let lock = dever_cli::libs::LibResolver::resolve(
+        &local_resolver(&registry, &FixtureArtifactStore::default()),
+        &["pip:a@1.0.0".parse().unwrap()],
+    )
+    .unwrap();
+    // No registry entries exist: a fallback would produce a distinct missing-path error.
+    assert!(
+        restore_artifacts(
+            &lock,
+            &LocalRegistry(BTreeMap::new()),
+            &CorruptCache,
+            &RuntimeOnly,
+            "linux-x86_64",
+            &sumdb_tests::verifier()
+        )
+        .unwrap_err()
+        .contains("byte/hash")
+    );
+}
+
+#[test]
+fn lib_install_preserves_setting_and_lock_bytes_and_rejects_declaration_drift() {
+    let root = TemporaryDirectory::new();
+    fs::create_dir(root.path().join("config")).unwrap();
+    let setting = b"{\n  \"lib\": [\"pip:fixture@1.0.0\"]\n}\n";
+    fs::write(root.path().join("config/setting.json"), setting).unwrap();
+    assert!(
+        dever_cli::libs::execute("install", root.path(), &[])
+            .unwrap_err()
+            .contains("existing dever.lock")
+    );
+    let lock = FixtureRegistry::builtin()
+        .resolve(&["pip:fixture@1.0.0".parse().unwrap()])
+        .unwrap();
+    lock.write_atomic(root.path()).unwrap();
+    let before = fs::read(root.path().join("dever.lock")).unwrap();
+    dever_cli::libs::execute("install", root.path(), &[]).unwrap();
+    assert_eq!(
+        fs::read(root.path().join("config/setting.json")).unwrap(),
+        setting
+    );
+    assert_eq!(fs::read(root.path().join("dever.lock")).unwrap(), before);
+    assert!(
+        dever_cli::libs::execute("install", root.path(), &["pip:fixture@1.0.0".into()])
+            .unwrap_err()
+            .contains("no Lib specs")
+    );
+    fs::write(root.path().join("config/setting.json"), b"{\"lib\":[]}").unwrap();
+    assert!(
+        dever_cli::libs::execute("install", root.path(), &[])
+            .unwrap_err()
+            .contains("declared closure")
+    );
+    assert_eq!(fs::read(root.path().join("dever.lock")).unwrap(), before);
+}