feat: publish Dever core with project scaffolding and versioned AI skills

test/dever-cli-tests/packages/managed_compile.rs

@@ -0,0 +1,456 @@
+//! Actual managed compilation, using ephemeral signing keys and an owned daemon.
+use super::*;
+use dever_cli::toolchain::compilation::{CompileKind, CompileRequest};
+use dever_cli::toolchain::{BuildTarget, CacheStore, compile};
+use dever_core::source::SourceMap;
+use dever_runtime::config::Settings;
+use std::io::Read;
+use std::os::unix::net::UnixStream;
+use std::time::{Duration, Instant};
+
+fn source_project() -> TemporaryDirectory {
+    let root = project();
+    fs::write(
+        root.path().join("module/portal/value/app.dever"),
+        "value() (result: Text) { result = \"two\" }\n",
+    )
+    .unwrap();
+    root
+}
+
+fn request(root: &Path, kind: CompileKind) -> CompileRequest {
+    let sources = SourceMap::load_with_packages(
+        &root.join("module"),
+        if kind == CompileKind::TestSuite {
+            Some(root.join("test"))
+        } else {
+            None
+        }
+        .as_deref(),
+        &[],
+    )
+    .unwrap();
+    let bindings = (kind == CompileKind::Application)
+        .then(|| Settings::load_project(root).unwrap().compilation_bindings());
+    CompileRequest::new(
+        "0.1.0".into(),
+        kind,
+        &sources,
+        bindings,
+        &[],
+        BuildTarget::host().unwrap(),
+    )
+    .unwrap()
+}
+
+#[test]
+fn compilation_snapshot_excludes_secrets_and_rejects_ambiguous_or_unsafe_inputs() {
+    let root = source_project();
+    fs::write(root.path().join("config/setting.json"), serde_json::to_vec(&serde_json::json!({
+        "database":{"default":{"type":"sqlite","path":"private-database-marker.sqlite"}},
+        "auth":{"providers":{"session":{"verify":"user.account.verify",
+            "jwtSecret":"private-jwt-marker-long-enough-for-hmac-2026","ttlSeconds":3600,"cookie":"secret-cookie-name"}}},
+        "sites":{"admin":{"path":"admin","auth":"session","origin":"https://private-origin.example"}}
+    })).unwrap()).unwrap();
+    let original = request(root.path(), CompileKind::Application);
+    let encoded = original.encode().unwrap();
+    let text = std::str::from_utf8(&encoded).unwrap();
+    for secret in [
+        "private-database-marker",
+        "private-jwt-marker",
+        "secret-cookie-name",
+        "private-origin",
+        root.path().to_str().unwrap(),
+    ] {
+        assert!(!text.contains(secret), "compile input leaked {secret}");
+    }
+    assert!(text.contains("user.account.verify"));
+    assert!(text.contains("sqlite"));
+    CompileRequest::decode(&encoded).unwrap();
+    let duplicate = text.replace("\"databases\":", "\"databases\":{},\"databases\":");
+    assert!(CompileRequest::decode(duplicate.as_bytes()).is_err());
+    for path in ["/tmp/read-me.dever", "../main.dever", "portal/value/app.rs"] {
+        let mut invalid = original.clone();
+        invalid.sources[0].logical_path = path.into();
+        assert!(invalid.encode().is_err(), "accepted source path {path}");
+    }
+    let mut reserved = original.clone();
+    reserved.sources[0].logical_path = "dever/auth/app.dever".into();
+    assert!(
+        dever_core::check(&reserved.source_map().unwrap()).is_err(),
+        "reserved standard namespace bypassed checking"
+    );
+    let mut invalid = original.clone();
+    invalid.sources.push(invalid.sources[0].clone());
+    assert!(invalid.encode().is_err());
+    let mut invalid = original;
+    invalid.sources[0].is_test = true;
+    assert!(invalid.encode().is_err());
+    let suite = request(root.path(), CompileKind::TestSuite);
+    assert!(suite.bindings.is_none());
+    assert!(
+        suite
+            .source_map()
+            .unwrap()
+            .files()
+            .iter()
+            .any(|source| source.is_test())
+    );
+}
+
+#[test]
+fn compilation_target_is_required_canonical_and_part_of_cache_identity() {
+    let root = source_project();
+    let mut input = request(root.path(), CompileKind::Application);
+    input.target = BuildTarget::LinuxX86_64;
+    let host = input.encode().unwrap();
+    input.target = BuildTarget::LinuxAarch64;
+    let arm = input.encode().unwrap();
+    assert_ne!(host, arm);
+    assert_eq!(
+        CompileRequest::decode(&arm).unwrap().target,
+        BuildTarget::LinuxAarch64
+    );
+    let text = std::str::from_utf8(&arm).unwrap();
+    for invalid in [
+        text.replace("\"target\":\"linux-aarch64\",", ""),
+        text.replace("linux-aarch64", "aarch64-unknown-linux-gnu"),
+        text.replace("\"target\":", "\"target\":\"linux-x86_64\",\"target\":"),
+    ] {
+        assert!(CompileRequest::decode(invalid.as_bytes()).is_err());
+    }
+    let mut suite = request(root.path(), CompileKind::TestSuite);
+    suite.target = if BuildTarget::host().unwrap() == BuildTarget::LinuxX86_64 {
+        BuildTarget::LinuxAarch64
+    } else {
+        BuildTarget::LinuxX86_64
+    };
+    assert!(suite.encode().unwrap_err().contains("host target"));
+}
+
+#[test]
+#[ignore = "requires explicitly prepared native archive; starts an owned daemon"]
+fn trusted_compilation_cache_requires_sources_and_revalidates_signed_pack() {
+    let root = source_project();
+    let (layout, _) = installed_dever(root.path());
+    let _daemon = TestDaemon::start(&layout);
+    let input = request(root.path(), CompileKind::Application);
+    let started = Instant::now();
+    let first = compile(&layout, &input).unwrap();
+    let cold_seconds = started.elapsed().as_secs_f64();
+    let initial_cache = cache_status(&layout).unwrap();
+    assert_eq!(initial_cache.entries, 1);
+    let started = Instant::now();
+    assert_eq!(compile(&layout, &input).unwrap(), first);
+    let hit_seconds = started.elapsed().as_secs_f64();
+    assert_eq!(cache_status(&layout).unwrap().entries, 1);
+    assert!(artifact_get(&layout, &digest(&first), first.len() as u64).is_err());
+    let mut invalid = input.clone();
+    invalid
+        .sources
+        .iter_mut()
+        .find(|source| source.logical_path.ends_with("app.dever"))
+        .unwrap()
+        .text = "value() (result: Text) { result = missing_function() }".into();
+    let error = compile(&layout, &invalid).unwrap_err();
+    assert!(error.contains("missing_function"), "{error}");
+    assert_eq!(cache_status(&layout).unwrap().entries, 1);
+    assert_eq!(compile(&layout, &input).unwrap(), first);
+    let mut changed = input.clone();
+    changed
+        .sources
+        .iter_mut()
+        .find(|source| source.logical_path.ends_with("app.dever"))
+        .unwrap()
+        .text = "value() (result: Text) { result = \"three\" }".into();
+    // 两个独立客户端同时提交同一份新源码，必须收敛到一个已验证产物。
+    let barrier = std::sync::Barrier::new(2);
+    let concurrent = std::thread::scope(|scope| {
+        let clients: Vec<_> = (0..2)
+            .map(|_| {
+                scope.spawn(|| {
+                    barrier.wait();
+                    let started = Instant::now();
+                    let bytes = compile(&layout, &changed).unwrap();
+                    (bytes, started.elapsed().as_secs_f64())
+                })
+            })
+            .collect();
+        clients
+            .into_iter()
+            .map(|client| client.join().unwrap())
+            .collect::<Vec<_>>()
+    });
+    assert_eq!(concurrent[0].0, concurrent[1].0);
+    assert_ne!(concurrent[0].0, first);
+    let shared_cache = cache_status(&layout).unwrap();
+    assert_eq!(shared_cache.entries, 2);
+
+    // The worker compiles the suite; only the caller runs its failing case.
+    fs::write(
+        root.path().join("test/portal/value/smoke.dever"),
+        "smoke() () { assert_eq(1, 2) }",
+    )
+    .unwrap();
+    let suite = compile(&layout, &request(root.path(), CompileKind::TestSuite)).unwrap();
+    let program = dever_core::native::NativeProgram::from_verified_bytes(&suite).unwrap();
+    assert!(
+        !Command::new(program.executable())
+            .arg("0")
+            .env_clear()
+            .output()
+            .unwrap()
+            .status
+            .success()
+    );
+    assert_eq!(cache_status(&layout).unwrap().entries, 3);
+
+    // A valid signature over an incomplete list must not authorize unsigned
+    // link inputs, even when the native manifest still names those inputs.
+    let version_root = layout.versions().join("0.1.0");
+    let trusted_key = fs::read(layout.state().join("trusted-release-key")).unwrap();
+    let signed_manifest = fs::read(version_root.join("manifest.json")).unwrap();
+    let signature = fs::read(version_root.join("manifest.sig")).unwrap();
+    let mut manifest: ReleaseManifest = serde_json::from_slice(&signed_manifest).unwrap();
+    manifest
+        .artifacts
+        .retain(|artifact| !artifact.path.ends_with("/crt1.o"));
+    let key_bytes = Ed25519KeyPair::generate_pkcs8(&SystemRandom::new()).unwrap();
+    let key = Ed25519KeyPair::from_pkcs8(key_bytes.as_ref()).unwrap();
+    let bytes = serde_json::to_vec(&manifest).unwrap();
+    fs::write(
+        layout.state().join("trusted-release-key"),
+        hex(key.public_key().as_ref()),
+    )
+    .unwrap();
+    fs::write(version_root.join("manifest.json"), &bytes).unwrap();
+    fs::write(
+        version_root.join("manifest.sig"),
+        hex(key.sign(&bytes).as_ref()),
+    )
+    .unwrap();
+    assert!(
+        compile(&layout, &input)
+            .unwrap_err()
+            .contains("not covered by the signed release")
+    );
+    fs::write(layout.state().join("trusted-release-key"), trusted_key).unwrap();
+    fs::write(version_root.join("manifest.json"), signed_manifest).unwrap();
+    fs::write(version_root.join("manifest.sig"), signature).unwrap();
+
+    let pack = layout.versions().join("0.1.0/runtime/linux-x86_64/crt1.o");
+    fs::write(pack, b"changed signed link input").unwrap();
+    let error = compile(&layout, &input).unwrap_err();
+    assert!(error.contains("integrity"), "{error}");
+    assert_eq!(cache_status(&layout).unwrap().entries, 3);
+    assert_eq!(
+        fs::read_dir(layout.native_cache().join("staging"))
+            .unwrap()
+            .count(),
+        0
+    );
+    assert_eq!(
+        fs::read_dir(layout.native_cache().join("operations"))
+            .unwrap()
+            .count(),
+        0
+    );
+    let report = serde_json::json!({
+        "format": "dever-shared-compile-measurement-v1",
+        "scope": "owned signed compiler and daemon; client wall time includes verification and IPC; warm OS caches",
+        "cold_seconds": cold_seconds,
+        "cache_hit_seconds": hit_seconds,
+        "artifact_bytes": first.len(),
+        "artifact_sha256": digest(&first),
+        "initial_cache": initial_cache,
+        "concurrent_clients": concurrent.len(),
+        "concurrent_miss_seconds": concurrent.iter().map(|(_, seconds)| seconds).collect::<Vec<_>>(),
+        "concurrent_same_artifact": true,
+        "shared_cache": shared_cache,
+    });
+    fs::write(
+        Path::new(env!("CARGO_MANIFEST_DIR"))
+            .join("../../target/closure-shared-compile-measurement.json"),
+        serde_json::to_vec_pretty(&report).unwrap(),
+    )
+    .unwrap();
+}
+
+#[test]
+#[ignore = "requires root and prepared archive; only owned clients change UID"]
+fn different_machine_users_share_compilation_without_private_cache() {
+    use std::os::unix::fs::PermissionsExt;
+    use std::os::unix::process::CommandExt;
+    assert_eq!(rustix::process::geteuid().as_raw(), 0);
+    let root = source_project();
+    let other = source_project();
+    // Only these owned fixtures are shared; the usual temporary root is 0700.
+    fs::set_permissions(root.path(), fs::Permissions::from_mode(0o755)).unwrap();
+    fs::set_permissions(other.path(), fs::Permissions::from_mode(0o755)).unwrap();
+    let (layout, core) = installed_dever(root.path());
+    let _daemon = TestDaemon::start(&layout);
+    assert_cli_success(
+        &core,
+        &[
+            "run",
+            root.path().to_str().unwrap(),
+            "--",
+            "portal.value.show",
+            "{}",
+        ],
+    );
+    assert_eq!(cache_status(&layout).unwrap().entries, 1);
+    let result = Command::new(&core)
+        .uid(65534)
+        .gid(65534)
+        .env_clear()
+        .args([
+            "run",
+            other.path().to_str().unwrap(),
+            "--",
+            "portal.value.show",
+            "{}",
+        ])
+        .output()
+        .unwrap();
+    assert!(
+        result.status.success(),
+        "{}",
+        String::from_utf8_lossy(&result.stderr)
+    );
+    assert!(String::from_utf8_lossy(&result.stdout).contains("two"));
+    assert_eq!(cache_status(&layout).unwrap().entries, 1);
+    assert!(!core.parent().unwrap().join("cache").exists());
+    assert!(!other.path().join("cache").exists());
+}
+
+fn frame(stream: &mut UnixStream) -> serde_json::Value {
+    let mut size = [0; 4];
+    stream.read_exact(&mut size).unwrap();
+    let mut bytes = vec![0; u32::from_be_bytes(size) as usize];
+    stream.read_exact(&mut bytes).unwrap();
+    serde_json::from_slice(&bytes).unwrap()
+}
+
+fn upload(layout: &Layout, input: &CompileRequest) -> UnixStream {
+    let payload = input.encode().unwrap();
+    let mut stream = UnixStream::connect(layout.service_socket()).unwrap();
+    stream
+        .set_read_timeout(Some(Duration::from_secs(10)))
+        .unwrap();
+    let token = fs::read_to_string(layout.service_token()).unwrap();
+    let control = serde_json::to_vec(&serde_json::json!({"token":token.trim(),
+        "operation":{"compile":{"bytes":payload.len()}},"installed":[]}))
+    .unwrap();
+    stream
+        .write_all(&(control.len() as u32).to_be_bytes())
+        .unwrap();
+    stream.write_all(&control).unwrap();
+    assert_eq!(frame(&mut stream)["ok"], true);
+    stream.write_all(&payload).unwrap();
+    stream
+}
+
+fn wait_for<T>(mode: &str, phase: &str, mut action: impl FnMut() -> Option<T>) -> T {
+    let started = Instant::now();
+    loop {
+        if let Some(value) = action() {
+            return value;
+        }
+        assert!(
+            started.elapsed() < Duration::from_secs(10),
+            "owned compilation mode={mode} timed out waiting for {phase}"
+        );
+        std::thread::sleep(Duration::from_millis(10));
+    }
+}
+
+#[test]
+#[ignore = "requires prepared archive, explicit author cc/LLVM inputs; supervises an owned signed sleeper"]
+fn cancellation_disconnect_and_version_lease_reap_worker_and_staging() {
+    let root = source_project();
+    let sleeper = root.path().join("signed-sleeper");
+    let source = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../test/dever-cli-tests/support/compiler_sleeper.c");
+    let status = Command::new("/usr/bin/cc")
+        .env_clear()
+        .args([
+            "-B/usr/bin/",
+            "-O2",
+            "-Wl,--disable-new-dtags,-rpath,$ORIGIN/lib,--no-as-needed",
+            "-L/usr/lib/llvm-18/lib",
+            "-lLLVM-18",
+            "-lstdc++",
+            "-lz",
+            "-lgcc_s",
+        ])
+        .arg(source)
+        .arg("-o")
+        .arg(&sleeper)
+        .status()
+        .unwrap();
+    assert!(
+        status.success(),
+        "explicit native sleeper preparation failed"
+    );
+    let (layout, _) = signed_core(root.path(), &sleeper, true);
+    let _daemon = TestDaemon::start(&layout);
+    for mode in ["cancel", "disconnect", "deadline"] {
+        let mut stream = upload(&layout, &request(root.path(), CompileKind::Application));
+        let pid = wait_for(mode, "worker PID publication", || {
+            fs::read_dir(layout.native_cache().join("staging"))
+                .ok()?
+                .filter_map(Result::ok)
+                .find_map(|entry| {
+                    fs::read_to_string(entry.path().join("worker.pid"))
+                        .ok()?
+                        .parse::<i32>()
+                        .ok()
+                        .filter(|pid| *pid > 0)
+                        .map(|pid| pid.to_string())
+                })
+        });
+        let lock = fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .open(layout.state().join("install.lock"))
+            .unwrap();
+        assert!(
+            fs2::FileExt::try_lock_exclusive(&lock).is_err(),
+            "active compiler version was not leased"
+        );
+        assert!(
+            CacheStore::new(&layout)
+                .clean(&Default::default())
+                .unwrap_err()
+                .contains("busy")
+        );
+        if mode == "cancel" {
+            stream.write_all(&[1]).unwrap();
+            let response = frame(&mut stream);
+            assert_eq!(response["ok"], false);
+            assert!(response["error"].as_str().unwrap().contains("cancel"));
+        } else if mode == "deadline" {
+            stream
+                .set_read_timeout(Some(Duration::from_secs(190)))
+                .unwrap();
+            let response = frame(&mut stream);
+            assert_eq!(response["ok"], false);
+            assert!(response["error"].as_str().unwrap().contains("deadline"));
+        }
+        drop(stream);
+        wait_for(mode, "worker exit and staging/operation cleanup", || {
+            let staging = fs::read_dir(layout.native_cache().join("staging"))
+                .ok()?
+                .count();
+            let operations = fs::read_dir(layout.native_cache().join("operations"))
+                .ok()?
+                .count();
+            (staging == 0 && operations == 0 && !Path::new("/proc").join(&pid).exists())
+                .then_some(())
+        });
+        fs2::FileExt::try_lock_exclusive(&lock).unwrap();
+        fs2::FileExt::unlock(&lock).unwrap();
+        assert_eq!(cache_status(&layout).unwrap().entries, 0);
+    }
+}